Skip to content

Commit 472a5fd

Browse files
ajfloederajfloeder
authored
Merge release v0.1.28
Release v0.1.28
2 parents 09d5492 + ac7e6ef commit 472a5fd

16 files changed

Lines changed: 785 additions & 39 deletions

‎.github/CODEOWNERS‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
# NNF release reviewers — used by the release agent to assign PR reviewers.
2+
# All code owners are assigned as reviewers except the person creating the release.
3+
* @matthew-richerson @bdevcich @roehrich-hpe @ajfloeder

‎.github/agents/release.agent.md‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
---
2+
description: "Use when performing an NNF software release, creating release branches, tagging releases, managing release PRs, running release-all.sh, comparing releases, or finalizing release notes. Covers releases across all NNF repositories."
3+
tools: [execute, read, edit, search, todo, web]
4+
argument-hint: "Specify the release type (patch, minor, or major) — defaults to patch"
5+
---
6+
7+
You are the NNF Release Agent. Your job is to execute NNF software releases by following the documented release process precisely, step by step, with user approval at each gate.
8+
9+
The entire release process is orchestrated by `release-all.sh` (located in `tools/release-all/`). It handles cloning, branching, PR creation, merging, and tagging for all NNF repos. The helper function `nnf_cmd` wraps `release-all.sh` with the correct flags.
10+
11+
## Required Reading
12+
13+
Before starting any release work, read these files completely:
14+
15+
1. [Release Plan](../../tools/release-all/RELEASE-PLAN.md) — The authoritative, self-contained guide for releases (repo table, dependency chain, all phases)
16+
2. [Agent Instructions](../../tools/release-all/Instructions.md) — Error handling specifics, key gotchas, and guidance for updating the plan
17+
18+
## Core Rules
19+
20+
- **ALWAYS wait for explicit user approval before proceeding to the next step.** After every step, state what you did, whether it succeeded, and the evidence. **WAIT for the user.**
21+
- **ONE command per turn.** Run the command for ONE repo, show the full output, state PASS or FAIL with evidence, then STOP. Do not run the next repo's command until the user explicitly says to proceed. This applies even when the plan shows a `for` loop — execute the loop body manually, one repo per turn.
22+
- **ALWAYS run commands in a terminal** so the user can see them.
23+
- **ALWAYS append `2>&1 | cat`** to every `release-all.sh` and `gh` command — no exceptions.
24+
- **ONE repo at a time.** Execute one phase per repo sequentially. **NEVER parallelize** across repos.
25+
- **STOP on failure.** If a command fails, analyze the error and propose a fix. **Do NOT retry blindly.**
26+
- **NEVER use `--force`, `--no-verify`, or destructive operations** without asking the user first.
27+
- If you discover gaps or errors in the plan, **update RELEASE-PLAN.md** and tell the user what you changed.
28+
29+
## Workflow
30+
31+
1. Ask the user: What is the release type? (`patch`, `minor`, or `major` — default `patch`)
32+
2. Read the Release Plan and Agent Instructions completely.
33+
3. Follow the plan step by step, using the todo tool to track progress.
34+
4. At each approval gate, present your evidence and wait.
35+
5. After all repos are released, run `final-release-notes.sh` and `compare-releases.sh`.

‎cmd/main.go‎

Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -104,6 +104,11 @@ func (cmd *DeployCmd) Run(ctx *Context) error {
104104
return nil
105105
}
106106

107+
// Wait for prerequisites that this module needs.
108+
if err := waitForModulePrereqs(ctx, module); err != nil {
109+
return err
110+
}
111+
107112
if err := createSystemConfigFromSOS(ctx, system, module); err != nil {
108113
return err
109114
}
@@ -1044,6 +1049,100 @@ func getExampleOverlay(ctx *Context, system *config.System, module string) (stri
10441049
return "", nil
10451050
}
10461051

1052+
// Modules that require cert-manager webhook to be ready before deploying.
1053+
var modulesCertManager = []string{
1054+
"lustre-fs-operator",
1055+
"dws",
1056+
"nnf-sos",
1057+
}
1058+
1059+
// Modules that require the DWS controller-manager to be ready before deploying,
1060+
// because they use DWS CRDs that have a conversion webhook served by DWS.
1061+
var modulesDWSController = []string{
1062+
"nnf-sos",
1063+
"nnf-dm",
1064+
}
1065+
1066+
// waitForModulePrereqs waits for prerequisite services that a module depends on.
1067+
func waitForModulePrereqs(ctx *Context, module string) error {
1068+
if ctx.DryRun {
1069+
return nil
1070+
}
1071+
1072+
for _, m := range modulesCertManager {
1073+
if m == module {
1074+
if err := waitForCertManagerWebhook(ctx); err != nil {
1075+
return err
1076+
}
1077+
break
1078+
}
1079+
}
1080+
1081+
for _, m := range modulesDWSController {
1082+
if m == module {
1083+
if err := waitForDWSControllerManager(ctx); err != nil {
1084+
return err
1085+
}
1086+
break
1087+
}
1088+
}
1089+
1090+
return nil
1091+
}
1092+
1093+
// waitForCertManagerWebhook waits for the cert-manager webhook to be ready and
1094+
// functional. Modules that create cert-manager Certificate or Issuer resources
1095+
// need this webhook to be operational.
1096+
func waitForCertManagerWebhook(ctx *Context) error {
1097+
fmt.Println(" Waiting for cert-manager webhook...")
1098+
cmd := exec.Command("kubectl", "wait", "deploy", "-n", "cert-manager",
1099+
"--timeout=180s", "cert-manager-webhook",
1100+
"--for", "jsonpath={.status.availableReplicas}=1")
1101+
if _, err := runCommand(ctx, cmd); err != nil {
1102+
return fmt.Errorf("cert-manager webhook deployment not ready: %w", err)
1103+
}
1104+
1105+
// The deployment being available doesn't guarantee the webhook is
1106+
// functional. Verify by attempting a server-side dry-run of an Issuer.
1107+
fmt.Println(" Verifying cert-manager webhook is functional...")
1108+
for attempts := 0; attempts < 60; attempts++ {
1109+
cmd = exec.Command("kubectl", "apply", "--dry-run=server", "-f", "-")
1110+
cmd.Stdin = strings.NewReader(`apiVersion: cert-manager.io/v1
1111+
kind: Issuer
1112+
metadata:
1113+
name: deploy-dryrun-test
1114+
namespace: default
1115+
spec:
1116+
selfSigned: {}
1117+
`)
1118+
if out, err := cmd.CombinedOutput(); err == nil {
1119+
_ = out
1120+
break
1121+
}
1122+
if attempts == 59 {
1123+
return fmt.Errorf("cert-manager webhook not functional after 120s")
1124+
}
1125+
time.Sleep(2 * time.Second)
1126+
}
1127+
1128+
return nil
1129+
}
1130+
1131+
// waitForDWSControllerManager waits for the DWS controller-manager to be fully
1132+
// ready. The DWS controller-manager serves the CRD conversion webhook needed by
1133+
// modules that access DWS custom resources with multiple API versions.
1134+
func waitForDWSControllerManager(ctx *Context) error {
1135+
fmt.Println(" Waiting for DWS controller-manager...")
1136+
cmd := exec.Command("kubectl", "wait", "deploy", "-n", "dws-system",
1137+
"--timeout=180s", "dws-controller-manager",
1138+
"--for", "jsonpath={.status.availableReplicas}=1")
1139+
if _, err := runCommand(ctx, cmd); err != nil {
1140+
return fmt.Errorf("DWS controller-manager not ready: %w", err)
1141+
}
1142+
1143+
return nil
1144+
}
1145+
10471146
func deployModule(ctx *Context, system *config.System, module string) error {
10481147

10491148
cmd := exec.Command("make", "deploy")

‎config/repositories.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ repositories:
4141
master: https://ghcr.io/nearnodeflash/lustre-fs-operator
4242
useRemoteK: false
4343
remoteReference:
44-
build: v0.1.17
44+
build: v0.1.18
4545
url: https://github.com/NearNodeFlash/lustre-fs-operator.git/config/default/?ref=%s
4646
buildConfiguration:
4747
# Environment variables to set when calling any 'make' or 'deploy'

‎dws‎

‎lustre-fs-operator‎

‎nnf-dm‎

Submodule nnf-dm updated 137 files

‎nnf-integration-test‎

‎nnf-sos‎

‎tools/kind.sh‎

Lines changed: 113 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
#!/bin/bash
22

3-
# Copyright 2021-2024 Hewlett Packard Enterprise Development LP
3+
# Copyright 2021-2026 Hewlett Packard Enterprise Development LP
44
# Other additional copyright holders may be indicated within.
55
#
66
# The entirety of this work is licensed under the Apache License,
@@ -22,6 +22,42 @@
2222
set -e
2323
set -o pipefail
2424

25+
usage() {
26+
echo "Usage: $0 [--no-argocd] <CMD>"
27+
echo
28+
echo "Options:"
29+
echo " --no-argocd Deploy without ArgoCD using the legacy overlay."
30+
echo " Installs cert-manager, mpi-operator, and lustre"
31+
echo " operators directly instead of via ArgoCD."
32+
echo
33+
echo "Commands:"
34+
echo " create Create a new KIND cluster."
35+
echo " destroy Destroy the existing KIND cluster."
36+
echo " reset Destroy the existing KIND cluster and create"
37+
echo " a new cluster. Note: locally-built images are"
38+
echo " not automatically reloaded -- re-run 'push' or"
39+
echo " './nnf-deploy make kind-push' after a reset."
40+
echo " push Execute 'make kind-push' in each submodule dir."
41+
echo " argocd_attach <new_password>"
42+
echo " Login to the argocd instance and set the new"
43+
echo " password. Then add the Git repo to the instance."
44+
echo " argocd_login <new_password>"
45+
echo " Only do the login to the argocd instance and set"
46+
echo " the password."
47+
echo " argocd_add_git_repo Only add the git repo to the argocd instance."
48+
echo " help Show this help message."
49+
}
50+
51+
NO_ARGOCD=
52+
while [[ "$1" == --* ]]; do
53+
case "$1" in
54+
--no-argocd) NO_ARGOCD=1 ;;
55+
--help) usage; exit 0 ;;
56+
*) echo "Unknown option: $1"; exit 1 ;;
57+
esac
58+
shift
59+
done
60+
2561
CMD="$1"
2662
shift
2763

@@ -32,6 +68,67 @@ function clear_mock_fs {
3268
fi
3369
}
3470

71+
function inject_ca_certs {
72+
local certs="$KIND_CA_CERTS"
73+
local tmp_certs=""
74+
75+
# If KIND_CA_CERTS is not set, auto-extract system CA certs.
76+
if [[ -z "$certs" ]]; then
77+
tmp_certs=$(mktemp /tmp/kind-ca-certs.XXXXXX.pem)
78+
if [[ "$(uname)" == "Darwin" ]]; then
79+
if [[ -f /Library/Keychains/System.keychain ]]; then
80+
security find-certificate -a -p /Library/Keychains/System.keychain > "$tmp_certs" 2>/dev/null
81+
fi
82+
else
83+
# Linux: copy the system CA bundle.
84+
if [[ -f /etc/ssl/certs/ca-certificates.crt ]]; then
85+
cp /etc/ssl/certs/ca-certificates.crt "$tmp_certs"
86+
elif [[ -f /etc/pki/tls/certs/ca-bundle.crt ]]; then
87+
cp /etc/pki/tls/certs/ca-bundle.crt "$tmp_certs"
88+
fi
89+
fi
90+
91+
if [[ ! -s "$tmp_certs" ]]; then
92+
echo "WARNING: Could not extract system CA certificates. Set KIND_CA_CERTS to a PEM file if image pulls fail with TLS errors."
93+
rm -f "$tmp_certs"
94+
return
95+
fi
96+
certs="$tmp_certs"
97+
fi
98+
99+
if [[ ! -f "$certs" ]]; then
100+
echo "ERROR: KIND_CA_CERTS file not found: $certs"
101+
return 1
102+
fi
103+
104+
echo "Injecting CA certificates from $certs into KIND nodes..."
105+
for node in $(kind get nodes); do
106+
docker cp "$certs" "$node:/usr/local/share/ca-certificates/extra-ca-certs.crt"
107+
docker exec "$node" update-ca-certificates
108+
docker exec "$node" systemctl restart containerd
109+
echo " $node: done"
110+
done
111+
112+
# Clean up temp file if we created one.
113+
[[ -n "$tmp_certs" ]] && rm -f "$tmp_certs"
114+
}
115+
116+
# Create the integration test user on KIND Rabbit worker nodes.
117+
# The nnf-integration-test suite verifies that UID/GID exist on Rabbit nodes
118+
# (defaults: UID=1051 GID=1052, the flux user). Without this, the BeforeSuite
119+
# check fails and no tests run.
120+
function create_test_user {
121+
local uid=${NNF_USER_ID:-1051}
122+
local gid=${NNF_GROUP_ID:-1052}
123+
124+
echo "Creating test user (UID=$uid, GID=$gid) on KIND Rabbit worker nodes..."
125+
for node in $(kind get nodes | grep -v control-plane); do
126+
# Create group and user if they don't already exist
127+
docker exec "$node" sh -c "getent group $gid >/dev/null 2>&1 || groupadd -g $gid testgroup"
128+
docker exec "$node" sh -c "getent passwd $uid >/dev/null 2>&1 || useradd -u $uid -g $gid -M -s /bin/bash testuser"
129+
done
130+
}
131+
35132
function create_cluster {
36133
CONFIG=kind-config.yaml
37134

@@ -92,10 +189,23 @@ EOF
92189

93190
kind create cluster --wait 60s --image=kindest/node:v1.31.2 --config $CONFIG
94191

192+
# If corporate/custom CA certificates are available, inject them into
193+
# each KIND node so containerd can pull from registries behind a TLS
194+
# intercepting proxy. Set KIND_CA_CERTS to a PEM file path, or it
195+
# defaults to /tmp/corporate-certs.pem.
196+
inject_ca_certs
197+
95198
# Use the same init routines that we use on real hardware.
96199
# This applies taints and labels to rabbit nodes, and installs other
97200
# services that rabbit software requires.
201+
if [[ -n $NO_ARGOCD ]]; then
202+
echo "Deploying without ArgoCD (legacy overlay mode)..."
203+
cp config/overlay-legacy.yaml-template overlay-legacy.yaml
204+
fi
98205
./nnf-deploy init
206+
207+
# Create the test user on Rabbit nodes for integration tests
208+
create_test_user
99209
}
100210

101211
function have_argocd {
@@ -180,23 +290,7 @@ function push_submodules {
180290
done
181291
}
182292

183-
usage() {
184-
echo "Usage: $0 <CMD>"
185-
echo
186-
echo " create Create a new KIND cluster."
187-
echo " destroy Destroy the existing KIND cluster."
188-
echo " reset Destroy the existing KIND cluster and create"
189-
echo " a new cluster."
190-
echo " push Execute 'make kind-push' in each submodule dir."
191-
echo " argocd_attach <new_password>"
192-
echo " Login to the argocd instance and set the new"
193-
echo " password. Then add the Git repo to the instance."
194-
echo " argocd_login <new_password>"
195-
echo " Only do the login to the argocd instance and set"
196-
echo " the password."
197-
echo " argocd_add_git_repo Only add the git repo to the argocd instance."
198293

199-
}
200294

201295
if [[ "$CMD" == "create" ]]; then
202296
create_cluster
@@ -213,6 +307,8 @@ elif [[ "$CMD" == "argocd_login" ]]; then
213307
argocd_login "$*"
214308
elif [[ "$CMD" == "argocd_add_git_repo" ]]; then
215309
argocd_add_git_repo
310+
elif [[ "$CMD" == "help" || "$CMD" == "--help" ]]; then
311+
usage
216312
else
217313
usage
218314
exit 1

0 commit comments

Comments
 (0)