diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 00000000..625e1a55 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,3 @@ +# NNF release reviewers — used by the release agent to assign PR reviewers. +# All code owners are assigned as reviewers except the person creating the release. +* @matthew-richerson @bdevcich @roehrich-hpe @ajfloeder diff --git a/.github/agents/release.agent.md b/.github/agents/release.agent.md new file mode 100644 index 00000000..8c3ab7b6 --- /dev/null +++ b/.github/agents/release.agent.md @@ -0,0 +1,35 @@ +--- +description: "Use when performing an NNF software release, creating release branches, tagging releases, managing release PRs, running release-all.sh, comparing releases, or finalizing release notes. Covers releases across all NNF repositories." +tools: [execute, read, edit, search, todo, web] +argument-hint: "Specify the release type (patch, minor, or major) — defaults to patch" +--- + +You are the NNF Release Agent. Your job is to execute NNF software releases by following the documented release process precisely, step by step, with user approval at each gate. + +The entire release process is orchestrated by `release-all.sh` (located in `tools/release-all/`). It handles cloning, branching, PR creation, merging, and tagging for all NNF repos. The helper function `nnf_cmd` wraps `release-all.sh` with the correct flags. + +## Required Reading + +Before starting any release work, read these files completely: + +1. [Release Plan](../../tools/release-all/RELEASE-PLAN.md) — The authoritative, self-contained guide for releases (repo table, dependency chain, all phases) +2. [Agent Instructions](../../tools/release-all/Instructions.md) — Error handling specifics, key gotchas, and guidance for updating the plan + +## Core Rules + +- **ALWAYS wait for explicit user approval before proceeding to the next step.** After every step, state what you did, whether it succeeded, and the evidence. **WAIT for the user.** +- **ONE command per turn.** Run the command for ONE repo, show the full output, state PASS or FAIL with evidence, then STOP. Do not run the next repo's command until the user explicitly says to proceed. This applies even when the plan shows a `for` loop — execute the loop body manually, one repo per turn. +- **ALWAYS run commands in a terminal** so the user can see them. +- **ALWAYS append `2>&1 | cat`** to every `release-all.sh` and `gh` command — no exceptions. +- **ONE repo at a time.** Execute one phase per repo sequentially. **NEVER parallelize** across repos. +- **STOP on failure.** If a command fails, analyze the error and propose a fix. **Do NOT retry blindly.** +- **NEVER use `--force`, `--no-verify`, or destructive operations** without asking the user first. +- If you discover gaps or errors in the plan, **update RELEASE-PLAN.md** and tell the user what you changed. + +## Workflow + +1. Ask the user: What is the release type? (`patch`, `minor`, or `major` — default `patch`) +2. Read the Release Plan and Agent Instructions completely. +3. Follow the plan step by step, using the todo tool to track progress. +4. At each approval gate, present your evidence and wait. +5. After all repos are released, run `final-release-notes.sh` and `compare-releases.sh`. diff --git a/cmd/main.go b/cmd/main.go index 89e322f5..ede6beec 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -104,6 +104,11 @@ func (cmd *DeployCmd) Run(ctx *Context) error { return nil } + // Wait for prerequisites that this module needs. + if err := waitForModulePrereqs(ctx, module); err != nil { + return err + } + if err := createSystemConfigFromSOS(ctx, system, module); err != nil { return err } @@ -1044,6 +1049,100 @@ func getExampleOverlay(ctx *Context, system *config.System, module string) (stri return "", nil } +// Modules that require cert-manager webhook to be ready before deploying. +var modulesCertManager = []string{ + "lustre-fs-operator", + "dws", + "nnf-sos", +} + +// Modules that require the DWS controller-manager to be ready before deploying, +// because they use DWS CRDs that have a conversion webhook served by DWS. +var modulesDWSController = []string{ + "nnf-sos", + "nnf-dm", +} + +// waitForModulePrereqs waits for prerequisite services that a module depends on. +func waitForModulePrereqs(ctx *Context, module string) error { + if ctx.DryRun { + return nil + } + + for _, m := range modulesCertManager { + if m == module { + if err := waitForCertManagerWebhook(ctx); err != nil { + return err + } + break + } + } + + for _, m := range modulesDWSController { + if m == module { + if err := waitForDWSControllerManager(ctx); err != nil { + return err + } + break + } + } + + return nil +} + +// waitForCertManagerWebhook waits for the cert-manager webhook to be ready and +// functional. Modules that create cert-manager Certificate or Issuer resources +// need this webhook to be operational. +func waitForCertManagerWebhook(ctx *Context) error { + fmt.Println(" Waiting for cert-manager webhook...") + cmd := exec.Command("kubectl", "wait", "deploy", "-n", "cert-manager", + "--timeout=180s", "cert-manager-webhook", + "--for", "jsonpath={.status.availableReplicas}=1") + if _, err := runCommand(ctx, cmd); err != nil { + return fmt.Errorf("cert-manager webhook deployment not ready: %w", err) + } + + // The deployment being available doesn't guarantee the webhook is + // functional. Verify by attempting a server-side dry-run of an Issuer. + fmt.Println(" Verifying cert-manager webhook is functional...") + for attempts := 0; attempts < 60; attempts++ { + cmd = exec.Command("kubectl", "apply", "--dry-run=server", "-f", "-") + cmd.Stdin = strings.NewReader(`apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: deploy-dryrun-test + namespace: default +spec: + selfSigned: {} +`) + if out, err := cmd.CombinedOutput(); err == nil { + _ = out + break + } + if attempts == 59 { + return fmt.Errorf("cert-manager webhook not functional after 120s") + } + time.Sleep(2 * time.Second) + } + + return nil +} + +// waitForDWSControllerManager waits for the DWS controller-manager to be fully +// ready. The DWS controller-manager serves the CRD conversion webhook needed by +// modules that access DWS custom resources with multiple API versions. +func waitForDWSControllerManager(ctx *Context) error { + fmt.Println(" Waiting for DWS controller-manager...") + cmd := exec.Command("kubectl", "wait", "deploy", "-n", "dws-system", + "--timeout=180s", "dws-controller-manager", + "--for", "jsonpath={.status.availableReplicas}=1") + if _, err := runCommand(ctx, cmd); err != nil { + return fmt.Errorf("DWS controller-manager not ready: %w", err) + } + + return nil +} + func deployModule(ctx *Context, system *config.System, module string) error { cmd := exec.Command("make", "deploy") diff --git a/config/repositories.yaml b/config/repositories.yaml index 641ae8f2..cd59bc75 100644 --- a/config/repositories.yaml +++ b/config/repositories.yaml @@ -41,7 +41,7 @@ repositories: master: https://ghcr.io/nearnodeflash/lustre-fs-operator useRemoteK: false remoteReference: - build: v0.1.17 + build: v0.1.18 url: https://github.com/NearNodeFlash/lustre-fs-operator.git/config/default/?ref=%s buildConfiguration: # Environment variables to set when calling any 'make' or 'deploy' diff --git a/dws b/dws index b3a25a36..64087888 160000 --- a/dws +++ b/dws @@ -1 +1 @@ -Subproject commit b3a25a361a1f94837608b693b824e612b00d09f4 +Subproject commit 640878885c332227ce72ec39e283e8e41e4f3315 diff --git a/lustre-fs-operator b/lustre-fs-operator index 941f0d19..9bcf1c18 160000 --- a/lustre-fs-operator +++ b/lustre-fs-operator @@ -1 +1 @@ -Subproject commit 941f0d195a6626051785bcdb74ec0980b52d739f +Subproject commit 9bcf1c18142c32fd8aa333ba7b009091c7a1b27c diff --git a/nnf-dm b/nnf-dm index 52237ef7..d0b0d4e5 160000 --- a/nnf-dm +++ b/nnf-dm @@ -1 +1 @@ -Subproject commit 52237ef70735109048047301d49451dc722a2c95 +Subproject commit d0b0d4e5b4ec601b40185e88213227631299cc9b diff --git a/nnf-integration-test b/nnf-integration-test index fadb5f42..fc761762 160000 --- a/nnf-integration-test +++ b/nnf-integration-test @@ -1 +1 @@ -Subproject commit fadb5f4268c8491b6f7f6040a6fb781399dd71cb +Subproject commit fc761762b8d28f68cdbd3260a6d4b0e55437a70b diff --git a/nnf-sos b/nnf-sos index f0ca0f4b..1acd3ed8 160000 --- a/nnf-sos +++ b/nnf-sos @@ -1 +1 @@ -Subproject commit f0ca0f4bdc2adcf06629d6bda731082058b096cf +Subproject commit 1acd3ed884bb253a409b004d04c9be60a4783de8 diff --git a/tools/kind.sh b/tools/kind.sh index 18f5acbe..be4620d7 100755 --- a/tools/kind.sh +++ b/tools/kind.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright 2021-2024 Hewlett Packard Enterprise Development LP +# Copyright 2021-2026 Hewlett Packard Enterprise Development LP # Other additional copyright holders may be indicated within. # # The entirety of this work is licensed under the Apache License, @@ -22,6 +22,42 @@ set -e set -o pipefail +usage() { + echo "Usage: $0 [--no-argocd] " + echo + echo "Options:" + echo " --no-argocd Deploy without ArgoCD using the legacy overlay." + echo " Installs cert-manager, mpi-operator, and lustre" + echo " operators directly instead of via ArgoCD." + echo + echo "Commands:" + echo " create Create a new KIND cluster." + echo " destroy Destroy the existing KIND cluster." + echo " reset Destroy the existing KIND cluster and create" + echo " a new cluster. Note: locally-built images are" + echo " not automatically reloaded -- re-run 'push' or" + echo " './nnf-deploy make kind-push' after a reset." + echo " push Execute 'make kind-push' in each submodule dir." + echo " argocd_attach " + echo " Login to the argocd instance and set the new" + echo " password. Then add the Git repo to the instance." + echo " argocd_login " + echo " Only do the login to the argocd instance and set" + echo " the password." + echo " argocd_add_git_repo Only add the git repo to the argocd instance." + echo " help Show this help message." +} + +NO_ARGOCD= +while [[ "$1" == --* ]]; do + case "$1" in + --no-argocd) NO_ARGOCD=1 ;; + --help) usage; exit 0 ;; + *) echo "Unknown option: $1"; exit 1 ;; + esac + shift +done + CMD="$1" shift @@ -32,6 +68,67 @@ function clear_mock_fs { fi } +function inject_ca_certs { + local certs="$KIND_CA_CERTS" + local tmp_certs="" + + # If KIND_CA_CERTS is not set, auto-extract system CA certs. + if [[ -z "$certs" ]]; then + tmp_certs=$(mktemp /tmp/kind-ca-certs.XXXXXX.pem) + if [[ "$(uname)" == "Darwin" ]]; then + if [[ -f /Library/Keychains/System.keychain ]]; then + security find-certificate -a -p /Library/Keychains/System.keychain > "$tmp_certs" 2>/dev/null + fi + else + # Linux: copy the system CA bundle. + if [[ -f /etc/ssl/certs/ca-certificates.crt ]]; then + cp /etc/ssl/certs/ca-certificates.crt "$tmp_certs" + elif [[ -f /etc/pki/tls/certs/ca-bundle.crt ]]; then + cp /etc/pki/tls/certs/ca-bundle.crt "$tmp_certs" + fi + fi + + if [[ ! -s "$tmp_certs" ]]; then + echo "WARNING: Could not extract system CA certificates. Set KIND_CA_CERTS to a PEM file if image pulls fail with TLS errors." + rm -f "$tmp_certs" + return + fi + certs="$tmp_certs" + fi + + if [[ ! -f "$certs" ]]; then + echo "ERROR: KIND_CA_CERTS file not found: $certs" + return 1 + fi + + echo "Injecting CA certificates from $certs into KIND nodes..." + for node in $(kind get nodes); do + docker cp "$certs" "$node:/usr/local/share/ca-certificates/extra-ca-certs.crt" + docker exec "$node" update-ca-certificates + docker exec "$node" systemctl restart containerd + echo " $node: done" + done + + # Clean up temp file if we created one. + [[ -n "$tmp_certs" ]] && rm -f "$tmp_certs" +} + +# Create the integration test user on KIND Rabbit worker nodes. +# The nnf-integration-test suite verifies that UID/GID exist on Rabbit nodes +# (defaults: UID=1051 GID=1052, the flux user). Without this, the BeforeSuite +# check fails and no tests run. +function create_test_user { + local uid=${NNF_USER_ID:-1051} + local gid=${NNF_GROUP_ID:-1052} + + echo "Creating test user (UID=$uid, GID=$gid) on KIND Rabbit worker nodes..." + for node in $(kind get nodes | grep -v control-plane); do + # Create group and user if they don't already exist + docker exec "$node" sh -c "getent group $gid >/dev/null 2>&1 || groupadd -g $gid testgroup" + docker exec "$node" sh -c "getent passwd $uid >/dev/null 2>&1 || useradd -u $uid -g $gid -M -s /bin/bash testuser" + done +} + function create_cluster { CONFIG=kind-config.yaml @@ -92,10 +189,23 @@ EOF kind create cluster --wait 60s --image=kindest/node:v1.31.2 --config $CONFIG + # If corporate/custom CA certificates are available, inject them into + # each KIND node so containerd can pull from registries behind a TLS + # intercepting proxy. Set KIND_CA_CERTS to a PEM file path, or it + # defaults to /tmp/corporate-certs.pem. + inject_ca_certs + # Use the same init routines that we use on real hardware. # This applies taints and labels to rabbit nodes, and installs other # services that rabbit software requires. + if [[ -n $NO_ARGOCD ]]; then + echo "Deploying without ArgoCD (legacy overlay mode)..." + cp config/overlay-legacy.yaml-template overlay-legacy.yaml + fi ./nnf-deploy init + + # Create the test user on Rabbit nodes for integration tests + create_test_user } function have_argocd { @@ -180,23 +290,7 @@ function push_submodules { done } -usage() { - echo "Usage: $0 " - echo - echo " create Create a new KIND cluster." - echo " destroy Destroy the existing KIND cluster." - echo " reset Destroy the existing KIND cluster and create" - echo " a new cluster." - echo " push Execute 'make kind-push' in each submodule dir." - echo " argocd_attach " - echo " Login to the argocd instance and set the new" - echo " password. Then add the Git repo to the instance." - echo " argocd_login " - echo " Only do the login to the argocd instance and set" - echo " the password." - echo " argocd_add_git_repo Only add the git repo to the argocd instance." -} if [[ "$CMD" == "create" ]]; then create_cluster @@ -213,6 +307,8 @@ elif [[ "$CMD" == "argocd_login" ]]; then argocd_login "$*" elif [[ "$CMD" == "argocd_add_git_repo" ]]; then argocd_add_git_repo +elif [[ "$CMD" == "help" || "$CMD" == "--help" ]]; then + usage else usage exit 1 diff --git a/tools/release-all/Instructions.md b/tools/release-all/Instructions.md new file mode 100644 index 00000000..a8887992 --- /dev/null +++ b/tools/release-all/Instructions.md @@ -0,0 +1,11 @@ +# Agent Instructions for NNF Software Release + +> This file supplements `RELEASE-PLAN.md` (the authoritative reference) with agent-specific guidance. **All procedural steps and gotchas are in the plan — do not duplicate them here.** + +## Updating the Plan + +If you encounter an issue not covered by RELEASE-PLAN.md, or if a step's success criteria are unclear: + +1. Resolve the issue with the user's guidance. +2. Update RELEASE-PLAN.md with what you learned. +3. Note the update to the user. diff --git a/tools/release-all/RELEASE-PLAN.md b/tools/release-all/RELEASE-PLAN.md new file mode 100644 index 00000000..66af28cf --- /dev/null +++ b/tools/release-all/RELEASE-PLAN.md @@ -0,0 +1,307 @@ +# NNF Software Release Plan + +This document is the authoritative, self-contained guide for executing an NNF software release. All release phases are driven by `release-all.sh` in this directory, wrapped by the `nnf_cmd` helper function. The release process is the same regardless of how many repos have changes — the `release` phase automatically detects which repos have new commits since the last release and skips those that don't. An agent or human following this document should be able to execute the entire release without external context. + +--- + +## Repository Reference + +| Script Name | GitHub Repo | Owner | Default Branch | Notes | +| --- | --- | --- | --- | --- | +| `dws` | `DataWorkflowServices/dws` | DataWorkflowServices | `master` | | +| `lustre_csi_driver` | `HewlettPackard/lustre-csi-driver` | HewlettPackard | `master` | | +| `lustre_fs_operator` | `NearNodeFlash/lustre-fs-operator` | NearNodeFlash | `master` | | +| `nnf_mfu` | `NearNodeFlash/nnf-mfu` | NearNodeFlash | `master` | Standalone, not a submodule | +| `nnf_ec` | `NearNodeFlash/nnf-ec` | NearNodeFlash | `master` | Standalone, not a submodule | +| `nnf_storedversions_maint` | `NearNodeFlash/nnf-storedversions-maint` | NearNodeFlash | **`main`** (not master) | Standalone, not a submodule | +| `nnf_sos` | `NearNodeFlash/nnf-sos` | NearNodeFlash | `master` | Requires `-M` flag | +| `nnf_dm` | `NearNodeFlash/nnf-dm` | NearNodeFlash | `master` | Vendors nnf-sos | +| `nnf_integration_test` | `NearNodeFlash/nnf-integration-test` | NearNodeFlash | `master` | Vendors nnf-sos | +| `nnf_deploy` | `NearNodeFlash/nnf-deploy` | NearNodeFlash | `master` | Has submodules for most repos | +| `nnf_doc` | `NearNodeFlash/NearNodeFlash.github.io` | NearNodeFlash | **`main`** (not master) | Released last; tracks nnf-deploy version | + +**PR Reviewers:** Defined in `.github/CODEOWNERS`. Assign all code owners **except** the person creating the release. The `setup-release-env.sh` helper reads CODEOWNERS automatically and sets `$RELEASE_REVIEWERS`. + +**Dependency chain (release order):** + +```text +dws → lustre_csi_driver → lustre_fs_operator → nnf_mfu → nnf_ec → nnf_storedversions_maint → nnf_sos → nnf_dm → nnf_integration_test → nnf_deploy → nnf_doc +``` + +**Vendoring dependencies (peer modules in `go.mod`):** + +- `lustre_fs_operator` vendors `dws` +- `nnf_sos` vendors `dws`, `lustre-fs-operator`, `nnf-ec` +- `nnf_dm` vendors `dws`, `lustre-fs-operator`, `nnf-ec` (indirect), `nnf-sos` +- `nnf_integration_test` vendors `dws`, `lustre-fs-operator`, `nnf-ec` (indirect), `nnf-sos` +- `nnf_storedversions_maint` has **no** NNF peer vendoring dependencies + +If an upstream repo changes, **all downstream repos that vendor it** may need revendoring before the release (see Step 2a). The vendoring check in Step 2 catches these automatically. + +--- + +## Release Execution Plan + +### Prerequisites (Pre-flight Checklist) + +1. Ensure these tools are installed: `gh`, `yq` (Go version v4.x), `jq`, `perl`, `git`, `make`, `tput`, `sed` +2. Set `GH_TOKEN` env var with a GitHub **classic** token (not fine-grained) with `repo` scope. The token is 40 characters, starting with `ghp_`. +3. Verify `gh` authentication: `gh auth status`. Confirm it shows the expected user and token scopes. +4. Verify SSH access to all repo URLs (`ssh -T git@github.com`) +5. Decide the release type (`-B major|minor|patch`). **Always pass `-B` explicitly** — do not rely on the script default, which could change. Most releases use `-B patch`. +6. **Pre-release vendoring check:** If any upstream repo has had changes merged to master since the last release that affect downstream vendoring, ensure downstream repos have been revendored **before** starting the release. For example, if `nnf-sos` changed, then `nnf-dm` and `nnf-integration-test` (which vendor `nnf-sos`) must have their vendor directories updated via PRs merged to master. Phase 1's vendoring checks will catch any mismatches. + +### Phase 0: Fresh Clone + +> The release uses a **volatile** working directory that is deleted and recreated at the start of each release. Set `$RELEASE_WORKDIR` to the desired location (e.g., `RELEASE_WORKDIR=~/release-work`). + +#### Step 0 — Clone nnf-deploy with submodules + +If `$RELEASE_WORKDIR` already exists, confirm with the user that it can be deleted. **If they say no, halt the release process.** + +```sh +# Set the working directory for this release +export RELEASE_WORKDIR=~/release-work # adjust as needed + +# If $RELEASE_WORKDIR exists, ask the user before proceeding. +# If they decline, stop here. + +# Clean build artifacts before removing (rm -rf can fail on build outputs) +if [ -d "$RELEASE_WORKDIR/nnf-deploy/workingspace" ]; then + for repo in "$RELEASE_WORKDIR"/nnf-deploy/workingspace/*/; do + ( cd "$repo" && make clean-bin 2>/dev/null || true ) + done +fi + +rm -rf "$RELEASE_WORKDIR" +mkdir -p "$RELEASE_WORKDIR" +cd "$RELEASE_WORKDIR" +git clone --recurse-submodules git@github.com:NearNodeFlash/nnf-deploy.git +cd nnf-deploy +``` + +All subsequent steps run from `$RELEASE_WORKDIR/nnf-deploy`. + +### General Notes + +> **Helper functions:** After sourcing `setup-release-env.sh`, use `nnf_cmd ` for all `release-all.sh` invocations. It automatically handles the `-B` flag, the `-M` flag for `nnf_sos`, and pipes through `cat` to avoid pager issues. Use `nnf_create_pr ` for the `create-pr` phase (captures `$PR_NUMBER`) and `nnf_add_reviewers ` to assign reviewers. +> +> **Manual invocations:** If you need to call `release-all.sh` directly, always pass `-B patch` (or `-B minor`/`-B major`) explicitly, add `-M` for `nnf_sos`, and append `2>&1 | cat`. + +### Phase 1: Discovery & Validation + +#### Step 1 — Set up and list repos + +```sh +cd tools/release-all +``` + +Source the release environment helper to auto-compute versions, detect the release operator, and set reviewers: + +```sh +source ./setup-release-env.sh -B patch # or -B minor / -B major +``` + +> **Important:** Never pipe this `source` command (e.g. `source ... | cat`). Piping runs `source` in a subshell, which means the exported variables and function definitions (`nnf_cmd`, etc.) are lost to the calling shell. If you need to capture output, redirect to a file or run without the pipe. + +This exports `$PREVIOUS_RELEASE`, `$NNF_RELEASE`, `$RELEASE_TYPE`, and `$RELEASE_REVIEWERS`, and defines helper functions `nnf_cmd`, `nnf_create_pr`, `nnf_add_reviewers`, and `nnf_gh_repo`. Confirm the printed summary with the user before proceeding. + +List repos: + +```sh +./release-all.sh -L +``` + +Save the ordered list: `dws`, `lustre_csi_driver`, `lustre_fs_operator`, `nnf_mfu`, `nnf_ec`, `nnf_storedversions_maint`, `nnf_sos`, `nnf_dm`, `nnf_integration_test`, `nnf_deploy`, `nnf_doc`. + +#### Step 2 — Check vendoring *(sequential, per repo)* + +> Must be error-free before proceeding. This runs on **all 10 repos** including `nnf_doc` — vendoring checks validate the current state of master/main, which is independent of the release branching. + +```sh +for repo in $(./release-all.sh -L); do + nnf_cmd master "$repo" +done +``` + +> **Important:** Run vendoring checks on **all** repos, not just the ones you plan to release. This catches stale submodule pointers in `nnf-deploy` (e.g., a force-push on a submodule repo can leave the pointer at an orphaned commit SHA). + +#### Step 2a — Fix stale vendoring *(if Step 2 fails)* + +If the vendoring check for a repo reports **"Peer modules are behind"**, the error output includes the exact `go get` commands needed to update the vendor. Follow this procedure for each failing repo, working **in dependency order** per the vendoring table above (e.g., fix `lustre_fs_operator` before `nnf_sos`, fix `nnf_sos` before `nnf_dm` or `nnf_integration_test`). + +For each failing repo: + +1. **Read the error output.** It contains the `go get` command(s) with the specific module paths and branch targets. +2. **Clone the repo** into a temporary directory using `nnf_gh_repo` to resolve the GitHub `owner/repo` path. +3. **Create a branch** named `update-vendor`. +4. **Run the `go get` commands** from the error output, then run `go mod tidy` and `go mod vendor`. +5. **Commit** all changes with a signed-off commit (`-s`) and message `"Update vendor dependencies"`. +6. **Push** the branch and **create a PR** titled `"Update vendor dependencies"` with body `"Pre-release vendoring update."`. +7. **Assign reviewers** from `$RELEASE_REVIEWERS`. +8. **Wait for CI** and reviewer approval, then **merge** the PR. +9. **Verify the fix against origin:** Re-clone (repeat Step 0) so you're working from what's actually on GitHub, then re-run the `nnf_cmd master ` check starting from the repo you just fixed and continuing through all remaining downstream repos in order. The fixed repo must now pass. If a downstream repo fails, repeat this procedure (steps 1–9) for it. +10. Return to `$RELEASE_WORKDIR/nnf-deploy/tools/release-all`. + +> **Dependency order matters:** An upstream repo's vendoring PR must be merged to master before any downstream repo that vendors it can be fixed. For example, if both `nnf_sos` and `nnf_dm` fail, merge the `nnf_sos` fix first — `nnf_dm` vendors `nnf_sos` and needs the updated master. +> +> **`nnf_deploy` submodule pointers:** Any submodule repo that has had commits merged to master since the last submodule pointer update — whether from development work or vendoring fixes — will cause `nnf_deploy`'s Step 2 check to fail with "Submodules are not up to date." The script catches this automatically. Fix it the same way: clone `nnf_deploy`, create a branch, run `git submodule update --remote` for the affected submodules, commit, PR, merge. Then re-clone and verify. + +After fixing and verifying the last failing repo, **re-run Step 2 on all repos** as a final safety net to confirm everything passes end to end. + +### Phase 2: Branch Creation + +#### Step 3 — Create trial release branches *(sequential, per repo)* + +Process all repos **except `nnf_doc`**, which is deferred (see below): + +```sh +for repo in $(./release-all.sh -L | grep -v nnf_doc); do + nnf_cmd release "$repo" +done +``` + +Review output for merge conflicts before continuing. + +> **Note:** `nnf_mfu`, `nnf_ec`, and `nnf_storedversions_maint` are standalone repos that often have no new commits between releases. If any reports "No new changes to release", this is normal — **skip that repo in all subsequent phases** (Steps 4a–4d and the `nnf_doc` sequence). Do not attempt to push, PR, merge, or tag a repo that had no changes. + +#### `nnf_doc` deferral + +`nnf_doc` must be deferred until after `nnf_deploy` is tagged and its GitHub Release is published. The `nnf_doc` script updates `mkdocs.yml` with the latest `nnf-deploy` release version by querying GitHub Releases (not just tags). Follow this sequence: + +1. Complete Steps 3–4d for all repos through `nnf_deploy` +2. **Wait ~60 seconds** for `nnf_deploy`'s "Handle Release Tag" GitHub Actions workflow to publish the GitHub Release +3. Verify: `gh release view $NNF_RELEASE -R NearNodeFlash/nnf-deploy 2>&1 | cat` should succeed. If it fails with "release not found", wait another 30 seconds and retry. The workflow typically completes within 2 minutes; if it hasn't after 5 minutes, check the Actions tab on the `nnf-deploy` repo for errors. +4. Then run `nnf_doc` through Steps 3–4d: + +```sh +nnf_cmd release nnf_doc +nnf_cmd release-push nnf_doc +nnf_create_pr nnf_doc +nnf_add_reviewers nnf_doc +nnf_cmd merge-pr nnf_doc +nnf_cmd tag-release nnf_doc +``` + +> **Important:** `nnf_doc`'s repo is `NearNodeFlash/NearNodeFlash.github.io` and uses the `main` branch (not `master`). + +### Phase 3: Release Generation + +*Complete steps 4a–4d for each repo sequentially before moving on to the next. Present the results of all four sub-steps to the user and wait for approval once per repo (after step 4d), rather than after each individual sub-step.* + +> **Why sequential order matters:** Phase 2 creates release branches (not tags). Phase 3 then processes each repo through push → PR → merge → tag **in dependency order**. By the time a downstream repo (e.g., `nnf_dm`) reaches `create-pr`, any upstream repo it depends on (e.g., `nnf_sos`) is already tagged. The script's `release` phase handles submodule pointers to the correct release branches automatically. + +#### Step 4a — Push release branch + +*Only if no merge conflicts.* + +```sh +nnf_cmd release-push +``` + +#### Step 4a-alt — Merge conflict resolution + +*Only if merge conflicts exist.* **Stop and present the conflict details to the user.** Merge conflicts during a release are rare and require human judgment. Do not attempt to resolve them automatically. Wait for the user to provide guidance before proceeding. + +#### Step 4b — Create PR + +```sh +nnf_create_pr +``` + +The `create-pr` output includes the PR URL (e.g., `https://github.com///pull/123`). The `nnf_create_pr` function extracts the PR number and sets `$PR_NUMBER` automatically. + +> **Note:** The script does **not** assign PR reviewers automatically. After each `create-pr`, assign reviewers: + +```sh +nnf_add_reviewers +``` + +#### Step 4c — Merge PR + +> Do NOT manually merge — let the tool do it. + +```sh +nnf_cmd merge-pr +``` + +> **Note:** Before running `merge-pr`, ensure CI status checks on the PR have passed. If required checks are still running, `merge-pr` may fail. +> +> **Note:** `merge-pr` may produce no visible output even on success. Verify the merge completed: +> +> ```sh +> gh api repos///pulls/ 2>&1 | grep -o '"merged":[^,]*' +> ``` +> +> Expected output: `"merged":true` + +#### Step 4d — Tag the release + +*Creates the annotated tag required by CI/CD.* + +```sh +nnf_cmd tag-release +``` + +> **Note:** `tag-release` output will include "Bypassed rule violations for refs/tags/...". This is expected. The repos have rulesets ("Auto-imported tag create protections" and "Auto-imported tag delete protections") that block creation, update, and deletion of `v*` tags by default — protecting release tags from unauthorized changes. Your account bypasses these rules because it has an admin or maintain role, which is in the ruleset's bypass list. The tags are created correctly. + +### Phase 4: Finalize + +#### Step 5 — Finalize release notes + +*Run only after ALL repos, including `nnf_doc`, are released. Run from `tools/release-all/`.* + +```sh +./final-release-notes.sh -r $NNF_RELEASE 2>&1 | cat # preview +./final-release-notes.sh -r $NNF_RELEASE -C 2>&1 | cat # commit +``` + +### Phase 5: Verification + +#### Step 6 — Compare release manifests + +*Run from `tools/release-all/`.* + +```sh +./compare-releases.sh -i $PREVIOUS_RELEASE $NNF_RELEASE 2>&1 | cat +``` + +The `-i` flag displays image version changes inline. Use `-d` to display the full diff. The diff file is always saved to `workingspace/manifest--to-.diff`. + +#### Step 7 — Verify GitHub releases + +Check each repo for correct tag, release notes, and artifacts (`manifests.tar` + `manifests-kind.tar` on `nnf-deploy`). + +--- + +### Relevant files + +- `tools/release-all/release-all.sh` — Main orchestration script, all phases +- `tools/release-all/final-release-notes.sh` — Release notes finalization +- `tools/release-all/compare-releases.sh` — Manifest diff tool +- `tools/release-all/README.md` — Minimal, needs expansion +- `.github/workflows/handle_release_tag.yaml` — CI/CD that creates GitHub releases on tag push +- `config/repositories.yaml` — Submodule version tracking updated by release-push phase +- External docs source: NearNodeFlash/NearNodeFlash.github.io repo + +### Decisions + +- Release execution follows the strict 10-repo dependency order — no parallelization +- `nnf-mfu` and `nnf-ec` are released standalone (positions 4 & 5) despite not being submodules +- `nnf_doc` released last, version should match nnf-deploy +- Default `-B patch` unless breaking changes present + +### Verification + +1. `./release-all.sh -L` confirms 10 repos in dependency order +2. Zero errors from each `master` phase before proceeding +3. GitHub release page shows correct annotated tag after each `tag-release` +4. `compare-releases.sh` diff validates expected changes +5. `final-release-notes.sh` output includes all submodule notes + CRD API info +6. nnf-deploy GitHub release has manifests.tar and manifests-kind.tar + +### Further Considerations + +1. **Release type** — Is this patch, minor, or major? Determines `-B` flag. Recommend: default `patch` unless breaking changes. +2. **Doc improvements timing** — Corrections applied first (branch `improve-release-docs` in NearNodeFlash.github.io), to be used as a guide during the release. Enhancements can follow after the release. +3. **README expansion** — Add prerequisites and quick-start to `tools/release-all/README.md` inline; keep detailed guide external. diff --git a/tools/release-all/compare-releases.sh b/tools/release-all/compare-releases.sh index 47c797b4..6989b69b 100755 --- a/tools/release-all/compare-releases.sh +++ b/tools/release-all/compare-releases.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright 2024 Hewlett Packard Enterprise Development LP +# Copyright 2024-2026 Hewlett Packard Enterprise Development LP # Other additional copyright holders may be indicated within. # # The entirety of this work is licensed under the Apache License, @@ -39,22 +39,32 @@ msg(){ NNF_URL="git@github.com:NearNodeFlash/nnf-deploy.git" WORKSPACE="workingspace" +DISPLAY_DIFF=false +IMAGES_ONLY=false usage() { echo "Usage: $PROG [-h]" - echo " $PROG [-w workspace_dir] [-u nnf_deploy_url] " + echo " $PROG [-d] [-i] [-w workspace_dir] [-u nnf_deploy_url] " echo echo " VERSION1 Starting version for comparison. This may be" echo " a tag such as 'v0.1.7'." echo " VERSION2 Ending version for comparison. This may be" echo " a tag such as 'v0.1.8'." + echo " -d Display the full diff output." + echo " -i Display only image version changes." echo " -w workspace_dir Name for working directory. Default: '$WORKSPACE'" echo " -u nnf_deploy_url URL to the nnf-deploy repo. Default: '$NNF_URL'" echo } -while getopts "hw:u:" opt; do +while getopts "dhiw:u:" opt; do case $opt in + d) + DISPLAY_DIFF=true + ;; + i) + IMAGES_ONLY=true + ;; u) NNF_URL="$OPTARG" ;; @@ -106,7 +116,7 @@ prep_for_manifest() { rm -rf "$ver" fi - if ! gh release view "$ver" > /dev/null 2>&1; then + if ! gh release view -R "$NNF_URL" "$ver" > /dev/null 2>&1; then do_fail "Release $ver does not exist" fi } @@ -116,10 +126,14 @@ fetch_and_unpack_manifest() { local vtar="$2" gh release download -R "$NNF_URL" -O "$vtar" -p manifests.tar "$ver" || do_fail "Unable to find manifests.tar for $ver" + [[ -s "$vtar" ]] || do_fail "Downloaded tarball $vtar is empty" mkdir "$ver" || do_fail "Cannot mkdir $ver" cd "$ver" tar xfo "../$vtar" || do_fail "Unable to extract tar $vtar" + local count + count=$(find . -type f | wc -l) + [[ $count -gt 0 ]] || do_fail "No files were extracted from $vtar" cd .. } @@ -135,6 +149,32 @@ manifest "$ver1" "$v1tar" manifest "$ver2" "$v2tar" diff_file="manifest-$ver1-to-$ver2.diff" -diff -uNr "$ver1" "$ver2" > "$diff_file" || echo -msg "Manifest diffs for $ver1 to $ver2 are in $WORKSPACE/$diff_file" +rc=0 +diff -uNr "$ver1" "$ver2" > "$diff_file" || rc=$? +if [[ $rc -eq 0 ]]; then + msg "No differences found between $ver1 and $ver2" + rm -f "$diff_file" +elif [[ $rc -eq 1 ]]; then + lines=$(wc -l < "$diff_file" | tr -d ' ') + changed_files="" + changed_files=$(diff -rq "$ver1" "$ver2") || true + changed=$(echo "$changed_files" | wc -l | tr -d ' ') + msg "Manifest diffs for $ver1 to $ver2 are in $WORKSPACE/$diff_file ($lines lines, $changed files changed)" + msg "Changed files:" + echo "$changed_files" + + if [[ $IMAGES_ONLY == true ]]; then + msg "" + msg "Image version changes:" + grep -E '^[+-].*image:.*:' "$diff_file" | grep -v '^[+-][+-][+-]' || echo " (no image changes found)" + fi + + if [[ $DISPLAY_DIFF == true ]]; then + msg "" + msg "Full diff:" + cat "$diff_file" + fi +else + do_fail "diff encountered an error (rc=$rc) comparing $ver1 and $ver2" +fi diff --git a/tools/release-all/final-release-notes.sh b/tools/release-all/final-release-notes.sh index 5e5c4d9a..a61a8741 100755 --- a/tools/release-all/final-release-notes.sh +++ b/tools/release-all/final-release-notes.sh @@ -23,6 +23,7 @@ PROG=$(basename "$0") NNF_DOC_URL='git@github.com:NearNodeFlash/NearNodeFlash.github.io.git' NNF_DEPLOY_URL='git@github.com:NearNodeFlash/nnf-deploy.git' +NNF_STOREDVERSIONS_MAINT_URL='git@github.com:NearNodeFlash/nnf-storedversions-maint.git' BOLD=$(tput bold) NORMAL=$(tput sgr0) @@ -382,6 +383,11 @@ dep_url="$NNF_DEPLOY_URL" dep_name=$(get_repo_dir_name "$dep_url") check_release_vX "$dep_repo_short_name" "$dep_name" "$dep_url" "releases/v0" +svm_repo_short_name="nnf_storedversions_maint" +svm_url="$NNF_STOREDVERSIONS_MAINT_URL" +svm_name=$(get_repo_dir_name "$svm_url") +check_release_vX "$svm_repo_short_name" "$svm_name" "$svm_url" "releases/v0" + doc_repo_short_name="nnf_doc" doc_url="$NNF_DOC_URL" doc_name=$(get_repo_dir_name "$doc_url") diff --git a/tools/release-all/release-all.sh b/tools/release-all/release-all.sh index 4a0ef8de..59e928e2 100755 --- a/tools/release-all/release-all.sh +++ b/tools/release-all/release-all.sh @@ -27,22 +27,24 @@ repomap_keys[1]=lustre_csi_driver repomap_keys[2]=lustre_fs_operator repomap_keys[3]=nnf_mfu repomap_keys[4]=nnf_ec -repomap_keys[5]=nnf_sos -repomap_keys[6]=nnf_dm -repomap_keys[7]=nnf_integration_test -repomap_keys[8]=nnf_deploy -repomap_keys[9]=nnf_doc +repomap_keys[5]=nnf_storedversions_maint +repomap_keys[6]=nnf_sos +repomap_keys[7]=nnf_dm +repomap_keys[8]=nnf_integration_test +repomap_keys[9]=nnf_deploy +repomap_keys[10]=nnf_doc declare "repomap_${repomap_keys[0]}"='git@github.com:DataWorkflowServices/dws.git' declare "repomap_${repomap_keys[1]}"='git@github.com:HewlettPackard/lustre-csi-driver.git' declare "repomap_${repomap_keys[2]}"='git@github.com:NearNodeFlash/lustre-fs-operator.git' declare "repomap_${repomap_keys[3]}"='git@github.com:NearNodeFlash/nnf-mfu.git' declare "repomap_${repomap_keys[4]}"='git@github.com:NearNodeFlash/nnf-ec.git' -declare "repomap_${repomap_keys[5]}"='git@github.com:NearNodeFlash/nnf-sos.git' -declare "repomap_${repomap_keys[6]}"='git@github.com:NearNodeFlash/nnf-dm.git' -declare "repomap_${repomap_keys[7]}"='git@github.com:NearNodeFlash/nnf-integration-test.git' -declare "repomap_${repomap_keys[8]}"='git@github.com:NearNodeFlash/nnf-deploy.git' -declare "repomap_${repomap_keys[9]}"='git@github.com:NearNodeFlash/NearNodeFlash.github.io.git' +declare "repomap_${repomap_keys[5]}"='git@github.com:NearNodeFlash/nnf-storedversions-maint.git' +declare "repomap_${repomap_keys[6]}"='git@github.com:NearNodeFlash/nnf-sos.git' +declare "repomap_${repomap_keys[7]}"='git@github.com:NearNodeFlash/nnf-dm.git' +declare "repomap_${repomap_keys[8]}"='git@github.com:NearNodeFlash/nnf-integration-test.git' +declare "repomap_${repomap_keys[9]}"='git@github.com:NearNodeFlash/nnf-deploy.git' +declare "repomap_${repomap_keys[10]}"='git@github.com:NearNodeFlash/NearNodeFlash.github.io.git' getter() { # Getter for the associative-array-ish thingy that works in bash v3 for Mac. @@ -235,6 +237,7 @@ get_default_branch() { local default_branch=master [[ $repo_short_name = nnf_doc ]] && default_branch=main + [[ $repo_short_name = nnf_storedversions_maint ]] && default_branch=main echo "$default_branch" } @@ -512,6 +515,11 @@ update_own_release_references() { update_kustomization_file "$k_yaml" controller "$release_ver" "$indent" git add "$k_yaml" ;; + nnf_storedversions_maint) + k_yaml="config/manager" + update_kustomization_file "$k_yaml" controller "$release_ver" "$indent" + git add "$k_yaml" + ;; esac if [[ $(git status -s | wc -l) -gt 0 ]]; then diff --git a/tools/release-all/setup-release-env.sh b/tools/release-all/setup-release-env.sh new file mode 100755 index 00000000..e1ff34f9 --- /dev/null +++ b/tools/release-all/setup-release-env.sh @@ -0,0 +1,141 @@ +#!/usr/bin/env bash + +# Copyright 2026 Hewlett Packard Enterprise Development LP +# Other additional copyright holders may be indicated within. +# +# The entirety of this work is licensed under the Apache License, +# Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. +# +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# setup-release-env.sh — Source this to set up the release environment. +# Usage: source ./setup-release-env.sh [-B patch|minor|major] +# +# Exports: RELEASE_TYPE, PREVIOUS_RELEASE, NNF_RELEASE, RELEASE_USER, +# RELEASE_REVIEWERS +# Defines: nnf_cmd, nnf_create_pr, nnf_add_reviewers, nnf_gh_repo + +# NOTE: This file is sourced, not executed. Do not use 'set -euo pipefail' +# here — those options would persist in the caller's shell and cause any +# non-zero exit (even transient) to kill the terminal session. + +# --- Parse -B flag ----------------------------------------------------------- +OPTIND=1 +RELEASE_TYPE="patch" +while getopts "B:" opt; do + case $opt in + B) RELEASE_TYPE="$OPTARG" ;; + *) echo "Usage: source ./setup-release-env.sh [-B patch|minor|major]" >&2; return 1 ;; + esac +done +export RELEASE_TYPE + +# --- Compute versions -------------------------------------------------------- +export PREVIOUS_RELEASE +PREVIOUS_RELEASE=$(gh release view --json tagName --jq '.tagName' \ + -R NearNodeFlash/nnf-deploy 2>&1) +if [[ -z "$PREVIOUS_RELEASE" || "$PREVIOUS_RELEASE" == *"release not found"* ]]; then + echo "ERROR: Could not determine previous release from NearNodeFlash/nnf-deploy" >&2 + return 1 +fi + +case "$RELEASE_TYPE" in + major) PERL_CMD='($x=$F[0])=~s/^v//; printf("v%s", join(".", $x+1, 0, 0))' ;; + minor) PERL_CMD='print join(".", $F[0], $F[1]+1, 0)' ;; + patch) PERL_CMD='print join(".", $F[0], $F[1], $F[2]+1)' ;; + *) echo "ERROR: Invalid release type '$RELEASE_TYPE'" >&2; return 1 ;; +esac +export NNF_RELEASE +NNF_RELEASE=$(echo "$PREVIOUS_RELEASE" | perl -F'/\./' -ane "$PERL_CMD") + +# --- Detect operator & reviewers -------------------------------------------- +RELEASE_USER=$(gh api user --jq '.login' 2>&1) +CODEOWNERS_FILE="../../.github/CODEOWNERS" +if [[ ! -f "$CODEOWNERS_FILE" ]]; then + echo "ERROR: $CODEOWNERS_FILE not found. Run from tools/release-all/ inside an nnf-deploy clone." >&2 + return 1 +fi +REVIEWER_POOL=$(grep '^\*' "$CODEOWNERS_FILE" | tr -s ' ' | cut -d' ' -f2- | tr ' ' '\n' | sed 's/@//' | paste -sd, -) +export RELEASE_REVIEWERS +RELEASE_REVIEWERS=$(echo "$REVIEWER_POOL" | tr ',' '\n' \ + | grep -v "$RELEASE_USER" | paste -sd, -) + +if [[ -z "$RELEASE_REVIEWERS" ]]; then + echo "WARNING: RELEASE_REVIEWERS is empty — you are the only code owner in CODEOWNERS." >&2 + echo " You will need to assign reviewers manually for each PR." >&2 +fi + +# --- Helper: run release-all.sh with auto -M and -B ------------------------- +# Usage: nnf_cmd +nnf_cmd() { + local phase="$1" repo="$2" + local args=() _rc + [[ "$repo" == "nnf_sos" ]] && args+=("-M") + ./release-all.sh -B "$RELEASE_TYPE" -P "$phase" -R "$repo" "${args[@]}" 2>&1 | cat + # bash uses PIPESTATUS[0]; zsh uses pipestatus[1] (lowercase, 1-indexed) + _rc=${PIPESTATUS[0]:-${pipestatus[1]}} + return "${_rc:-0}" +} + +# --- Helper: create-pr + capture PR number ----------------------------------- +# Usage: nnf_create_pr +# Sets PR_NUMBER as a side effect. +nnf_create_pr() { + local repo="$1" + local args=() + [[ "$repo" == "nnf_sos" ]] && args+=("-M") + local output + output=$(./release-all.sh -B "$RELEASE_TYPE" -P create-pr -R "$repo" "${args[@]}" 2>&1 | tee /dev/tty) + PR_NUMBER=$(echo "$output" | grep -o 'https://github.com/[^/ ]*/[^/ ]*/pull/[0-9]*' | awk -F'/' '{print $NF}' | head -n1) + export PR_NUMBER + echo "PR number: $PR_NUMBER" +} + +# --- Helper: look up the GitHub / for a script name ------------- +# Usage: nnf_gh_repo +nnf_gh_repo() { + local repo="$1" + case "$repo" in + dws) echo "DataWorkflowServices/dws" ;; + lustre_csi_driver) echo "HewlettPackard/lustre-csi-driver" ;; + lustre_fs_operator) echo "NearNodeFlash/lustre-fs-operator" ;; + nnf_mfu) echo "NearNodeFlash/nnf-mfu" ;; + nnf_ec) echo "NearNodeFlash/nnf-ec" ;; + nnf_storedversions_maint) echo "NearNodeFlash/nnf-storedversions-maint" ;; + nnf_sos) echo "NearNodeFlash/nnf-sos" ;; + nnf_dm) echo "NearNodeFlash/nnf-dm" ;; + nnf_integration_test) echo "NearNodeFlash/nnf-integration-test" ;; + nnf_deploy) echo "NearNodeFlash/nnf-deploy" ;; + nnf_doc) echo "NearNodeFlash/NearNodeFlash.github.io" ;; + *) echo "UNKNOWN/$repo" ;; + esac +} + +# --- Helper: assign reviewers to the current PR ------------------------------ +# Usage: nnf_add_reviewers +# Requires PR_NUMBER to be set (via nnf_create_pr). +nnf_add_reviewers() { + local repo="$1" + local gh_repo + gh_repo=$(nnf_gh_repo "$repo") + gh pr edit "$PR_NUMBER" --add-reviewer "$RELEASE_REVIEWERS" \ + -R "$gh_repo" 2>&1 | cat +} + +# --- Summary ----------------------------------------------------------------- +echo "──────────────────────────────────────" +echo " Previous release : $PREVIOUS_RELEASE" +echo " New release : $NNF_RELEASE" +echo " Release type : $RELEASE_TYPE" +echo " Operator : $RELEASE_USER" +echo " Reviewers : $RELEASE_REVIEWERS" +echo "──────────────────────────────────────"