Release candidate: v1.76.0 (#5013)

Author: aslam-quad

.github/workflows/pr-precommit.yml

@@ -26,12 +26,21 @@ on:
     - main
     - develop
     - release-candidate
+    - hotfix-*
 
 jobs:
   pre-commit:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v4
+    - name: Free Disk Space (Ubuntu)
+      uses: jlumbroso/free-disk-space@main
+      with:
+        tool-cache: false
+        android: true
+        dotnet: true
+        haskell: true
+        large-packages: true
     - uses: actions/setup-python@v5
       with:
         # internal Python tests require Python 3.12
@@ -61,6 +70,14 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v4
+    - name: Free Disk Space (Ubuntu)
+      uses: jlumbroso/free-disk-space@main
+      with:
+        tool-cache: false
+        android: true
+        dotnet: true
+        haskell: true
+        large-packages: true
     - uses: actions/setup-python@v5
       with:
         # the slurm-files Python requirements.txt requires updating

Makefile

@@ -8,7 +8,7 @@ MIN_GOLANG_VERSION=1.24 # for building gcluster
         warn-go-version warn-terraform-version warn-packer-version \
         test-engine validate_configs validate_golden_copy packer-check \
         terraform-format packer-format mypy \
-        check-tflint check-pre-commit
+        check-tflint check-pre-commit ci-tests
 
 SHELL=/bin/bash -o pipefail
 ENG = ./cmd/... ./pkg/...
@@ -71,6 +71,7 @@ install-dev-deps: warn-terraform-version warn-packer-version check-pre-commit ch
 	go install mvdan.cc/sh/v3/cmd/shfmt@latest
 	go install golang.org/x/tools/cmd/goimports@latest
 	go install honnef.co/go/tools/cmd/staticcheck@latest
+	go install github.com/jstemmer/go-junit-report/v2@latest
 	pip install -r community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/requirements.txt
 	pip install -r community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/requirements-dev.txt
 	pip install mypy==1.18.2
@@ -93,6 +94,9 @@ clean:
 	done
 
 # RULES SUPPORTING THE ABOVE
+ci-tests: warn-go-missing
+	$(info **************** running gcluster unit tests for CI **************)
+	go test -v $(ENG) | go-junit-report > test-results.xml
 
 test-engine: warn-go-missing
 	$(info **************** vetting go code **********************)

README.md

@@ -119,7 +119,7 @@ The Cluster Toolkit officially supports the following VM images:
 
 * HPC Rocky Linux 8
 * Debian 11
-* Ubuntu 20.04 LTS
+* Ubuntu 22.04 LTS
 
 For more information on these and other images, see
 [docs/vm-images.md](docs/vm-images.md).

cmd/root.go

@@ -53,7 +53,7 @@ HPC deployments on the Google Cloud Platform.`,
 				logging.Fatal("cmd.Help function failed: %s", err)
 			}
 		},
-		Version:     "v1.75.1",
+		Version:     "v1.76.0",
 		Annotations: annotation,
 	}
 )

community/examples/hpc-slinky/hpc-slinky.yaml

@@ -49,7 +49,7 @@ deployment_groups:
           ip_cidr_range: 10.0.32.0/20
 
   - id: node_pool_service_account
-    source: community/modules/project/service-account
+    source: modules/project/service-account
     settings:
       name: gke-np-sa
       project_roles:
@@ -61,7 +61,7 @@ deployment_groups:
       - artifactregistry.reader
 
   - id: workload_service_account
-    source: community/modules/project/service-account
+    source: modules/project/service-account
     settings:
       name: gke-wl-sa
       project_roles:

…unity/examples/hpc-slurm-ubuntu2004.yaml …unity/examples/hpc-slurm-ubuntu2204.yamlcommunity/examples/hpc-slurm-ubuntu2004.yaml renamed to community/examples/hpc-slurm-ubuntu2204.yaml community/examples/hpc-slurm-ubuntu2004.yaml renamed to community/examples/hpc-slurm-ubuntu2204.yaml

@@ -14,7 +14,7 @@
 
 ---
 
-blueprint_name: hpc-slurm-ubuntu2004-v6
+blueprint_name: hpc-slurm-ubuntu2204-v6
 
 vars:
   project_id:  ## Set GCP Project ID Here ##

community/examples/slurm-gke/nccl-tests/README.md

@@ -0,0 +1,97 @@
+The examples in this directory are used to show how enroot + pyxis can be used
+to launch containerized workloads via Slurm running on GKE.
+
+Contents:
+
+* `build-nccl-tests.sh`: A Slurm batch script for building the nccl-tests.
+* `run-nccl-tests-rdma.sh`: A Slurm batch script for running the nccl-tests for A3 Ultra, A4, A4X with RDMA.
+  `all_gather_perf` benchmark.
+* `run-nccl-tests-tcpxo.sh`: A Slurm batch script for running the nccl-tests for A3 Mega with TCPXO.
+  `all_gather_perf` benchmark.
+
+# Running NCCL-Tests via Enroot/Pyxis
+
+In general the workflow to deploy GPUDirect-RDMA-enabled workloads via enroot-pyxis is
+the following:
+
+1. Convert your container into a squashfs based container image
+2. Set required environment variables
+3. Run your application workload
+
+## TLDR
+
+For an end-to-end example, copy the `build-nccl-tests.sh` and
+`run-nccl-tests-rdma.sh` or `run-nccl-tests-tcpxo.sh` to your login node.
+
+And run the following:
+
+```text
+BUILD_JOB=$(sbatch --parsable build-nccl-tests.sh) # takes ~4 minutes
+sbatch -d afterok:${BUILD_JOB} run-nccl-tests-rdma.sh # takes ~3 minutes
+```
+
+The latter should result in a slurm-XX.out file that contains the result of the nccl
+`all_gather_perf` benchmark:
+
+```text
+#
+#                                                              out-of-place                       in-place
+#       size         count      type   redop    root     time   algbw   busbw #wrong     time   algbw   busbw #wrong
+#        (B)    (elements)                               (us)  (GB/s)  (GB/s)            (us)  (GB/s)  (GB/s)
+   268435456       4194304     float    none      -1    XXXXX  XXX.XX  XXX.XX    N/A   XXXXXX  XXX.XX  XXX.XX      0
+   536870912       8388608     float    none      -1    XXXXX  XXX.XX  XXX.XX    N/A   XXXXXX  XXX.XX  XXX.XX      0
+  1073741824      16777216     float    none      -1    XXXXX  XXX.XX  XXX.XX    N/A   XXXXXX  XXX.XX  XXX.XX      0
+  2147483648      33554432     float    none      -1    XXXXX  XXX.XX  XXX.XX    N/A   XXXXXX  XXX.XX  XXX.XX      0
+  4294967296      67108864     float    none      -1    XXXXX  XXX.XX  XXX.XX    N/A   XXXXXX  XXX.XX  XXX.XX      0
+  8589934592     134217728     float    none      -1    XXXXX  XXX.XX  XXX.XX    N/A   XXXXXX  XXX.XX  XXX.XX      0
+# Out of bounds values : 0 OK
+# Avg bus bandwidth    : XXX.XX
+#
+```
+
+For more details, follow the remainder of this README.
+
+## Detailed Instructions
+
+All of the following should be done on the login node of your slurm cluster,
+and while somewhere on the shared Filestore filesystem (typically the user's
+home directory).
+
+### Building NCCL-tests
+
+See build-nccl-tests.sh for an example. Within it, you will see that first we'll
+create a squashfs version of the container using we want to launch using `enroot
+import`. We do this because otherwise we'd be pulling the (typically more than
+10GB) image multiple times from the source on each node, converting to sqsh each
+time, etc, which would make the job launch longer.
+
+For building the nccl-tests binaries, we use `pyxis` to run the enroot container
+and build the nccl-tests within that container to ensure the resulting binaries
+are compatible with the container environment.
+
+Both of the above (importing and building) are accomplished by running:
+
+```text
+sbatch build-nccl-tests.sh
+```
+
+### Running your application on a3-mega instances
+
+For a complete example, run:
+
+```text
+sbatch run-nccl-tests-tcpxo.sh
+```
+
+### Running your application on a3-ultra instances
+
+For a complete example, run:
+
+```text
+sbatch run-nccl-tests-rdma.sh
+```
+
+The output will appear in in a `slurm-<job#>.log` file. If the name of your a3-ultragpu
+partition is different than "gke", you will need to modify the `build-nccl-tests.sh`
+and `run-nccl-tests.sh` scripts's  `#SBATCH --partition` setting. Alternatively, you
+can run `sbatch -p <your partition> <script>`.

community/examples/slurm-gke/nccl-tests/build-nccl-tests.sh

@@ -0,0 +1,44 @@
+#!/bin/bash
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#SBATCH --exclusive
+#SBATCH --ntasks=1
+#SBATCH --partition=gke
+#SBATCH --ntasks-per-node=1
+#SBATCH --gpus-per-node=8
+
+# Usage: sbatch build-nccl-tests.sh
+
+set -x
+
+CONTAINER_IMAGE=./nvidia+pytorch+24.09-py3.sqsh
+
+# Import the pytorch container to enroot if not already present.
+if [ ! -f ${CONTAINER_IMAGE} ]; then
+	# This creates a file named "nvidia+pytorch+24.09-py3.sqsh", which
+	# uses ~18 GB of disk space. This should be run on a filesystem that
+	# can be seen by all worker nodes
+	enroot import docker://nvcr.io#nvidia/pytorch:24.09-py3
+fi
+
+# Install nccl-tests using openmpi from within pytorch container
+srun --container-mounts="$PWD:/nccl" \
+	--container-image=${CONTAINER_IMAGE} \
+	bash -c "
+       cd /nccl &&
+       git clone https://github.com/NVIDIA/nccl-tests.git &&
+       cd /nccl/nccl-tests/ &&
+       MPI=1 CC=mpicc CXX=mpicxx make -j
+    "

community/examples/slurm-gke/nccl-tests/run-nccl-tests-rdma.sh

@@ -0,0 +1,57 @@
+#!/bin/bash
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#SBATCH --partition=gke
+#SBATCH --mem=0
+#SBATCH -N 2
+#SBATCH --gpus-per-node=8
+#SBATCH --ntasks-per-node=8
+#SBATCH --exclusive
+
+# Usage: sbatch run-nccl-tests.sh
+
+set -x
+# This should be set to the squashfs file that you created for your application
+CONTAINER_IMAGE=./nvidia+pytorch+24.09-py3.sqsh
+
+# Set up NCCL Environment variables
+# The following two can be useful for debugging
+# export NCCL_DEBUG=INFO
+# export NCCL_DEBUG_SUBSYS=INIT,NET
+
+# These parameters should not be modified
+source /usr/local/gib/scripts/set_nccl_env.sh
+export NCCL_NET=gIB
+
+# Mount /var/tmp to allow the rest of the enroot container to be read-only, and
+# mount current $PWD to /nccl to for accessing nccl-tests binary
+CONTAINER_MOUNTS="/var/tmp:/var/tmp"
+
+# Mount PWD to /nccl in the enroot container
+CONTAINER_MOUNTS=${CONTAINER_MOUNTS},"$PWD:/nccl"
+
+# Mount required directories for gIB libnccl-net
+CONTAINER_MOUNTS=${CONTAINER_MOUNTS},"/usr/local/gib"
+
+# Run the workload
+srun -l \
+	-N "${SLURM_NNODES}" \
+	--ntasks-per-node=8 \
+	--mpi=pmi2 \
+	--container-image="${CONTAINER_IMAGE}" \
+	--container-mounts="${CONTAINER_MOUNTS}" \
+	sh -c "export LD_LIBRARY_PATH=/usr/local/gib/lib64:/usr/lib/x86_64-linux-gnu:\$LD_LIBRARY_PATH;
+  /nccl/nccl-tests/build/all_gather_perf -b 256M -e 8G -f 2 -g 1 -w 5 --iters 200;
+  "

community/examples/slurm-gke/nccl-tests/run-nccl-tests-tcpxo.sh

@@ -0,0 +1,65 @@
+#!/bin/bash
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#SBATCH --partition=gke
+#SBATCH --mem=0
+#SBATCH -N 2
+#SBATCH --gpus-per-node=8
+#SBATCH --ntasks-per-node=8
+#SBATCH --exclusive
+
+# Usage: sbatch run-nccl-tests.sh
+
+set -x
+# This should be set to the squashfs file that you created for your application
+CONTAINER_IMAGE=./nvidia+pytorch+24.09-py3.sqsh
+
+# Set up NCCL Environment variables
+# The following two can be useful for debugging
+# export NCCL_DEBUG=INFO
+# export NCCL_DEBUG_SUBSYS=INIT,NET
+
+# These parameters should not be modified
+NCCL_LIB_DIR="/usr/local/nvidia/lib64" source /usr/local/nvidia/lib64/nccl-env-profile.sh
+export NCCL_FASTRAK_LLCM_DEVICE_DIRECTORY=/dev/aperture_devices
+
+# Here we grab all the environment variables that need to be
+# passed down into the container. Slurm would otherwise only pass these env vars
+# to the job environment on the host.
+# shellcheck disable=SC2001
+HOST_VARS=$(sed 's/ \{1,\}/,/g' <<<"${!NCCL*}")
+
+# Mount /var/tmp to allow the rest of the enroot container to be read-only, and
+# mount current $PWD to /nccl to for accessing nccl-tests binary
+CONTAINER_MOUNTS="/var/tmp:/var/tmp"
+
+# Mount PWD to /nccl in the enroot container
+CONTAINER_MOUNTS=${CONTAINER_MOUNTS},"$PWD:/nccl"
+
+# Mount required directories for GPUDirect-TCPXO functionality
+CONTAINER_MOUNTS=${CONTAINER_MOUNTS},"/usr/local/nvidia/lib64/"
+
+# Run the workload
+srun -l \
+	-N "${SLURM_NNODES}" \
+	--mpi=pmi2 \
+	--ntasks-per-node=8 \
+	--container-image="${CONTAINER_IMAGE}" \
+	--container-env="${HOST_VARS}" \
+	--container-mounts="${CONTAINER_MOUNTS}" \
+	sh -c "
+  export LD_LIBRARY_PATH=/usr/local/nvidia/lib64:/usr/lib/x86_64-linux-gnu:\$LD_LIBRARY_PATH;
+  /nccl/nccl-tests/build/all_gather_perf -b 8M -e 8G -f 2 -g 1 -w 5 --iters 200;
+  "