Merge branch 'develop' of github.com:shubpal07/cluster-toolkit into shubham/kueue-on-helm

Author: shubpal07

community/examples/gke-tpu-v6/gke-tpu-v6.yaml

@@ -56,9 +56,10 @@ vars:
 deployment_groups:
 - group: primary
   modules:
-  - id: gke-tpu-v6-net
+  - id: gke-tpu-v6-net-0
     source: modules/network/vpc
     settings:
+      network_name: $(vars.deployment_name)-net-0
       subnetworks:
       - subnet_name: $(vars.deployment_name)-sub-0
         subnet_region: $(vars.region)
@@ -72,7 +73,25 @@ deployment_groups:
           ip_cidr_range: 10.0.32.0/20
       firewall_rules:
       - name: $(vars.deployment_name)-internal-0
-        ranges: [192.168.0.0/18]
+        ranges: [192.168.0.0/16]
+        allow:
+        - protocol: tcp
+          ports: ["0-65535"]
+        - protocol: udp
+          ports: ["0-65535"]
+        - protocol: icmp
+
+  - id: gke-tpu-v6-net-1
+    source: modules/network/vpc
+    settings:
+      network_name: $(vars.deployment_name)-net-1
+      subnetworks:
+      - subnet_name: $(vars.deployment_name)-sub-1
+        subnet_region: $(vars.region)
+        subnet_ip: 192.168.64.0/18
+      firewall_rules:
+      - name: $(vars.deployment_name)-internal-1
+        ranges: [192.168.0.0/16]
         allow:
         - protocol: tcp
           ports: ["0-65535"]
@@ -107,7 +126,7 @@ deployment_groups:
 
   - id: gke-tpu-v6-cluster
     source: modules/scheduler/gke-cluster
-    use: [gke-tpu-v6-net, workload_service_account]
+    use: [gke-tpu-v6-net-0, workload_service_account]
     settings:
       system_node_pool_machine_type: "n2-standard-8"
       system_node_pool_disk_size_gb: $(vars.system_node_pool_disk_size_gb)
@@ -117,6 +136,21 @@ deployment_groups:
       master_authorized_networks:
       - cidr_block: $(vars.authorized_cidr) # Allows your machine to run the kubectl command. Required for multi network setup.
         display_name: "kubectl-access-network"
+      additional_networks:
+        $(concat(
+          [{
+            network=gke-tpu-v6-net-1.network_name,
+            subnetwork=gke-tpu-v6-net-1.subnetwork_name,
+            subnetwork_project=vars.project_id,
+            nic_type="GVNIC",
+            queue_count=null,
+            network_ip=null,
+            stack_type=null,
+            access_config=[{nat_ip=null, public_ptr_domain_name=null, network_tier=null}],
+            ipv6_access_config=[],
+            alias_ip_range=[]
+          }]
+        ))
       # Cluster versions cannot be updated through the toolkit after creation
       # Please manage cluster version from the Google Cloud Console directly
       version_prefix: "1.32."
@@ -140,6 +174,21 @@ deployment_groups:
       zones: [$(vars.zone)]
       disk_size_gb: $(vars.v6e_node_pool_disk_size_gb)
       static_node_count: $(vars.static_node_count)
+      additional_networks:
+        $(concat(
+          [{
+            network=gke-tpu-v6-net-1.network_name,
+            subnetwork=gke-tpu-v6-net-1.subnetwork_name,
+            subnetwork_project=vars.project_id,
+            nic_type="GVNIC",
+            queue_count=null,
+            network_ip=null,
+            stack_type=null,
+            access_config=[{nat_ip=null, public_ptr_domain_name=null, network_tier=null}],
+            ipv6_access_config=[],
+            alias_ip_range=[]
+          }]
+        ))
       reservation_affinity:
         consume_reservation_type: SPECIFIC_RESERVATION
         specific_reservations:

community/front-end/ofe/requirements.txt

@@ -1,17 +1,17 @@
 altgraph==0.17.4
 archspec==0.2.5
 argcomplete==3.6.2
-asgiref==3.9.1
+asgiref==3.10.0
 astroid==3.3.11
 attrs==25.3.0
 # This should be supported by zoneinfo in Python 3.9+
 backports.zoneinfo==0.2.1;python_version<"3.9"
 cachetools>=2.0,<6.0
-certifi==2025.8.3
+certifi==2025.10.5
 cffi>=2.0.0
 cfgv==3.4.0
 charset-normalizer==3.4.3
-click==8.2.1
+click==8.3.0
 crispy-bootstrap5==2025.6
 cryptography==46.0.2
 decorator==5.2.1
@@ -21,42 +21,42 @@ distlib==0.4.0
 # django-revproxy==0.11.0 released but not yet in pypi
 git+https://github.com/jazzband/django-revproxy.git@d2234005135dc0771b7c4e0bb0465664ccfa5787
 Django==5.2.7
-django-allauth==65.11.1
+django-allauth==65.12.0
 django-crispy-forms==2.4
 django-extensions==3.2.3
 djangorestframework==3.16.1
 filelock==3.19.1
-google-api-core==2.25.1
-google-api-python-client==2.179.0
-google-auth==2.40.3
+google-api-core==2.25.2
+google-api-python-client==2.184.0
+google-auth==2.41.1
 google-auth-httplib2==0.2.0
 google-cloud-artifact-registry==1.16.1
 google-cloud-secret-manager==2.24.0
-google-cloud-build==3.31.3
+google-cloud-build==3.32.0
 google-cloud-billing==1.16.3
 google-cloud-core==2.4.3
 google-cloud-pubsub==2.31.1
-google-cloud-compute==1.35.0
-google-cloud-storage==3.3.1
+google-cloud-compute==1.38.0
+google-cloud-storage==3.4.0
 google-cloud-secret-manager==2.24.0
 google-crc32c==1.7.1
 google-resumable-media==2.7.2
 googleapis-common-protos==1.70.0
 grafana-api==1.0.3
 grpc-google-iam-v1==0.14.2
-grpcio==1.74.0
-grpcio-status==1.74.0
+grpcio==1.75.1
+grpcio-status==1.75.1
 h11==0.16.0
-httplib2==0.30.0
-identify==2.6.13
+httplib2==0.31.0
+identify==2.6.15
 idna==3.10
 importlib-resources==6.5.2
-isort==6.0.1
+isort==6.1.0
 Jinja2==3.1.6
 jsonschema==4.25.1
 jsonschema-specifications==2025.9.1
 lazy-object-proxy==1.12.0
-libcst==1.8.2
+libcst==1.8.5
 macholib==1.16.3
 MarkupSafe==2.1.3
 mccabe==0.7.0
@@ -71,23 +71,23 @@ proto-plus==1.26.1
 protobuf>=6.31.1,<7.0.0
 pyasn1==0.6.1
 pyasn1-modules==0.4.2
-pycparser==2.22
+pycparser==2.23
 PyJWT==2.10.1
-pylint==3.3.8
+pylint==3.3.9
 pylint-django==2.6.1
 pylint-plugin-utils==0.9.0
-pyparsing==3.2.3
+pyparsing==3.2.5
 python3-openid==3.2.0
 pytz==2023.3
-PyYAML==6.0.2
+PyYAML==6.0.3
 referencing==0.36.2
 requests==2.32.5
 requests-oauthlib==2.0.0
 retry==0.9.2
 rpds-py==0.27.1
 rsa==4.9.1
 ruamel.yaml==0.18.15
-ruamel.yaml.clib==0.2.12
+ruamel.yaml.clib==0.2.14
 semantic-version==2.10.0
 setuptools-rust==1.12.0
 six==1.17.0
@@ -99,7 +99,7 @@ typing-inspect==0.9.0
 typing_extensions==4.15.0
 uritemplate==4.2.0
 urllib3==2.5.0
-uvicorn==0.35.0
+uvicorn==0.37.0
 virtualenv==20.34.0
 wrapt==1.17.3
 xmltodict==1.0.2

community/modules/scripts/wait-for-startup/scripts/wait-for-startup-status.sh

@@ -60,6 +60,10 @@ FINISH_LINE="startup-script exit status"
 # Match string for failures on the new guest agent
 FINISH_LINE_ERR="Script.*failed with error:"
 
+# NEW: Accept also these finish lines as success.
+STARTUP_SCRIPT_SUCCEEDED_LINE="google-startup-scripts.service: Succeeded."
+STARTUP_SCRIPT_FINISHED_LINE="Finished Google Compute Engine Startup Scripts."
+
 NON_FATAL_ERRORS=(
 	"Internal error"
 )
@@ -68,7 +72,7 @@ until [[ now -gt deadline ]]; do
 	ser_log=$(
 		set -o pipefail
 		${fetch_cmd} 2>"${error_file}" |
-			c1grep "${FINISH_LINE}\|${FINISH_LINE_ERR}"
+			c1grep "${FINISH_LINE}\|${FINISH_LINE_ERR}\|${STARTUP_SCRIPT_SUCCEEDED_LINE}\|${STARTUP_SCRIPT_FINISHED_LINE}"
 	) || {
 		err=$(cat "${error_file}")
 		echo "$err"
@@ -100,6 +104,12 @@ if [[ "${STATUS}" == 0 ]]; then
 elif [[ "${STATUS}" == 1 ]]; then
 	echo "startup-script finished with errors, ${INSPECT_OUTPUT_TEXT}"
 	echo "${fetch_cmd}"
+elif echo "${ser_log}" | grep -qE "${STARTUP_SCRIPT_SUCCEEDED_LINE}"; then
+	echo "startup-script finished successfully (startup script succeeded line detected)"
+	exit 0
+elif echo "${ser_log}" | grep -qE "${STARTUP_SCRIPT_FINISHED_LINE}"; then
+	echo "startup-script finished successfully (startup script finished line detected)"
+	exit 0
 elif [[ now -ge deadline ]]; then
 	echo "startup-script timed out after ${TIMEOUT} seconds"
 	echo "${INSPECT_OUTPUT_TEXT}"

examples/hpc-slurm-h4d.yaml

@@ -26,6 +26,7 @@ vars:
   #Provisioning model, select one, lack of selection will rely on on-demand capacity
   h4d_dws_flex_enabled: false
   h4d_reservation_name: ""
+  h4d_enable_spot_vm: false
 
 # Documentation for each of the modules used below can be found at
 # https://github.com/GoogleCloudPlatform/hpc-toolkit/blob/main/modules/README.md
@@ -85,6 +86,7 @@ deployment_groups:
       node_count_static: 2
       node_count_dynamic_max: 0
       enable_placement: false
+      enable_spot_vm: $(vars.h4d_enable_spot_vm)
 
       #Provisioning models
       reservation_name: $(vars.h4d_reservation_name)

examples/machine-learning/a3-ultragpu-8g/nemo-framework/Dockerfile

@@ -1,10 +1,10 @@
-# Copyright 2024 "Google LLC"
+# Copyright 2025 "Google LLC"
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
-#     http://www.apache.org/licenses/LICENSE-2.0
+# http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
@@ -13,25 +13,38 @@
 # limitations under the License.
 
 ARG NEMOFW_VERSION=24.12
+ARG NCCL_GIB_VERSION=v1.1.0
+
+# Stage 1: Use the nccl-plugin-gib container as an installer
+FROM us-docker.pkg.dev/gce-ai-infra/gpudirect-gib/nccl-plugin-gib:${NCCL_GIB_VERSION} AS nccl-gib
+RUN /scripts/container_entry.sh install --install-nccl
+
+# Stage 2: Build the final NeMo container
 FROM nvcr.io/nvidia/nemo:$NEMOFW_VERSION
 
-ENV NCCL_DEBUG=INFO,WARN
-ENV NCCL_DEBUG_SUBSYS=NET,COLL,GPU,INIT,ENV
-ENV NCCL_SOCKET_IFNAME=enp0s19,enp192s20
-ENV GLOO_SOCKET_IFNAME=enp0s19,enp192s20
+# Copy the NCCL gIB plugin and other binaries from the installer stage.
+# This replaces the need to manually install and mount /var/lib/gib from the host.
+COPY --from=nccl-gib /var/lib/gib /usr/local/gib
+
+# Set the following to match what is in /usr/local/gib/scripts/set_nccl_env.sh
+ENV NCCL_NET=gIB
 ENV NCCL_CROSS_NIC=0
 ENV NCCL_NET_GDR_LEVEL=PIX
-#ENV PMIX_MCA_gds=^ds12
 ENV NCCL_P2P_NET_CHUNKSIZE=131072
 ENV NCCL_NVLS_CHUNKSIZE=524288
-ENV NCCL_IB_GID_INDEX=3
 ENV NCCL_IB_ADAPTIVE_ROUTING=1
 ENV NCCL_IB_QPS_PER_CONNECTION=4
 ENV NCCL_IB_TC=52
 ENV NCCL_IB_FIFO_TC=84
-ENV NCCL_SHIMNET_GUEST_CONFIG_CHECKER_CONFIG_FILE=/usr/local/gib/configs/guest_config.txtpb
-ENV NCCL_TUNER_CONFIG_PATH=/usr/local/gib/configs/tuner_config.txtpb
-ENV NCCL_NET=gIB
+# Use A3 Ultra tuner config
+ENV NCCL_TUNER_CONFIG_PATH=/usr/local/gib/configs/tuner_config_a3u.txtpb
+
+
+# Add NCCL DEBUG at WARN level (useful for debugging)
+ENV NCCL_DEBUG=WARN
+ENV NCCL_DEBUG_SUBSYS=NET,COLL,GPU,INIT,ENV
+ENV GLOO_SOCKET_IFNAME=enp0s19,enp192s20
 
+# Set up ldconfig to load nccl-gib
 RUN echo "/usr/local/gib/lib64" >> /etc/ld.so.conf.d/gib.conf && ldconfig
 ENV LD_LIBRARY_PATH=/usr/local/gib/lib64:$LD_LIBRARY_PATH

examples/machine-learning/a3-ultragpu-8g/nemo-framework/README.md

@@ -43,7 +43,7 @@ README
 
    ```shell
    cd launcher_scripts
-   mkdir data
+   mkdir -p data
 
    MAX_STEPS=10
    NUM_NODES=8
@@ -53,8 +53,8 @@ README
        stages=[training] \
        training=gpt3/5b \
        env_vars.TRANSFORMERS_OFFLINE=0 \
-       container=../nemo-24.12.sqsh \
-       container_mounts=[${HOME}/.cache,/usr/local/gib] \
+       container=../nemo-24.12-v1.1.0.sqsh \
+       container_mounts=[${HOME}/.cache] \
        cluster.srun_args=["--container-writable"] \
        training.model.data.data_impl=mock \
        training.model.data.data_prefix=[] \

examples/machine-learning/a3-ultragpu-8g/nemo-framework/setup_nemo.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-# Copyright 2024 "Google LLC"
+# Copyright 2025 "Google LLC"
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -19,12 +19,19 @@
 #SBATCH --exclusive
 
 : "${NEMOFW_VERSION:=24.12}"
+: "${NCCL_GIB_VERSION:=v1.1.0}"
 
-srun docker build --build-arg="NEMOFW_VERSION=${NEMOFW_VERSION}" -t nemo-"${NEMOFW_VERSION}" .
-srun rm -f nemo-"${NEMOFW_VERSION}".sqsh
-srun enroot import dockerd://nemo-"${NEMOFW_VERSION}"
+# This ensures that the docker process on the compute node can access us-docker.pkg.dev
+srun gcloud auth configure-docker us-docker.pkg.dev --quiet
+
+srun docker build \
+	--build-arg="NEMOFW_VERSION=${NEMOFW_VERSION}" \
+	--build-arg="NCCL_GIB_VERSION=${NCCL_GIB_VERSION}" \
+	-t nemo-"${NEMOFW_VERSION}"-"${NCCL_GIB_VERSION}" .
+srun rm -f nemo-"${NEMOFW_VERSION}"-"${NCCL_GIB_VERSION}".sqsh
+srun enroot import dockerd://nemo-"${NEMOFW_VERSION}"-"${NCCL_GIB_VERSION}"
 
 srun \
 	--container-mounts="${PWD}":/workspace/mount_dir,/var/tmp:/var/tmp \
-	--container-image=./nemo-"${NEMOFW_VERSION}".sqsh \
+	--container-image=./nemo-"${NEMOFW_VERSION}"-"${NCCL_GIB_VERSION}".sqsh \
 	bash -c "cp -r /opt/NeMo-Framework-Launcher/requirements.txt /opt/NeMo-Framework-Launcher/launcher_scripts /opt/NeMo-Framework-Launcher/auto_configurator /workspace/mount_dir/"

modules/network/vpc/README.md

@@ -173,9 +173,9 @@ limitations under the License.
 
 | Name | Source | Version |
 |------|--------|---------|
-| <a name="module_cloud_router"></a> [cloud\_router](#module\_cloud\_router) | terraform-google-modules/cloud-router/google | ~> 6.0 |
+| <a name="module_cloud_router"></a> [cloud\_router](#module\_cloud\_router) | terraform-google-modules/cloud-router/google | ~> 7.3 |
 | <a name="module_nat_ip_addresses"></a> [nat\_ip\_addresses](#module\_nat\_ip\_addresses) | terraform-google-modules/address/google | ~> 4.1 |
-| <a name="module_vpc"></a> [vpc](#module\_vpc) | terraform-google-modules/network/google | ~> 10.0 |
+| <a name="module_vpc"></a> [vpc](#module\_vpc) | terraform-google-modules/network/google | ~> 12.0 |
 
 ## Resources
 

modules/network/vpc/main.tf

@@ -180,7 +180,7 @@ resource "terraform_data" "network_profile_firewall_validation" {
 
 module "vpc" {
   source  = "terraform-google-modules/network/google"
-  version = "~> 10.0"
+  version = "~> 12.0"
 
   depends_on = [terraform_data.network_profile_firewall_validation]
 
@@ -235,7 +235,7 @@ module "nat_ip_addresses" {
 
 module "cloud_router" {
   source  = "terraform-google-modules/cloud-router/google"
-  version = "~> 6.0"
+  version = "~> 7.3"
 
   depends_on = [terraform_data.cloud_nat_validation]