Merge pull request GoogleCloudPlatform#4490 from bytetwin/nccl_gib_upgrade

bytetwin · web-flow · commit 5947f835d218 · 2025-08-13T10:22:58.000+05:30
Update nccl gib plugin version
diff --git a/examples/gke-a3-ultragpu/gke-a3-ultragpu.yaml b/examples/gke-a3-ultragpu/gke-a3-ultragpu.yaml
@@ -261,7 +261,7 @@ deployment_groups:
         install: true  # NCCL gIB plugin via DaemonSet initContainer
         path: $(vars.gib_installer_path)
         template_vars:
-          version: v1.0.6
+          version: v1.1.0
           accelerator_count: 8
 
   - id: job-template
diff --git a/examples/gke-a3-ultragpu/nccl-jobset-example.yaml b/examples/gke-a3-ultragpu/nccl-jobset-example.yaml
@@ -125,7 +125,7 @@ spec:
             - name: nccl
               stdin: true
               tty: true
-              image: us-docker.pkg.dev/gce-ai-infra/gpudirect-gib/nccl-plugin-gib-diagnostic:v1.0.6
+              image: us-docker.pkg.dev/gce-ai-infra/gpudirect-gib/nccl-plugin-gib-diagnostic:v1.1.0
               securityContext:
                 privileged: true
               env:
diff --git a/examples/gke-a3-ultragpu/system_benchmarks/ramble-nccl.yaml b/examples/gke-a3-ultragpu/system_benchmarks/ramble-nccl.yaml
@@ -126,7 +126,7 @@ data:
           | sed 's/ / -x /g')
 
         container_name: nccl-tests
-        container_uri: us-docker.pkg.dev/gce-ai-infra/gpudirect-gib/nccl-plugin-gib-diagnostic:v1.0.6
+        container_uri: us-docker.pkg.dev/gce-ai-infra/gpudirect-gib/nccl-plugin-gib-diagnostic:v1.1.0
         gke_container_name: nccl
         gke_namespace: ramble
         gpus_per_node: 8
diff --git a/examples/gke-a4/gke-a4.yaml b/examples/gke-a4/gke-a4.yaml
@@ -277,7 +277,7 @@ deployment_groups:
         install: true  # NCCL gIB plugin via DaemonSet initContainer
         path: $(vars.gib_installer_path)
         template_vars:
-          version: v1.0.6
+          version: v1.1.0
           accelerator_count: 8
 
   - id: job-template
diff --git a/examples/gke-a4/nccl-jobset-example.yaml b/examples/gke-a4/nccl-jobset-example.yaml
@@ -125,7 +125,7 @@ spec:
             - name: nccl
               stdin: true
               tty: true
-              image: us-docker.pkg.dev/gce-ai-infra/gpudirect-gib/nccl-plugin-gib-diagnostic:v1.0.6
+              image: us-docker.pkg.dev/gce-ai-infra/gpudirect-gib/nccl-plugin-gib-diagnostic:v1.1.0
               securityContext:
                 privileged: true
               env:
diff --git a/examples/gke-a4x/nccl-jobset-example.yaml b/examples/gke-a4x/nccl-jobset-example.yaml
@@ -87,7 +87,7 @@ spec:
             - name: nccl-test
               stdin: true
               tty: true
-              image: us-docker.pkg.dev/gce-ai-infra/gpudirect-gib/nccl-plugin-gib-diagnostic-arm64:v1.0.4
+              image: us-docker.pkg.dev/gce-ai-infra/gpudirect-gib/nccl-plugin-gib-diagnostic-arm64:v1.0.6
               env:
               - name: MY_NODE_NAME
                 valueFrom:
diff --git a/examples/gke-consumption-options/dws-flex-start-queued-provisioning/nccl-installer.yaml b/examples/gke-consumption-options/dws-flex-start-queued-provisioning/nccl-installer.yaml
@@ -70,7 +70,7 @@ spec:
           sysctl -w net.ipv4.conf.gpu5rdma0.log_martians=0
           sysctl -w net.ipv4.conf.gpu6rdma0.log_martians=0
           sysctl -w net.ipv4.conf.gpu7rdma0.log_martians=0
-      - image: us-docker.pkg.dev/gce-ai-infra/gpudirect-gib/nccl-plugin-gib:v1.0.6
+      - image: us-docker.pkg.dev/gce-ai-infra/gpudirect-gib/nccl-plugin-gib:v1.1.0
         name: nccl-rdma-installer
         resources:
           requests:
diff --git a/examples/gke-consumption-options/dws-flex-start-queued-provisioning/nccl-jobset-example.yaml b/examples/gke-consumption-options/dws-flex-start-queued-provisioning/nccl-jobset-example.yaml
@@ -128,7 +128,7 @@ spec:
             - name: nccl
               stdin: true
               tty: true
-              image: us-docker.pkg.dev/gce-ai-infra/gpudirect-gib/nccl-plugin-gib-diagnostic:v1.0.6
+              image: us-docker.pkg.dev/gce-ai-infra/gpudirect-gib/nccl-plugin-gib-diagnostic:v1.1.0
               securityContext:
                 privileged: true
               env:
diff --git a/examples/gke-consumption-options/dws-flex-start/nccl-installer.yaml b/examples/gke-consumption-options/dws-flex-start/nccl-installer.yaml
@@ -70,7 +70,7 @@ spec:
           sysctl -w net.ipv4.conf.gpu5rdma0.log_martians=0
           sysctl -w net.ipv4.conf.gpu6rdma0.log_martians=0
           sysctl -w net.ipv4.conf.gpu7rdma0.log_martians=0
-      - image: us-docker.pkg.dev/gce-ai-infra/gpudirect-gib/nccl-plugin-gib:v1.0.6
+      - image: us-docker.pkg.dev/gce-ai-infra/gpudirect-gib/nccl-plugin-gib:v1.1.0
         name: nccl-rdma-installer
         resources:
           requests:
diff --git a/examples/gke-consumption-options/dws-flex-start/nccl-jobset-example.yaml b/examples/gke-consumption-options/dws-flex-start/nccl-jobset-example.yaml
@@ -125,7 +125,7 @@ spec:
             - name: nccl
               stdin: true
               tty: true
-              image: us-docker.pkg.dev/gce-ai-infra/gpudirect-gib/nccl-plugin-gib-diagnostic:v1.0.6
+              image: us-docker.pkg.dev/gce-ai-infra/gpudirect-gib/nccl-plugin-gib-diagnostic:v1.1.0
               securityContext:
                 privileged: true
               env:
diff --git a/examples/hypercompute_clusters/a3u-slurm-ubuntu-gcs/a3u-slurm-ubuntu-gcs.yaml b/examples/hypercompute_clusters/a3u-slurm-ubuntu-gcs/a3u-slurm-ubuntu-gcs.yaml
@@ -48,7 +48,7 @@ vars:
     project: $(vars.project_id)
     family: $(vars.deployment_name)-u22
   disk_size_gb: 200
-  nccl_plugin_version: v1.0.6
+  nccl_plugin_version: v1.1.0
   benchmark_dir: $(ghpc_stage("system_benchmarks"))
 
   # Here we define a set of startup script runners that are used to configure
diff --git a/examples/hypercompute_clusters/a3u-slurm-ubuntu-gcs/system_benchmarks/run-nccl-tests-via-ramble.sh b/examples/hypercompute_clusters/a3u-slurm-ubuntu-gcs/system_benchmarks/run-nccl-tests-via-ramble.sh
@@ -96,7 +96,7 @@ ramble:
 
     container_dir: "/opt/apps/ramble/sqsh"
     container_name: nccl-plugin-gib-diagnostic
-    container_uri: docker://us-docker.pkg.dev#gce-ai-infra/gpudirect-gib/nccl-plugin-gib-diagnostic:latest
+    container_uri: docker://us-docker.pkg.dev#gce-ai-infra/gpudirect-gib/nccl-plugin-gib-diagnostic:v1.1.0
     processes_per_node: 8
     processes_per_node: '{gpus_per_node}'
     gpus_per_node: '8'
diff --git a/examples/machine-learning/a3-ultragpu-8g/a3ultra-slurm-blueprint.yaml b/examples/machine-learning/a3-ultragpu-8g/a3ultra-slurm-blueprint.yaml
@@ -39,7 +39,7 @@ vars:
     project: $(vars.project_id)
     family: $(vars.deployment_name)-u22
   disk_size_gb: 100
-  nccl_plugin_version: v1.0.6
+  nccl_plugin_version: v1.1.0
   base_network_name: $(vars.deployment_name)
 
   #Provisioning models (set to true or fill in reservation name, pick only one)
diff --git a/examples/machine-learning/a3-ultragpu-8g/nemo-framework/Dockerfile b/examples/machine-learning/a3-ultragpu-8g/nemo-framework/Dockerfile
@@ -23,8 +23,6 @@ ENV NCCL_CROSS_NIC=0
 ENV NCCL_NET_GDR_LEVEL=PIX
 #ENV PMIX_MCA_gds=^ds12
 ENV NCCL_P2P_NET_CHUNKSIZE=131072
-ENV NCCL_P2P_PCI_CHUNKSIZE=131072
-ENV NCCL_P2P_NVL_CHUNKSIZE=524288
 ENV NCCL_NVLS_CHUNKSIZE=524288
 ENV NCCL_IB_GID_INDEX=3
 ENV NCCL_IB_ADAPTIVE_ROUTING=1
diff --git a/examples/machine-learning/a4-highgpu-8g/a4high-slurm-blueprint.yaml b/examples/machine-learning/a4-highgpu-8g/a4high-slurm-blueprint.yaml
@@ -39,7 +39,7 @@ vars:
     project: $(vars.project_id)
     family: $(vars.deployment_name)-u22
   disk_size_gb: 100
-  nccl_plugin_version: v1.0.6
+  nccl_plugin_version: v1.1.0
   benchmark_dir: $(ghpc_stage("system_benchmarks"))
   base_network_name: $(vars.deployment_name)
 
diff --git a/examples/machine-learning/a4-highgpu-8g/a4high-vm.yaml b/examples/machine-learning/a4-highgpu-8g/a4high-vm.yaml
@@ -290,8 +290,6 @@ deployment_groups:
           NCCL_CROSS_NIC=0
           NCCL_NET_GDR_LEVEL=PIX
           NCCL_P2P_NET_CHUNKSIZE=131072
-          NCCL_P2P_PCI_CHUNKSIZE=131072
-          NCCL_P2P_NVL_CHUNKSIZE=524288
           NCCL_NVLS_CHUNKSIZE=524288
           NCCL_IB_GID_INDEX=3
           NCCL_IB_ADAPTIVE_ROUTING=1
diff --git a/examples/machine-learning/a4-highgpu-8g/system_benchmarks/run-nccl-tests-via-ramble.sh b/examples/machine-learning/a4-highgpu-8g/system_benchmarks/run-nccl-tests-via-ramble.sh
@@ -97,8 +97,8 @@ ramble:
     hostlist: \${SLURM_JOB_NODELIST}
 
     container_dir: "/opt/apps/ramble/sqsh"
-    container_name: nccl-plugin-gib-diagnostic-v1.0.6
-    container_uri: docker://us-docker.pkg.dev#gce-ai-infra/gpudirect-gib/nccl-plugin-gib-diagnostic:v1.0.6
+    container_name: nccl-plugin-gib-diagnostic-v1.1.0
+    container_uri: docker://us-docker.pkg.dev#gce-ai-infra/gpudirect-gib/nccl-plugin-gib-diagnostic:v1.1.0
     processes_per_node: 8
     processes_per_node: '{gpus_per_node}'
     gpus_per_node: '8'
diff --git a/modules/management/kubectl-apply/variables.tf b/modules/management/kubectl-apply/variables.tf
@@ -17,7 +17,7 @@
 locals {
   kueue_supported_versions  = ["v0.12.2", "v0.11.4", "v0.10.1", "v0.10.0"]
   jobset_supported_versions = ["v0.8.1", "v0.7.2", "v0.5.2"]
-  gib_supported_versions    = ["v1.0.2", "v1.0.3", "v1.0.5", "v1.0.6"]
+  gib_supported_versions    = ["v1.0.2", "v1.0.3", "v1.0.5", "v1.0.6", "v1.1.0"]
 }
 
 resource "terraform_data" "kueue_validations" {
