Add NCCL test validation to G4 Integration tests (GoogleCloudPlatform#4933)

Author: kadupoornima

examples/gke-g4/nccl-test.yaml

@@ -19,7 +19,8 @@ metadata:
   labels:
     app: nccl-test
 spec:
-  ttlSecondsAfterFinished: 600
+  ttlSecondsAfterFinished: 1200
+  suspend: False
   template:
     metadata:
       labels:

tools/cloud-build/daily-tests/ansible_playbooks/test-validation/test-gke-g4.yml

@@ -0,0 +1,59 @@
+# Copyright 2025 "Google LLC"
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+- name: Assert variables are defined
+  ansible.builtin.assert:
+    that:
+    - region is defined
+    - custom_vars.project is defined
+- name: Get cluster credentials for kubectl
+  delegate_to: localhost
+  ansible.builtin.shell: |
+    gcloud container clusters get-credentials {{ deployment_name }} --region {{ region }} --project {{ custom_vars.project }} --verbosity=debug
+- name: Run the NCCL test
+  delegate_to: localhost
+  ansible.builtin.shell: |
+    kubectl create -f {{ workspace }}/examples/gke-g4/nccl-test.yaml -v=9
+  args:
+    executable: /bin/bash
+
+- name: Wait for Job to hit completion
+  delegate_to: localhost
+  ansible.builtin.shell: |
+    kubectl get job --field-selector status.successful=1 -v=9
+  register: job_completion
+  until: job_completion.stdout_lines | length > 1
+  retries: 10
+  delay: 60
+
+- name: Fetch logs from the pod and save to pod_logs.txt
+  delegate_to: localhost
+  ansible.builtin.shell: |
+    pod_name="$(kubectl get pods --no-headers -o custom-columns="Name:.metadata.name" | awk '{if ($1 ~ "nccl-test") print $0}')"
+    kubectl logs ${pod_name} > pod_logs.txt
+    cat pod_logs.txt
+  register: nccl_test_logs
+
+- name: Print the NCCL test logs from the pod
+  debug:
+    msg: "{{nccl_test_logs.stdout}}"
+- name: Ensure average bus bandwidth is >= 30 GB/s
+  delegate_to: localhost
+  ansible.builtin.shell: |
+    grep -o '# Avg bus bandwidth    : [0-9\.]*' pod_logs.txt | cut -d ':' -f 2 | awk '{$1=$1;print}'
+  register: avg_bus_bandwidth
+  failed_when: avg_bus_bandwidth.stdout | float < 30
+- name: Clean up
+  delegate_to: localhost
+  ansible.builtin.shell: |
+    kubectl delete jobs --all -v=9

tools/cloud-build/daily-tests/tests/gke-g4.yml

@@ -24,8 +24,8 @@ network: "{{ deployment_name }}-net-0"
 region: us-central1
 zone: us-central1-b
 remote_node: "{{ deployment_name }}-remote-node-0"
-machine_type: g4-standard-48
-num_gpus: 1
+machine_type: g4-standard-96
+num_gpus: 2
 static_node_count: 2
 cli_deployment_vars:
   region: "{{ region }}"
@@ -38,3 +38,4 @@ custom_vars:
   project: "{{ project }}"
 post_deploy_tests:
 - test-validation/test-gke-job.yml
+- test-validation/test-gke-g4.yml