Merge pull request GoogleCloudPlatform#4786 from Neelabh94/improvement/slurm_gpu_daily_tests

feat: Add default partition GPU check to A* SLURM daily tests

Author: Neelabh94

tools/cloud-build/daily-tests/ansible_playbooks/test-validation/test-default-partition.yml

@@ -0,0 +1,36 @@
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- name: Assert variables are defined
+  ansible.builtin.assert:
+    that:
+    - custom_vars.gpu_partition is defined
+
+- name: Get default partition configuration
+  ansible.builtin.shell: "scontrol show config | grep '^DefaultPartition'"
+  register: default_partition_config
+  retries: 10
+  delay: 12
+  until: default_partition_config.stdout | length > 0
+  changed_when: false
+
+- name: Extract default partition name
+  set_fact:
+    default_partition: "{{ (default_partition_config.stdout.split('='))[1] | trim }}"
+
+- name: Assert default partition is the GPU partition
+  ansible.builtin.assert:
+    that:
+    - default_partition == custom_vars.gpu_partition
+    msg: "The default partition is '{{ default_partition }}', but expected '{{ custom_vars.gpu_partition }}'."

tools/cloud-build/daily-tests/tests/ml-a3-highgpu-slurm-cluster.yml

@@ -27,6 +27,7 @@ network: default
 post_deploy_tests:
 - test-validation/test-mounts.yml
 - test-validation/test-partitions.yml
+- test-validation/test-default-partition.yml
 - test-validation/test-enroot.yml
 - test-validation/test-gpus-slurm.yml
 custom_vars:

tools/cloud-build/daily-tests/tests/ml-a3-megagpu-slurm-ubuntu.yml

@@ -30,6 +30,7 @@ sub_network: "{{ deployment_name }}-sub-net-0"
 post_deploy_tests:
 - test-validation/test-mounts.yml
 - test-validation/test-partitions.yml
+- test-validation/test-default-partition.yml
 - test-validation/test-enroot.yml
 - test-validation/test-gpus-slurm.yml
 - test-validation/test-nccl.yml

tools/cloud-build/daily-tests/tests/ml-a3-ultragpu-slurm.yml

@@ -28,6 +28,7 @@ network: "{{ test_name }}-net-0"
 post_deploy_tests:
 - test-validation/test-mounts.yml
 - test-validation/test-partitions.yml
+- test-validation/test-default-partition.yml
 - test-validation/test-enroot.yml
 - test-validation/test-gpus-slurm.yml
 post_destroy_tasks:

tools/cloud-build/daily-tests/tests/ml-a4-highgpu-slurm-flex.yml

@@ -28,6 +28,7 @@ network: "{{ test_name }}-net-0"
 post_deploy_tests:
 - test-validation/test-mounts.yml
 - test-validation/test-partitions.yml
+- test-validation/test-default-partition.yml
 - test-validation/test-enroot.yml
 - test-validation/test-gpus-slurm.yml
 post_destroy_tasks:

tools/cloud-build/daily-tests/tests/ml-a4-highgpu-slurm.yml

@@ -28,6 +28,7 @@ network: "{{ test_name }}-net-0"
 post_deploy_tests:
 - test-validation/test-mounts.yml
 - test-validation/test-partitions.yml
+- test-validation/test-default-partition.yml
 - test-validation/test-enroot.yml
 - test-validation/test-gpus-slurm.yml
 post_destroy_tasks: