Integrating G4 onspot (GoogleCloudPlatform#5102)

Author: kvenkatachala333

examples/ml-slurm-g4.yaml

@@ -40,6 +40,9 @@ vars:
 
   # Cluster Settings
   disk_size_gb: 100
+  #Provisioning models (set to true or fill in reservation name, pick only one)
+  g4_reservation_name: "" # To use a reservation, provide its name here.
+  g4_enable_spot_vm: false # To use Spot VMs, set this to true.
 
 deployment_groups:
 - group: image-env
@@ -108,8 +111,6 @@ deployment_groups:
                 name: "{{ item }}"
                 selection: hold
               loop: "{{ nvidia_packages }}"
-
-
       - type: data
         destination: /var/tmp/slurm_vars.json
         content: |
@@ -287,6 +288,9 @@ deployment_groups:
         family: $(vars.built_image_family)
       instance_image_custom: true
       enable_placement: false
+      #Provisioning models
+      reservation_name: $(vars.g4_reservation_name)
+      enable_spot_vm: $(vars.g4_enable_spot_vm)
 
       # Instructions for Dual NIC on g4-standard-384:
       # 4. Uncomment the 'network_interfaces' block below to define the two NICs.

tools/cloud-build/daily-tests/builds/ml-g4-onspot-slurm.yaml

@@ -0,0 +1,66 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+tags:
+- m.custom-image
+- m.filestore
+- m.schedmd-slurm-gcp-v6-controller
+- m.schedmd-slurm-gcp-v6-login
+- m.schedmd-slurm-gcp-v6-nodeset
+- m.schedmd-slurm-gcp-v6-partition
+- m.startup-script
+- m.vpc
+- slurm6
+
+timeout: 14400s  # 4hr
+steps:
+# While using static network names we are gaurding against more than 1 instance running at a time (for multi-group tests)
+- id: check_for_running_build
+  name: gcr.io/cloud-builders/gcloud
+  script: "tools/cloud-build/check_running_build.sh tools/cloud-build/daily-tests/builds/ml-g4-onspot-slurm.yaml"
+
+- id: ml-g4-onspot-slurm
+  name: us-central1-docker.pkg.dev/$PROJECT_ID/hpc-toolkit-repo/test-runner
+  entrypoint: /bin/bash
+  env:
+  - "ANSIBLE_HOST_KEY_CHECKING=false"
+  - "ANSIBLE_CONFIG=/workspace/tools/cloud-build/ansible.cfg"
+  - "PROJECT_ID=$PROJECT_ID"
+  - "NUM_NODES=2"
+  - "MACHINE_TYPE=g4-standard-48"
+  - "INSTANCE_PREFIX=g4osp"
+  - "BUILD_ID=$BUILD_ID"
+  - "OPTIONS_GCS_PATH=gs://hpc-ctk1357/g4options.txt"
+  args:
+  - -c
+  - |
+    set -e -u -o pipefail
+    echo "Sourcing find_available_zone.sh to determine zone."
+    source /workspace/tools/cloud-build/find_available_zone.sh
+    if [ -z "$${ZONE:-}" ]; then
+      echo "ERROR: ZONE not found" >&2
+      exit 1
+    fi
+    set -x
+    cd /workspace && make
+    REGION="$${ZONE%-*}"
+    BUILD_ID_SHORT="$${BUILD_ID:0:6}"
+    BLUEPRINT="/workspace/examples/ml-slurm-g4.yaml"
+
+    ansible-playbook tools/cloud-build/daily-tests/ansible_playbooks/slurm-integration-test.yml \
+      --user=sa_106486320838376751393 \
+      --extra-vars="project=$${PROJECT_ID} build=$${BUILD_ID_SHORT}" \
+      --extra-vars="region=$${REGION} zone=$${ZONE}" \
+      --extra-vars="@tools/cloud-build/daily-tests/tests/ml-g4-onspot-slurm.yml"

tools/cloud-build/daily-tests/tests/ml-g4-onspot-slurm.yml

@@ -0,0 +1,49 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+
+# region, zone must be defined in build file with --extra-vars flag!
+test_name: g4-onspot-slurm
+deployment_name: g4-onspot-slurm-{{ build }}
+slurm_cluster_name: "g4osp{{ build[0:5] }}"
+workspace: /workspace
+blueprint_yaml: "{{ workspace }}/examples/ml-slurm-g4.yaml"
+login_node: "{{ slurm_cluster_name }}-slurm-login-*"
+controller_node: "{{ slurm_cluster_name }}-controller"
+network: "{{ deployment_name }}-net-0"
+post_deploy_tests:
+- test-validation/test-mounts.yml
+- test-validation/test-partitions.yml
+- test-validation/test-default-partition.yml
+post_destroy_tasks:
+- post-destroy-tasks/delete-image.yml
+custom_vars:
+  gpu_count: 1
+  gpu_partition: g4
+  test_persistenced: true
+  partitions:
+  - g4
+  mounts:
+  - /home
+  instance_labels:
+    g4_onspot: true
+  enable_spot: true
+cli_deployment_vars:
+  region: "{{ region }}"
+  zone: "{{ zone }}"
+  slurm_cluster_name: "{{ slurm_cluster_name }}"
+  disk_size_gb: 100
+  cluster_size: 2
+  g4_enable_spot_vm: true