Skip to content

Commit 9a12e2b

Browse files
simrankaurbsimrankaurb
committed
Enable spot VM Testing for GKE: a3ultra
1 parent febb93f commit 9a12e2b

3 files changed

Lines changed: 148 additions & 0 deletions

File tree

‎tools/cloud-build/daily-tests/ansible_playbooks/base-integration-test.yml‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -181,6 +181,10 @@
181181
vars:
182182
timeout_seconds: "{{ startup_timeout_seconds }}"
183183

184+
- name: Add instance labels for spot instances
185+
ansible.builtin.include_tasks: tasks/add-instance-labels.yml
186+
when: custom_vars is defined and custom_vars.enable_spot | default(false) | bool
187+
184188
- name: Run Integration tests for Cluster Toolkit
185189
ansible.builtin.include_tasks: "{{ test }}"
186190
vars:
@@ -191,6 +195,19 @@
191195
loop_control:
192196
loop_var: test
193197

198+
rescue:
199+
- name: Display test failure message
200+
ansible.builtin.debug:
201+
msg: "A task within the integration tests failed. Conditional preemption check will run for spot instances."
202+
203+
- name: Check for recent preemptions for spot instances
204+
ansible.builtin.include_tasks: test-validation/test-preemption.yml
205+
when: custom_vars is defined and custom_vars.enable_spot | default(false) | bool
206+
207+
- name: Propagate original failure after rescue tasks
208+
ansible.builtin.fail:
209+
msg: "Integration tests failed. Rescue tasks were executed."
210+
194211
always:
195212
- name: Cleanup firewall and infrastructure
196213
ansible.builtin.include_tasks:

‎tools/cloud-build/daily-tests/builds/gke-a3-ultragpu-onspot.yaml‎

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,88 @@
1+
# Copyright 2025 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
---
16+
tags:
17+
- m.gke-job-template
18+
- gke
19+
- m.gke-cluster
20+
- m.gke-node-pool
21+
- m.service-account
22+
- m.gpu-rdma-vpc
23+
- m.kubectl-apply
24+
- m.vpc
25+
- m.cloud-storage-bucket
26+
- m.gke-persistent-volume
27+
- m.pre-existing-network-storage
28+
29+
timeout: 14400s # 4hr
30+
steps:
31+
# While using static network names we are guarding against more than 1 instance running at a time (for multi-group tests)
32+
- id: check_for_running_build
33+
name: gcr.io/cloud-builders/gcloud
34+
script: "tools/cloud-build/check_running_build.sh tools/cloud-build/daily-tests/builds/gke-a3-ultragpu-onspot.yaml"
35+
36+
- id: gke-a3-ultragpu-onspot
37+
name: us-central1-docker.pkg.dev/$PROJECT_ID/hpc-toolkit-repo/test-runner
38+
entrypoint: /bin/bash
39+
env:
40+
- "ANSIBLE_HOST_KEY_CHECKING=false"
41+
- "ANSIBLE_CONFIG=/workspace/tools/cloud-build/ansible.cfg"
42+
- "PROJECT_ID=$PROJECT_ID"
43+
- "NUM_NODES=4"
44+
- "MACHINE_TYPE=a3-ultragpu-8g"
45+
- "INSTANCE_PREFIX=a3uspgke"
46+
- "BUILD_ID=$BUILD_ID"
47+
- "OPTIONS_GCS_PATH=gs://hpc-ctk1357/a3uoptions.txt"
48+
args:
49+
- -c
50+
- |
51+
set -e -u -o pipefail
52+
echo "Sourcing find_available_zone.sh to determine zone."
53+
source /workspace/tools/cloud-build/find_available_zone.sh
54+
if [ -z "$${ZONE:-}" ]; then
55+
echo "ERROR: ZONE not found" >&2
56+
exit 1
57+
fi
58+
set -x
59+
cd /workspace && make
60+
REGION="$${ZONE%-*}"
61+
BUILD_ID_SHORT=$${BUILD_ID:0:6}
62+
EXAMPLE_BP=examples/gke-a3-ultragpu/gke-a3-ultragpu.yaml
63+
64+
# adding vm to act as remote node
65+
echo ' - id: remote-node' >> $${EXAMPLE_BP}
66+
echo ' source: modules/compute/vm-instance' >> $${EXAMPLE_BP}
67+
echo ' use: [gke-a3-ultra-net-0]' >> $${EXAMPLE_BP}
68+
echo ' settings:' >> $${EXAMPLE_BP}
69+
echo ' machine_type: e2-standard-2' >> $${EXAMPLE_BP}
70+
echo ' name_prefix: remote-node' >> $${EXAMPLE_BP}
71+
echo ' add_deployment_name_before_prefix: true' >> $${EXAMPLE_BP}
72+
echo ''
73+
echo ' - id: job_template_hostname' >> $${EXAMPLE_BP}
74+
echo ' source: modules/compute/gke-job-template' >> $${EXAMPLE_BP}
75+
echo ' use: [a3-ultragpu-pool]' >> $${EXAMPLE_BP}
76+
echo ' settings:' >> $${EXAMPLE_BP}
77+
echo ' image: nvidia/cuda:11.0.3-runtime-ubuntu20.04' >> $${EXAMPLE_BP}
78+
echo ' command:' >> $${EXAMPLE_BP}
79+
echo ' - nvidia-smi' >> $${EXAMPLE_BP}
80+
echo ' node_count: 1' >> $${EXAMPLE_BP}
81+
echo ' outputs: [instructions]' >> $${EXAMPLE_BP}
82+
# Setting placement policy as compact for nccl test to run on spot VMs, adding spot variable as true and removing reservation from blueprint.
83+
sed -i -e '/reservation_affinity:/,+3c\ placement_policy:\n type: COMPACT\n spot: true' $${EXAMPLE_BP}
84+
sed -i '/reservation/d' $${EXAMPLE_BP}
85+
ansible-playbook tools/cloud-build/daily-tests/ansible_playbooks/base-integration-test.yml \
86+
--user=sa_106486320838376751393 --extra-vars="project=${PROJECT_ID} build=$${BUILD_ID_SHORT}" \
87+
--extra-vars="region=$${REGION} zone=$${ZONE}" \
88+
--extra-vars="@tools/cloud-build/daily-tests/tests/gke-a3-ultragpu-onspot.yml"

‎tools/cloud-build/daily-tests/tests/gke-a3-ultragpu-onspot.yml‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
# Copyright 2025 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
---
16+
17+
# region, zone must be defined
18+
# in build file with --extra-vars flag!
19+
test_name: gke-a3u-spot
20+
deployment_name: gke-a3u-spot-{{ build }}
21+
workspace: /workspace
22+
blueprint_yaml: "{{ workspace }}/examples/gke-a3-ultragpu/gke-a3-ultragpu.yaml"
23+
network: "{{ deployment_name }}-net-0"
24+
remote_node: "{{ deployment_name }}-remote-node-0"
25+
static_node_count: 2
26+
instance_type: a3-ultra
27+
accelerator_type: nvidia-h200-141gb
28+
num_gpus: 16
29+
cli_deployment_vars:
30+
region: "{{ region }}"
31+
zone: "{{ zone }}"
32+
static_node_count: "{{ static_node_count }}"
33+
authorized_cidr: "{{ build_ip.stdout }}/32"
34+
gcp_public_cidrs_access_enabled: true
35+
custom_vars:
36+
project: "{{ project }}"
37+
instance_labels:
38+
a3ultra_onspot: true
39+
enable_spot: true
40+
post_deploy_tests:
41+
- test-validation/test-gke-job.yml
42+
- test-validation/test-gke-a3-ultra.yml
43+
- test-validation/test-gke-kueue-config.yml

0 commit comments

Comments
 (0)