Skip to content

Commit 6697b00

Browse files
kvenkatachala333kvenkatachala333
authored
Integration of Slurm+GKE H4D Spot (GoogleCloudPlatform#5163)
1 parent 5167f67 commit 6697b00

5 files changed

Lines changed: 229 additions & 0 deletions

File tree

‎examples/hpc-slurm-h4d/hpc-slurm-h4d.yaml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,8 @@ deployment_groups:
4040

4141
- id: h4d-slurm-net-0
4242
source: modules/network/vpc
43+
settings:
44+
network_name: $(vars.deployment_name)-net
4345

4446
- id: h4d-rdma-net
4547
source: modules/network/vpc

‎tools/cloud-build/daily-tests/builds/gke-h4d-onspot.yaml‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
# Copyright 2026 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
---
16+
tags:
17+
- gke
18+
- m.gke-cluster
19+
- m.gke-node-pool
20+
- m.service-account
21+
- m.vpc
22+
- m.multivpc
23+
- m.kubectl-apply
24+
- m.filestore
25+
- m.gke-job-template
26+
- m.gke-persistent-volume
27+
28+
timeout: 14400s # 4hr
29+
steps:
30+
# While using static network names we are guarding against more than 1 instance running at a time (for multi-group tests)
31+
- id: check_for_running_build
32+
name: gcr.io/cloud-builders/gcloud
33+
script: "tools/cloud-build/check_running_build.sh tools/cloud-build/daily-tests/builds/gke-h4d-onspot.yaml"
34+
35+
- id: gke-h4d-onspot
36+
name: us-central1-docker.pkg.dev/$PROJECT_ID/hpc-toolkit-repo/test-runner
37+
entrypoint: /bin/bash
38+
env:
39+
- "ANSIBLE_HOST_KEY_CHECKING=false"
40+
- "ANSIBLE_CONFIG=/workspace/tools/cloud-build/ansible.cfg"
41+
- "PROJECT_ID=$PROJECT_ID"
42+
- "NUM_NODES=2"
43+
- "MACHINE_TYPE=h4d-highmem-192-lssd"
44+
- "INSTANCE_PREFIX=h4dspgke"
45+
- "BUILD_ID=$BUILD_ID"
46+
- "OPTIONS_GCS_PATH=gs://hpc-ctk1357/h4doptions.txt"
47+
args:
48+
- -c
49+
- |
50+
set -e -u -o pipefail
51+
echo "Sourcing find_available_zone.sh to determine zone."
52+
source /workspace/tools/cloud-build/find_available_zone.sh
53+
if [ -z "$${ZONE:-}" ]; then
54+
echo "ERROR: ZONE not found" >&2
55+
exit 1
56+
fi
57+
set -x -e
58+
cd /workspace && make
59+
REGION="$${ZONE%-*}"
60+
BUILD_ID_FULL=$BUILD_ID
61+
BUILD_ID_SHORT=$${BUILD_ID_FULL:0:6}
62+
EXAMPLE_BP=examples/gke-h4d/gke-h4d.yaml
63+
# adding vm to act as remote node
64+
echo ' - id: remote-node' >> $${EXAMPLE_BP}
65+
echo ' source: modules/compute/vm-instance' >> $${EXAMPLE_BP}
66+
echo ' use: [gke-h4d-net]' >> $${EXAMPLE_BP}
67+
echo ' settings:' >> $${EXAMPLE_BP}
68+
echo ' machine_type: e2-standard-2' >> $${EXAMPLE_BP}
69+
echo ' name_prefix: remote-node' >> $${EXAMPLE_BP}
70+
echo ' add_deployment_name_before_prefix: true' >> $${EXAMPLE_BP}
71+
sed -i -e '/reservation_affinity:/,+3c\ spot: true' $${EXAMPLE_BP}
72+
sed -i '/reservation/d' $${EXAMPLE_BP}
73+
sed -i '/placement_policy:/,+1d' $${EXAMPLE_BP}
74+
cat $${EXAMPLE_BP}
75+
ansible-playbook tools/cloud-build/daily-tests/ansible_playbooks/base-integration-test.yml \
76+
--user=sa_106486320838376751393 --extra-vars="project=${PROJECT_ID} build=$${BUILD_ID_SHORT}" \
77+
--extra-vars="region=$${REGION} zone=$${ZONE}" \
78+
--extra-vars="@tools/cloud-build/daily-tests/tests/gke-h4d-onspot.yml"

‎tools/cloud-build/daily-tests/builds/ml-h4d-onspot-slurm.yaml‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
# Copyright 2026 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
---
16+
tags:
17+
- m.startup-script
18+
- slurm6
19+
- m.filestore
20+
- m.schedmd-slurm-gcp-v6-controller
21+
- m.schedmd-slurm-gcp-v6-login
22+
- m.schedmd-slurm-gcp-v6-nodeset
23+
- m.schedmd-slurm-gcp-v6-partition
24+
- m.vpc
25+
26+
timeout: 14400s # 4hr
27+
steps:
28+
- id: check_for_running_build
29+
name: gcr.io/cloud-builders/gcloud
30+
script: "tools/cloud-build/check_running_build.sh tools/cloud-build/daily-tests/builds/ml-h4d-onspot-slurm.yaml"
31+
32+
- id: ml-h4d-onspot-slurm
33+
name: us-central1-docker.pkg.dev/$PROJECT_ID/hpc-toolkit-repo/test-runner
34+
entrypoint: /bin/bash
35+
env:
36+
# General Ansible configuration
37+
- "ANSIBLE_HOST_KEY_CHECKING=false"
38+
- "ANSIBLE_CONFIG=/workspace/tools/cloud-build/ansible.cfg"
39+
- "MACHINE_TYPE=h4d-highmem-192-lssd"
40+
- "NUM_NODES=4"
41+
- "INSTANCE_PREFIX=h4dspon"
42+
- "PROJECT_ID=$PROJECT_ID"
43+
- "BUILD_ID=$BUILD_ID"
44+
- "OPTIONS_GCS_PATH=gs://hpc-ctk1357/h4doptions.txt"
45+
args:
46+
- -c
47+
- |
48+
set -e -u -o pipefail
49+
echo "Sourcing find_available_zone.sh to determine zone."
50+
source /workspace/tools/cloud-build/find_available_zone.sh
51+
if [ -z "$${ZONE:-}" ]; then
52+
echo "ERROR: ZONE not found" >&2
53+
exit 1
54+
fi
55+
set -x
56+
cd /workspace && make
57+
REGION="$${ZONE%-*}"
58+
BUILD_ID_SHORT="$${BUILD_ID:0:6}"
59+
BLUEPRINT="/workspace/examples/hpc-slurm-h4d/hpc-slurm-h4d.yaml"
60+
ansible-playbook tools/cloud-build/daily-tests/ansible_playbooks/slurm-integration-test.yml \
61+
--user=sa_106486320838376751393 \
62+
--extra-vars="project=$${PROJECT_ID} build=$${BUILD_ID_SHORT}" \
63+
--extra-vars="region=$${REGION} zone=$${ZONE}" \
64+
--extra-vars="@tools/cloud-build/daily-tests/tests/ml-h4d-onspot-slurm.yml"

‎tools/cloud-build/daily-tests/tests/gke-h4d-onspot.yml‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
# Copyright 2026 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
---
16+
17+
# region, zone must be defined
18+
# in build file with --extra-vars flag!
19+
test_name: gke-h4d-spot
20+
deployment_name: gke-h4d-spot-{{ build }}
21+
workspace: /workspace
22+
blueprint_yaml: "{{ workspace }}/examples/gke-h4d/gke-h4d.yaml"
23+
network: "{{ deployment_name }}-net"
24+
remote_node: "{{ deployment_name }}-remote-node-0"
25+
static_node_count: 2
26+
cli_deployment_vars:
27+
region: "{{ region }}"
28+
zone: "{{ zone }}"
29+
static_node_count: "{{ static_node_count }}"
30+
authorized_cidr: "{{ build_ip.stdout }}/32"
31+
custom_vars:
32+
project: "{{ project }}"
33+
instance_labels:
34+
h4d_onspot: true
35+
enable_spot: true
36+
post_deploy_tests:
37+
- test-validation/test-gke-job.yml

‎tools/cloud-build/daily-tests/tests/ml-h4d-onspot-slurm.yml‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
# Copyright 2026 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
---
16+
17+
# region, zone must be defined in build file with --extra-vars flag!
18+
test_name: h4d-onspot-slurm
19+
deployment_name: h4d-onspot-slurm-{{ build }}
20+
slurm_cluster_name: "h4dsp{{ build[0:5] }}"
21+
workspace: /workspace
22+
blueprint_yaml: "{{ workspace }}/examples/hpc-slurm-h4d/hpc-slurm-h4d.yaml"
23+
login_node: "{{ slurm_cluster_name }}-slurm-login-*"
24+
controller_node: "{{ slurm_cluster_name }}-controller"
25+
network: "{{ deployment_name }}-net"
26+
post_deploy_tests:
27+
- test-validation/test-mounts.yml
28+
- test-validation/test-partitions.yml
29+
- test-validation/test-default-partition.yml
30+
custom_vars:
31+
gpu_count: 8
32+
gpu_partition: h4d
33+
test_persistenced: true
34+
partitions:
35+
- h4d
36+
mounts:
37+
- /home
38+
instance_labels:
39+
h4d_onspot: true
40+
enable_spot: true
41+
cli_deployment_vars:
42+
deployment_name: "{{ deployment_name }}"
43+
region: "{{ region }}"
44+
zone: "{{ zone }}"
45+
slurm_cluster_name: "{{ slurm_cluster_name }}"
46+
disk_size_gb: 100
47+
h4d_cluster_size: 2
48+
h4d_enable_spot_vm: true

0 commit comments

Comments
 (0)