Skip to content

Commit 480ad71

Browse files
SwarnaBharathiMantenaSwarnaBharathiMantena
authored
Merge pull request GoogleCloudPlatform#4245 from SwarnaBharathiMantena/swarnabm/gke-a3u-dws-flexstart-tests
add integration tests for GKE A3U DWS flex-start and queued-provisioning
2 parents 2621630 + 8361c70 commit 480ad71

7 files changed

Lines changed: 360 additions & 2 deletions

File tree

‎examples/gke-consumption-options/dws-flex-start-queued-provisioning/gke-a3-ultragpu.yaml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,9 +22,9 @@ vars:
2222
region:
2323
zone:
2424
# Enable DWS Flex Start.
25-
enable_flex_start:
25+
enable_flex_start: true
2626
# Enable queued provisioning.
27-
enable_queued_provisioning:
27+
enable_queued_provisioning: true
2828
# Cidr block containing the IP of the machine calling terraform.
2929
# The following line must be updated for this example to work.
3030
authorized_cidr:

‎tools/cloud-build/daily-tests/ansible_playbooks/test-validation/test-gke-a3u-dws-flex-start-queued-provisioning.yml‎

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,91 @@
1+
# Copyright 2024 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
- name: Assert variables are defined
16+
ansible.builtin.assert:
17+
that:
18+
- region is defined
19+
- custom_vars.project is defined
20+
21+
- name: Get cluster credentials for kubectl
22+
delegate_to: localhost
23+
ansible.builtin.command: gcloud container clusters get-credentials {{ deployment_name }} --region {{ region }} --project {{ custom_vars.project }}
24+
25+
- name: Create a job
26+
delegate_to: localhost
27+
ansible.builtin.shell: |
28+
array=({{ workspace }}/examples/gke-consumption-options/dws-flex-start-queued-provisioning/sample-job.yaml)
29+
kubectl create -f ${array[0]}
30+
echo ${array[0]}
31+
args:
32+
executable: /bin/bash
33+
changed_when: False
34+
35+
- name: Wait for job to complete
36+
delegate_to: localhost
37+
ansible.builtin.command: |
38+
kubectl get job --field-selector status.successful=1
39+
register: job_completion
40+
until: job_completion.stdout_lines | length > 1
41+
retries: 30
42+
delay: 60
43+
44+
- name: Print job_completion debug output
45+
ansible.builtin.debug:
46+
var: job_completion.stdout_lines
47+
48+
- name: Clean up
49+
delegate_to: localhost
50+
ansible.builtin.shell: |
51+
kubectl delete jobset --all
52+
53+
- name: Run the NCCL test
54+
delegate_to: localhost
55+
ansible.builtin.shell: |
56+
kubectl create -f {{ workspace }}/examples/gke-consumption-options/dws-flex-start-queued-provisioning/nccl-jobset-example.yaml
57+
args:
58+
executable: /bin/bash
59+
60+
- name: Wait for Job to hit 2/2 completions
61+
delegate_to: localhost
62+
ansible.builtin.shell: |
63+
kubectl get job --field-selector status.successful=2
64+
register: job_completion
65+
until: job_completion.stdout_lines | length > 1
66+
retries: 60
67+
delay: 60
68+
69+
- name: Fetch logs from the 0-0 pod and save to pod_logs.txt
70+
delegate_to: localhost
71+
ansible.builtin.shell: |
72+
pod_0_0_name="$(kubectl get pods --no-headers -o custom-columns="Name:.metadata.name" | awk '{if ($1 ~ "0-0") print $0}')"
73+
kubectl logs ${pod_0_0_name} > pod_logs.txt
74+
cat pod_logs.txt
75+
register: nccl_test_logs
76+
77+
- name: Print the NCCL test logs from 0-0 pod
78+
debug:
79+
msg: "{{nccl_test_logs.stdout}}"
80+
81+
- name: Ensure average bus bandwidth is >= 100 GB/s
82+
delegate_to: localhost
83+
ansible.builtin.shell: |
84+
grep -o '# Avg bus bandwidth : [0-9\.]*' pod_logs.txt | cut -d ':' -f 2 | awk '{$1=$1;print}'
85+
register: avg_bus_bandwidth
86+
failed_when: avg_bus_bandwidth.stdout | float < 100
87+
88+
- name: Clean up
89+
delegate_to: localhost
90+
ansible.builtin.shell: |
91+
kubectl delete jobset --all

‎tools/cloud-build/daily-tests/ansible_playbooks/test-validation/test-gke-a3u-dws-flex-start.yml‎

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,91 @@
1+
# Copyright 2024 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
- name: Assert variables are defined
16+
ansible.builtin.assert:
17+
that:
18+
- region is defined
19+
- custom_vars.project is defined
20+
21+
- name: Get cluster credentials for kubectl
22+
delegate_to: localhost
23+
ansible.builtin.command: gcloud container clusters get-credentials {{ deployment_name }} --region {{ region }} --project {{ custom_vars.project }}
24+
25+
- name: Create a job
26+
delegate_to: localhost
27+
ansible.builtin.shell: |
28+
array=({{ workspace }}/examples/gke-consumption-options/dws-flex-start/dws-flex-start.yaml)
29+
kubectl create -f ${array[0]}
30+
echo ${array[0]}
31+
args:
32+
executable: /bin/bash
33+
changed_when: False
34+
35+
- name: Wait for job to complete
36+
delegate_to: localhost
37+
ansible.builtin.command: |
38+
kubectl get job --field-selector status.successful=1
39+
register: job_completion
40+
until: job_completion.stdout_lines | length > 1
41+
retries: 30
42+
delay: 60
43+
44+
- name: Print job_completion debug output
45+
ansible.builtin.debug:
46+
var: job_completion.stdout_lines
47+
48+
- name: Clean up
49+
delegate_to: localhost
50+
ansible.builtin.shell: |
51+
kubectl delete jobset --all
52+
53+
- name: Run the NCCL test
54+
delegate_to: localhost
55+
ansible.builtin.shell: |
56+
kubectl create -f {{ workspace }}/examples/gke-consumption-options/dws-flex-start/nccl-jobset-example.yaml
57+
args:
58+
executable: /bin/bash
59+
60+
- name: Wait for NCCL Job to hit 2/2 completions
61+
delegate_to: localhost
62+
ansible.builtin.shell: |
63+
kubectl get job --field-selector status.successful=2
64+
register: job_completion
65+
until: job_completion.stdout_lines | length > 1
66+
retries: 60
67+
delay: 60
68+
69+
- name: Fetch logs from the 0-0 pod and save to pod_logs.txt
70+
delegate_to: localhost
71+
ansible.builtin.shell: |
72+
pod_0_0_name="$(kubectl get pods --no-headers -o custom-columns="Name:.metadata.name" | awk '{if ($1 ~ "0-0") print $0}')"
73+
kubectl logs ${pod_0_0_name} > pod_logs.txt
74+
cat pod_logs.txt
75+
register: nccl_test_logs
76+
77+
- name: Print the NCCL test logs from 0-0 pod
78+
debug:
79+
msg: "{{nccl_test_logs.stdout}}"
80+
81+
- name: Ensure average bus bandwidth is >= 100 GB/s
82+
delegate_to: localhost
83+
ansible.builtin.shell: |
84+
grep -o '# Avg bus bandwidth : [0-9\.]*' pod_logs.txt | cut -d ':' -f 2 | awk '{$1=$1;print}'
85+
register: avg_bus_bandwidth
86+
failed_when: avg_bus_bandwidth.stdout | float < 100
87+
88+
- name: Clean up
89+
delegate_to: localhost
90+
ansible.builtin.shell: |
91+
kubectl delete jobset --all

‎tools/cloud-build/daily-tests/builds/gke-a3u-dws-flex-start-queued-provisioning.yaml‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
# Copyright 2023 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
---
16+
tags:
17+
- gke
18+
- m.gke-cluster
19+
- m.gke-node-pool
20+
- m.service-account
21+
- m.gpu-rdma-vpc
22+
- m.kubectl-apply
23+
- m.vpc
24+
25+
timeout: 14400s # 4hr
26+
steps:
27+
- id: gke-dws-flex-start
28+
name: us-central1-docker.pkg.dev/$PROJECT_ID/hpc-toolkit-repo/test-runner
29+
entrypoint: /bin/bash
30+
env:
31+
- "ANSIBLE_HOST_KEY_CHECKING=false"
32+
- "ANSIBLE_CONFIG=/workspace/tools/cloud-build/ansible.cfg"
33+
args:
34+
- -c
35+
- |
36+
set -x -e
37+
cd /workspace && make
38+
BUILD_ID_FULL=$BUILD_ID
39+
BUILD_ID_SHORT=$${BUILD_ID_FULL:0:6}
40+
EXAMPLE_BP=examples/gke-consumption-options/dws-flex-start-queued-provisioning/gke-a3-ultragpu.yaml
41+
42+
# adding vm to act as remote node
43+
echo ' - id: remote-node' >> $${EXAMPLE_BP}
44+
echo ' source: modules/compute/vm-instance' >> $${EXAMPLE_BP}
45+
echo ' use: [gke-a3-ultra-net-0]' >> $${EXAMPLE_BP}
46+
echo ' settings:' >> $${EXAMPLE_BP}
47+
echo ' machine_type: e2-standard-2' >> $${EXAMPLE_BP}
48+
echo ' name_prefix: remote-node' >> $${EXAMPLE_BP}
49+
echo ' add_deployment_name_before_prefix: true' >> $${EXAMPLE_BP}
50+
51+
ansible-playbook tools/cloud-build/daily-tests/ansible_playbooks/base-integration-test.yml \
52+
--user=sa_106486320838376751393 --extra-vars="project=${PROJECT_ID} build=$${BUILD_ID_SHORT}" \
53+
--extra-vars="@tools/cloud-build/daily-tests/tests/gke-a3u-dws-flex-start-queued-provisioning.yml"

‎tools/cloud-build/daily-tests/builds/gke-a3u-dws-flex-start.yaml‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
# Copyright 2023 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
---
16+
tags:
17+
- gke
18+
- m.gke-cluster
19+
- m.gke-node-pool
20+
- m.service-account
21+
- m.gpu-rdma-vpc
22+
- m.kubectl-apply
23+
- m.vpc
24+
25+
timeout: 14400s # 4hr
26+
steps:
27+
- id: gke-dws-flex-start
28+
name: us-central1-docker.pkg.dev/$PROJECT_ID/hpc-toolkit-repo/test-runner
29+
entrypoint: /bin/bash
30+
env:
31+
- "ANSIBLE_HOST_KEY_CHECKING=false"
32+
- "ANSIBLE_CONFIG=/workspace/tools/cloud-build/ansible.cfg"
33+
args:
34+
- -c
35+
- |
36+
set -x -e
37+
cd /workspace && make
38+
BUILD_ID_FULL=$BUILD_ID
39+
BUILD_ID_SHORT=$${BUILD_ID_FULL:0:6}
40+
EXAMPLE_BP=examples/gke-consumption-options/dws-flex-start/gke-a3-ultragpu.yaml
41+
42+
# adding vm to act as remote node
43+
echo ' - id: remote-node' >> $${EXAMPLE_BP}
44+
echo ' source: modules/compute/vm-instance' >> $${EXAMPLE_BP}
45+
echo ' use: [gke-a3-ultra-net-0]' >> $${EXAMPLE_BP}
46+
echo ' settings:' >> $${EXAMPLE_BP}
47+
echo ' machine_type: e2-standard-2' >> $${EXAMPLE_BP}
48+
echo ' name_prefix: remote-node' >> $${EXAMPLE_BP}
49+
echo ' add_deployment_name_before_prefix: true' >> $${EXAMPLE_BP}
50+
51+
ansible-playbook tools/cloud-build/daily-tests/ansible_playbooks/base-integration-test.yml \
52+
--user=sa_106486320838376751393 --extra-vars="project=${PROJECT_ID} build=$${BUILD_ID_SHORT}" \
53+
--extra-vars="@tools/cloud-build/daily-tests/tests/gke-a3u-dws-flex-start.yml"

‎tools/cloud-build/daily-tests/tests/gke-a3u-dws-flex-start-queued-provisioning.yml‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
# Copyright 2025 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
---
16+
17+
# region, zone must be defined
18+
# in build file with --extra-vars flag!
19+
test_name: gke-a3u-dws-flex-start
20+
deployment_name: gke-dws-fs-{{ build }}
21+
workspace: /workspace
22+
blueprint_yaml: "{{ workspace }}/examples/gke-consumption-options/dws-flex-start-queued-provisioning/gke-a3-ultragpu.yaml"
23+
network: "{{ deployment_name }}-net-0"
24+
region: europe-west1
25+
zone: europe-west1-b
26+
remote_node: "{{ deployment_name }}-remote-node-0"
27+
cli_deployment_vars:
28+
region: "{{ region }}"
29+
zone: "{{ zone }}"
30+
authorized_cidr: "{{ build_ip.stdout }}/32"
31+
gcp_public_cidrs_access_enabled: false
32+
custom_vars:
33+
project: "{{ project }}"
34+
post_deploy_tests:
35+
- test-validation/test-gke-a3u-dws-flex-start-queued-provisioning.yml

‎tools/cloud-build/daily-tests/tests/gke-a3u-dws-flex-start.yml‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
# Copyright 2025 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
---
16+
17+
# region, zone must be defined
18+
# in build file with --extra-vars flag!
19+
test_name: gke-a3u-dws-flex-start
20+
deployment_name: gke-dws-fs-{{ build }}
21+
workspace: /workspace
22+
blueprint_yaml: "{{ workspace }}/examples/gke-consumption-options/dws-flex-start/gke-a3-ultragpu.yaml"
23+
network: "{{ deployment_name }}-net-0"
24+
region: europe-west1
25+
zone: europe-west1-b
26+
remote_node: "{{ deployment_name }}-remote-node-0"
27+
cli_deployment_vars:
28+
region: "{{ region }}"
29+
zone: "{{ zone }}"
30+
authorized_cidr: "{{ build_ip.stdout }}/32"
31+
gcp_public_cidrs_access_enabled: false
32+
custom_vars:
33+
project: "{{ project }}"
34+
post_deploy_tests:
35+
- test-validation/test-gke-a3u-dws-flex-start.yml

0 commit comments

Comments
 (0)