Skip to content

Commit ee0b5d1

Browse files
harshthakkar01harshthakkar01
authored
Merge pull request #4299 from harshthakkar01/publish-hcs-images
Add recipe to build HCS images
2 parents a6d59d2 + c5db828 commit ee0b5d1

6 files changed

Lines changed: 673 additions & 1 deletion

File tree

‎examples/machine-learning/build-service-images/README.md‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
# Build Service Images
2+
3+
This process will build 2 images. One for a3-megagpu-8g and another one that
4+
can be used for all other machines. Each image it may take up to ~60 min to
5+
build. You may choose to run the two commands in parallel in 2 separate shells.
6+
7+
## Requirements
8+
9+
Make sure `gcluster`, `gcloud`, `terraform` and `packer`
10+
binaries to be available in `PATH`.
11+
12+
## Build A3M image
13+
14+
Run the following commands,
15+
16+
```sh
17+
cd cluster-toolkit/examples/machine-learning/build-service-images/
18+
PROJECT=<your-project-id>
19+
SUFFIX=slurm-image
20+
./build.sh a3m $SUFFIX $PROJECT
21+
```
22+
23+
## Build common image
24+
25+
Run the following commands,
26+
27+
```sh
28+
cd cluster-toolkit/examples/machine-learning/build-service-images/
29+
PROJECT=<your-project-id>
30+
SUFFIX=slurm-image
31+
./build.sh common $SUFFIX $PROJECT
32+
```
33+
34+
## Next Steps
35+
36+
The two images that were build can be found under `*-slurm-image` family. Use the following
37+
gcloud command to describe the images and confirm they were built.
38+
39+
```shell
40+
gcloud compute images describe-from-family a3m-slurm-image
41+
gcloud compute images describe-from-family common-slurm-image
42+
```
43+
44+
## Troubleshooting
45+
46+
If packer fails during execution of startup script it will output a
47+
command for grabbing logs, e.g.:
48+
49+
```shell
50+
==> roll.googlecompute.toolkit_image: Error waiting for startup script to finish: Startup script exited with error.
51+
==> roll.googlecompute.toolkit_image: Provisioning step had errors: Running the cleanup provisioner, if present...
52+
==> roll.googlecompute.toolkit_image: Running local shell script: /tmp/packer-shell2734275589
53+
roll.googlecompute.toolkit_image: Error building image try checking logs:
54+
roll.googlecompute.toolkit_image: gcloud logging --project <project-id> read 'logName=("projects/<project-id>/logs/GCEMetadataScripts" OR "projects/<project-id>/logs/google_metadata_script_runner") AND resource.labels.instance_id=<id>' --format="table(timestamp, resource.labels.instance_id, jsonPayload.message)" --order=asc
55+
```
56+
57+
**IMPORTANT:** If `build.sh` fails it will not clean up supporting
58+
infrastructure (VPC), please perform following:
59+
60+
```sh
61+
gcluster destroy /tmp/build_a3m-slurm-image/roll --auto-approve
62+
gcluster destroy /tmp/build_common-slurm-image/roll --auto-approve
63+
```

‎examples/machine-learning/build-service-images/a3m/blueprint.yaml‎

Lines changed: 306 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,306 @@
1+
# Copyright 2025 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
---
16+
17+
blueprint_name: roll-a3m-image
18+
19+
deployment_groups:
20+
- group: image-env
21+
modules:
22+
- id: image-net
23+
source: modules/network/vpc
24+
25+
- id: build-script
26+
source: modules/scripts/startup-script
27+
settings:
28+
install_ansible: true
29+
docker:
30+
enabled: true
31+
world_writable: true
32+
runners:
33+
- type: data
34+
destination: /etc/apt/preferences.d/block-broken-nvidia-container
35+
content: |
36+
Package: nvidia-container-toolkit nvidia-container-toolkit-base libnvidia-container-tools libnvidia-container1
37+
Pin: version 1.17.7-1
38+
Pin-Priority: 100
39+
- $(vars.runner_disable_unattended_upgrades)
40+
- type: shell
41+
destination: hold_google_services.sh
42+
content: |
43+
#!/bin/bash
44+
set -e -o pipefail
45+
apt-mark hold google-compute-engine
46+
apt-mark hold google-compute-engine-oslogin
47+
apt-mark hold google-guest-agent
48+
apt-mark hold google-osconfig-agent
49+
- type: data
50+
destination: /var/tmp/slurm_vars.json
51+
content: |
52+
{
53+
"reboot": false,
54+
"install_cuda": false,
55+
"install_ompi": true,
56+
"install_lustre": false,
57+
"install_managed_lustre": true,
58+
"install_gcsfuse": true,
59+
"monitoring_agent": "cloud-ops",
60+
"use_open_drivers": true
61+
}
62+
- type: shell
63+
destination: install_git.sh
64+
content: |
65+
#!/bin/bash
66+
set -e -o pipefail
67+
apt-get update
68+
apt-get install -y git
69+
- $(vars.runner_install_slurm)
70+
- type: ansible-local
71+
destination: update-gvnic.yml
72+
content: |
73+
---
74+
- name: Install updated gVNIC driver from GitHub
75+
hosts: all
76+
become: true
77+
vars:
78+
package_url: https://github.com/GoogleCloudPlatform/compute-virtual-ethernet-linux/releases/download/v1.4.3/gve-dkms_1.4.3_all.deb
79+
package_filename: /tmp/{{ package_url | basename }}
80+
tasks:
81+
- name: Install driver dependencies
82+
ansible.builtin.apt:
83+
name:
84+
- dkms
85+
- name: Download gVNIC package
86+
ansible.builtin.get_url:
87+
url: "{{ package_url }}"
88+
dest: "{{ package_filename }}"
89+
- name: Install updated gVNIC
90+
ansible.builtin.apt:
91+
deb: "{{ package_filename }}"
92+
state: present
93+
- type: shell
94+
destination: install-nvidia.sh
95+
content: |
96+
#!/bin/bash
97+
set -ex -o pipefail
98+
# Install nvidia container toolkit
99+
curl -fsSL https://nvidia.github.io/libnvidia-container/gpgkey | \
100+
gpg --dearmor -o /usr/share/keyrings/nvidia-container-toolkit-keyring.gpg && \
101+
curl -s -L https://nvidia.github.io/libnvidia-container/stable/deb/nvidia-container-toolkit.list | \
102+
sed 's#deb https://#deb [signed-by=/usr/share/keyrings/nvidia-container-toolkit-keyring.gpg] https://#g' | \
103+
tee /etc/apt/sources.list.d/nvidia-container-toolkit.list
104+
apt-get update -y
105+
apt-get install -y nvidia-container-toolkit
106+
# this duplicates the ulimits configuration of the HPC VM Image
107+
- $(vars.runner_setup_hpc_vm_image_ulimits)
108+
- type: data
109+
destination: /etc/enroot/enroot.conf
110+
content: |
111+
ENROOT_CONFIG_PATH ${HOME}/.enroot
112+
ENROOT_RUNTIME_PATH /mnt/localssd/${UID}/enroot/runtime
113+
ENROOT_CACHE_PATH /mnt/localssd/${UID}/enroot/cache
114+
ENROOT_DATA_PATH /mnt/localssd/${UID}/enroot/data
115+
ENROOT_TEMP_PATH /mnt/localssd/${UID}/enroot
116+
- type: ansible-local
117+
destination: configure_gpu_monitoring.yml
118+
content: |
119+
---
120+
- name: Install NVIDIA DCGM and Configure Ops Agent
121+
hosts: all
122+
become: true
123+
vars:
124+
distribution: "{{ ansible_distribution | lower }}{{ ansible_distribution_version | replace('.','') }}"
125+
package_url: https://developer.download.nvidia.com/compute/cuda/repos/{{ distribution }}/x86_64/cuda-keyring_1.1-1_all.deb
126+
package_filename: /tmp/{{ package_url | basename }}
127+
enable_ops_agent: true
128+
enable_nvidia_dcgm: true
129+
nvidia_packages:
130+
- cuda-toolkit-12-8
131+
- datacenter-gpu-manager-4-cuda12
132+
- libnvidia-cfg1-570-server
133+
- libnvidia-nscq-570
134+
- nvidia-compute-utils-570-server
135+
tasks:
136+
- name: Download NVIDIA repository package
137+
ansible.builtin.get_url:
138+
url: "{{ package_url }}"
139+
dest: "{{ package_filename }}"
140+
- name: Install NVIDIA repository package
141+
ansible.builtin.apt:
142+
deb: "{{ package_filename }}"
143+
state: present
144+
- name: Reduce NVIDIA repository priority
145+
ansible.builtin.copy:
146+
dest: /etc/apt/preferences.d/cuda-repository-pin-600
147+
mode: 0o0644
148+
owner: root
149+
group: root
150+
content: |
151+
Package: nsight-compute
152+
Pin: origin *ubuntu.com*
153+
Pin-Priority: -1
154+
155+
Package: nsight-systems
156+
Pin: origin *ubuntu.com*
157+
Pin-Priority: -1
158+
159+
Package: *
160+
Pin: release l=NVIDIA CUDA
161+
Pin-Priority: 400
162+
- name: Install NVIDIA fabric and CUDA
163+
ansible.builtin.apt:
164+
name: "{{ item }}"
165+
update_cache: true
166+
loop: "{{ nvidia_packages }}"
167+
- name: Freeze NVIDIA fabric and CUDA
168+
ansible.builtin.dpkg_selections:
169+
name: "{{ item }}"
170+
selection: hold
171+
loop: "{{ nvidia_packages }}"
172+
- name: Create nvidia-persistenced override directory
173+
ansible.builtin.file:
174+
path: /etc/systemd/system/nvidia-persistenced.service.d
175+
state: directory
176+
owner: root
177+
group: root
178+
mode: 0o755
179+
- name: Configure nvidia-persistenced override
180+
ansible.builtin.copy:
181+
dest: /etc/systemd/system/nvidia-persistenced.service.d/persistence_mode.conf
182+
owner: root
183+
group: root
184+
mode: 0o644
185+
content: |
186+
[Service]
187+
ExecStart=
188+
ExecStart=/usr/bin/nvidia-persistenced --user nvidia-persistenced --verbose
189+
notify: Reload SystemD
190+
handlers:
191+
- name: Reload SystemD
192+
ansible.builtin.systemd:
193+
daemon_reload: true
194+
post_tasks:
195+
- name: Enable Google Cloud Ops Agent
196+
ansible.builtin.service:
197+
name: google-cloud-ops-agent.service
198+
state: "{{ 'started' if enable_ops_agent else 'stopped' }}"
199+
enabled: "{{ enable_ops_agent }}"
200+
- name: Disable NVIDIA DCGM by default (enable during boot on GPU nodes)
201+
ansible.builtin.service:
202+
name: nvidia-dcgm.service
203+
state: stopped
204+
enabled: false
205+
- name: Disable nvidia-persistenced SystemD unit (enable during boot on GPU nodes)
206+
ansible.builtin.service:
207+
name: nvidia-persistenced.service
208+
state: stopped
209+
enabled: false
210+
- type: ansible-local
211+
destination: install_dmabuf.yml
212+
content: |
213+
---
214+
- name: Install DMBABUF import helper
215+
hosts: all
216+
become: true
217+
tasks:
218+
- name: Setup apt-transport-artifact-registry repository
219+
ansible.builtin.apt_repository:
220+
repo: deb http://packages.cloud.google.com/apt apt-transport-artifact-registry-stable main
221+
state: present
222+
- name: Install driver dependencies
223+
ansible.builtin.apt:
224+
name:
225+
- dkms
226+
- apt-transport-artifact-registry
227+
- name: Setup gpudirect-tcpxo apt repository
228+
ansible.builtin.apt_repository:
229+
repo: deb [arch=all trusted=yes ] ar+https://us-apt.pkg.dev/projects/gce-ai-infra gpudirect-tcpxo-apt main
230+
state: present
231+
- name: Install DMABUF import helper DKMS package
232+
ansible.builtin.apt:
233+
name: dmabuf-import-helper
234+
state: present
235+
- type: ansible-local
236+
destination: aperture_devices.yml # This still needs to be run during startup http://google3/cloud/hosted/hypercomputecluster/terraform/slurm/nodeset_a3m.tf;l=16;rcl=767407261
237+
content: |
238+
---
239+
- name: Setup GPUDirect-TCPXO aperture devices
240+
hosts: all
241+
become: true
242+
tasks:
243+
- name: Mount aperture devices to /dev and make writable
244+
ansible.builtin.copy:
245+
dest: /etc/udev/rules.d/00-a3-megagpu.rules
246+
owner: root
247+
group: root
248+
mode: 0o644
249+
content: |
250+
ACTION=="add", SUBSYSTEM=="pci", ATTR{vendor}=="0x1ae0", ATTR{device}=="0x0084", TAG+="systemd", \
251+
RUN+="/usr/bin/mkdir --mode=0755 -p /dev/aperture_devices", \
252+
RUN+="/usr/bin/systemd-mount --type=none --options=bind --collect %S/%p /dev/aperture_devices/%k", \
253+
RUN+="/usr/bin/bash -c '/usr/bin/chmod 0666 /dev/aperture_devices/%k/resource*'"
254+
notify: Update initramfs
255+
handlers:
256+
- name: Update initramfs
257+
ansible.builtin.command: /usr/sbin/update-initramfs -u -k all
258+
- type: shell
259+
destination: remove_snap_gcloud.sh
260+
content: |
261+
#!/bin/bash
262+
# IMPORTANT: This script should be run *last* in any sequence of setup steps
263+
# that use 'gsutil' or other gcloud commands.
264+
# This is because removing the Snap version of the GCloud SDK can temporarily
265+
# break existing 'gsutil' paths, which might disrupt other scripts still running
266+
# that rely on the Snap-installed version.
267+
268+
set -e -o pipefail
269+
270+
# Remove the previously installed Google Cloud SDK (google-cloud-cli) and
271+
# the LXD container manager, both of which might have been installed via Snap.
272+
# This step is crucial to prevent conflicts with the upcoming APT installation
273+
# and address potential issues with Snapd and NFS mounts in specific environments
274+
snap remove google-cloud-cli lxd
275+
# Install key and google-cloud-cli from apt repo
276+
GCLOUD_APT_SOURCE="/etc/apt/sources.list.d/google-cloud-sdk.list"
277+
if [ ! -f "${GCLOUD_APT_SOURCE}" ]; then
278+
# indentation matters in EOT below; do not blindly edit!
279+
cat <<EOT > "${GCLOUD_APT_SOURCE}"
280+
deb [signed-by=/usr/share/keyrings/cloud.google.asc] https://packages.cloud.google.com/apt cloud-sdk main
281+
EOT
282+
fi
283+
curl -o /usr/share/keyrings/cloud.google.asc https://packages.cloud.google.com/apt/doc/apt-key.gpg
284+
apt-get update
285+
apt-get install --assume-yes google-cloud-cli
286+
# Clean up the bash executable hash for subsequent steps using gsutil
287+
hash -r
288+
289+
290+
- group: image
291+
modules:
292+
- id: image
293+
source: modules/packer/custom-image
294+
kind: packer
295+
use:
296+
- image-net
297+
- build-script
298+
settings:
299+
disk_size: 100
300+
machine_type: c2-standard-8
301+
302+
source_image_family: ubuntu-accelerator-2204-amd64-with-nvidia-570
303+
source_image_project_id: [ubuntu-os-accelerator-images]
304+
305+
image_family: $(vars.family)
306+
omit_external_ip: false

0 commit comments

Comments
 (0)