Skip to content

Commit 56991d8

Browse files
LAVEENLAVEEN
authored
Adding ML dependencies for G4 & guidance to use dual NIC (GoogleCloudPlatform#4922)
2 parents 057decf + 00719c4 commit 56991d8

1 file changed

Lines changed: 281 additions & 61 deletions

File tree

‎examples/ml-slurm-g4.yaml‎

Lines changed: 281 additions & 61 deletions
Original file line numberDiff line numberDiff line change
@@ -13,102 +13,322 @@
1313
# limitations under the License.
1414

1515
---
16-
1716
blueprint_name: ml-slurm-g4
1817

1918
vars:
20-
project_id: ## Set GCP Project ID Here ##
21-
deployment_name: slurm-g4
19+
project_id: ## Set Project ID ##
20+
deployment_name: g4-ml-example
2221
region: us-central1
2322
zone: us-central1-b
24-
g4_reservation_name: "" ## Set Reservation Name
25-
disk_size_gb: 200
26-
new_image:
27-
family: slurm-gcp-6-11-ubuntu-2204-lts-nvidia-570
28-
project: schedmd-slurm-public
29-
30-
# Documentation for each of the modules used below can be found at
31-
# https://github.com/GoogleCloudPlatform/hpc-toolkit/blob/main/modules/README.md
23+
g4_machine_type: g4-standard-48
24+
# To enable Dual NIC, you MUST use g4-standard-384
25+
# g4_machine_type: g4-standard-384
26+
cluster_size: 2 # Number of static compute nodes
27+
# Image settings
28+
source_image_family: ubuntu-2404-lts-amd64
29+
source_image_project: ubuntu-os-cloud
30+
image_build_machine_type: n2-standard-4
31+
image_disk_size_gb: 50
32+
built_image_family: slurm-g4-ubuntu2404
33+
34+
build_slurm_from_git_ref: 6.10.6 # Or a newer stable tag
35+
36+
# Cluster env settings
37+
net0_range: 192.168.0.0/24
38+
# net1_range: 192.168.2.0/24 # Uncomment for Dual NIC on g4-standard-384
39+
filestore_ip_range: 192.168.1.0/29
40+
41+
# Cluster Settings
42+
disk_size_gb: 100
43+
3244
deployment_groups:
33-
- group: primary
45+
- group: image-env
3446
modules:
35-
- id: network
47+
- id: builder-net
3648
source: modules/network/vpc
3749
settings:
50+
network_name: $(vars.deployment_name)-build-net
51+
52+
- group: image-build-script
53+
modules:
54+
- id: slurm-build-script
55+
source: modules/scripts/startup-script
56+
settings:
57+
runners:
58+
# Install nvidia drivers/utilities
59+
- type: ansible-local
60+
destination: install_nvidia_drivers.yml
61+
content: |
62+
# ---
63+
- name: Install NVIDIA Drivers and Utils
64+
hosts: all
65+
become: true
66+
vars:
67+
distribution: "{{ ansible_distribution | lower }}{{ ansible_distribution_version | replace('.','') }}"
68+
cuda_repo_url: https://developer.download.nvidia.com/compute/cuda/repos/{{ distribution }}/x86_64/cuda-keyring_1.1-1_all.deb
69+
cuda_repo_filename: /tmp/{{ cuda_repo_url | basename }}
70+
enable_nvidia_dcgm: false
71+
nvidia_packages:
72+
# 1. Open Kernel Driver: Installs the open-source kernel components.
73+
- nvidia-kernel-open-575
74+
# 2. Driver Metapackage: Pulls in the 580.xx driver version.
75+
- cuda-drivers-575
76+
# 3. CUDA Toolkit: Installs the 12.9.1 toolkit.
77+
- cuda-toolkit-12-9
78+
- datacenter-gpu-manager
79+
80+
tasks:
81+
# Download the NVIDIA repo
82+
- name: Download NVIDIA repository package
83+
ansible.builtin.get_url:
84+
url: "{{ cuda_repo_url }}"
85+
dest: "{{ cuda_repo_filename }}"
86+
87+
# Install the NVIDIA repo
88+
- name: Install NVIDIA repository package
89+
ansible.builtin.apt:
90+
deb: "{{ cuda_repo_filename }}"
91+
state: present
92+
93+
# Update apt cache
94+
- name: Update apt cache
95+
ansible.builtin.apt:
96+
update_cache: true
97+
98+
# Install the package
99+
- name: Install NVIDIA fabric and CUDA
100+
ansible.builtin.apt:
101+
name: "{{ item }}"
102+
loop: "{{ nvidia_packages }}"
103+
104+
# This section will optionally freeze the packages with
105+
# an apt-mark hold.
106+
- name: Freeze NVIDIA fabric and CUDA
107+
ansible.builtin.dpkg_selections:
108+
name: "{{ item }}"
109+
selection: hold
110+
loop: "{{ nvidia_packages }}"
111+
112+
113+
- type: data
114+
destination: /var/tmp/slurm_vars.json
115+
content: |
116+
{
117+
"reboot": false,
118+
"install_ompi": true,
119+
"install_lustre": false,
120+
"install_gcsfuse": false,
121+
"install_cuda": false,
122+
"allow_kernel_upgrades": false,
123+
"monitoring_agent": "cloud-ops",
124+
"install_managed_lustre": false
125+
}
126+
127+
- type: shell
128+
destination: install_slurm.sh
129+
content: |
130+
#!/bin/bash
131+
set -e -o pipefail
132+
sudo apt update
133+
ansible-pull \
134+
-U https://github.com/GoogleCloudPlatform/slurm-gcp -C $(vars.build_slurm_from_git_ref) \
135+
-i localhost, --limit localhost --connection=local \
136+
-e @/var/tmp/slurm_vars.json \
137+
ansible/playbook.yml
138+
139+
- group: image
140+
modules:
141+
- id: slurm-g4-image
142+
source: modules/packer/custom-image
143+
kind: packer
144+
settings:
145+
disk_size: $(vars.image_disk_size_gb)
146+
machine_type: $(vars.image_build_machine_type)
147+
source_image_family: $(vars.source_image_family)
148+
source_image_project_id: [$(vars.source_image_project)]
149+
image_family: $(vars.built_image_family)
150+
omit_external_ip: false
151+
152+
metadata:
153+
user-data: |
154+
#cloud-config
155+
create_hostname_file: true
156+
write_files:
157+
- path: /etc/apt/apt.conf.d/20auto-upgrades
158+
permissions: '0644'
159+
owner: root
160+
content: |
161+
APT::Periodic::Update-Package-Lists "0";
162+
APT::Periodic::Unattended-Upgrade "0";
163+
use:
164+
- builder-net
165+
- slurm-build-script
166+
167+
- group: cluster-env
168+
modules:
169+
- id: net0
170+
source: modules/network/vpc
171+
settings:
172+
network_name: $(vars.deployment_name)-net-0
173+
subnetworks:
174+
- subnet_name: $(vars.deployment_name)-sub-0
175+
subnet_region: $(vars.region)
176+
subnet_ip: $(vars.net0_range)
177+
firewall_rules:
178+
- name: $(vars.deployment_name)-internal-0
179+
ranges: [$(vars.net0_range)]
180+
allow:
181+
- protocol: tcp
182+
- protocol: udp
183+
- protocol: icmp
184+
185+
# Instructions for Dual NIC on g4-standard-384:
186+
# 1. Uncomment the 'net1' module below.
187+
# 2. Ensure 'vars.net1_range' is also uncommented and defined.
188+
#
189+
# - id: net1 # Added second network
190+
# source: modules/network/vpc
191+
# settings:
192+
# network_name: $(vars.deployment_name)-net-1
193+
# subnetworks:
194+
# - subnet_name: $(vars.deployment_name)-sub-1
195+
# subnet_region: $(vars.region)
196+
# subnet_ip: $(vars.net1_range)
197+
# firewall_rules:
198+
# - name: $(vars.deployment_name)-internal-1
199+
# ranges: [$(vars.net1_range)]
200+
# allow:
201+
# - protocol: tcp
202+
# - protocol: udp
203+
# - protocol: icmp
204+
205+
- id: homefs
206+
source: modules/file-system/filestore
207+
use:
208+
- net0 # Filestore attached to net0
209+
settings:
210+
filestore_tier: BASIC_SSD
211+
size_gb: 2560
212+
local_mount: /home
213+
reserved_ip_range: $(vars.filestore_ip_range)
38214

39215
- group: cluster
40216
modules:
217+
- id: g4_node_startup
218+
source: modules/scripts/startup-script
219+
settings:
220+
runners:
221+
- type: ansible-local
222+
destination: enable_dcgm.yml
223+
content: |
224+
---
225+
- name: Enable NVIDIA DCGM and Persistence on GPU nodes
226+
hosts: all
227+
become: true
228+
vars:
229+
enable_ops_agent: true
230+
enable_nvidia_dcgm: true
231+
tasks:
232+
- name: Update Ops Agent configuration for DCGM
233+
ansible.builtin.blockinfile:
234+
path: /etc/google-cloud-ops-agent/config.yaml
235+
create: yes
236+
insertafter: EOF
237+
block: |
238+
metrics:
239+
receivers:
240+
dcgm:
241+
type: dcgm
242+
receiver_version: 2
243+
service:
244+
pipelines:
245+
dcgm:
246+
receivers:
247+
- dcgm
248+
notify:
249+
- Restart Google Cloud Ops Agent
250+
handlers:
251+
- name: Restart Google Cloud Ops Agent
252+
ansible.builtin.service:
253+
name: google-cloud-ops-agent.service
254+
state: "{{ 'restarted' if enable_ops_agent else 'stopped' }}"
255+
enabled: "{{ enable_ops_agent }}"
256+
post_tasks:
257+
- name: Ensure Google Cloud Ops Agent is running
258+
ansible.builtin.service:
259+
name: google-cloud-ops-agent.service
260+
state: "{{ 'started' if enable_ops_agent else 'stopped' }}"
261+
enabled: "{{ enable_ops_agent }}"
262+
- name: Enable and Start NVIDIA DCGM
263+
ansible.builtin.service:
264+
name: nvidia-dcgm.service
265+
state: "{{ 'started' if enable_nvidia_dcgm else 'stopped' }}"
266+
enabled: "{{ enable_nvidia_dcgm }}"
267+
# Enable persistenced service
268+
- name: Enable and Start nvidia-persistenced
269+
ansible.builtin.service:
270+
name: nvidia-persistenced.service
271+
state: started
272+
enabled: true
41273
42274
- id: g4_nodeset
43275
source: community/modules/compute/schedmd-slurm-gcp-v6-nodeset
44-
use: [network]
276+
# Instructions for Dual NIC on g4-standard-384:
277+
# 3. Add 'net1' to the 'use' list below (e.g., use: [net0, net1, g4_node_startup])
278+
use: [net0, g4_node_startup]
45279
settings:
280+
node_count_static: $(vars.cluster_size)
46281
node_count_dynamic_max: 0
47-
enable_placement: false
48-
node_count_static: 1
49-
bandwidth_tier: gvnic_enabled
50-
machine_type: g4-standard-48
282+
machine_type: $(vars.g4_machine_type)
51283
disk_type: hyperdisk-balanced
52-
instance_image: $(vars.new_image)
284+
disk_size_gb: $(vars.disk_size_gb)
285+
instance_image:
286+
project: $(vars.project_id)
287+
family: $(vars.built_image_family)
53288
instance_image_custom: true
54-
reservation_name: $(vars.g4_reservation_name)
289+
enable_placement: false
290+
291+
# Instructions for Dual NIC on g4-standard-384:
292+
# 4. Uncomment the 'network_interfaces' block below to define the two NICs.
293+
#
294+
# network_interfaces:
295+
# - subnetwork: $(net0.subnetwork_self_link)
296+
# nic_type: GVNIC
297+
# - subnetwork: $(net1.subnetwork_self_link)
298+
# nic_type: GVNIC
55299

56300
- id: g4_partition
57301
source: community/modules/compute/schedmd-slurm-gcp-v6-partition
58-
use: [g4_nodeset]
302+
use:
303+
- g4_nodeset
59304
settings:
60-
is_default: true
61305
partition_name: g4
306+
is_default: true
62307
exclusive: false
63308

64309
- id: slurm_login
65310
source: community/modules/scheduler/schedmd-slurm-gcp-v6-login
66-
use: [network]
311+
use:
312+
- net0
67313
settings:
68-
machine_type: e2-standard-2
69-
enable_login_public_ips: true
70-
instance_image: $(vars.new_image)
314+
machine_type: n2-standard-4
315+
instance_image:
316+
project: $(vars.project_id)
317+
family: $(vars.built_image_family)
71318
instance_image_custom: true
72-
73-
- id: homefs
74-
source: modules/file-system/filestore
75-
use: [network]
76-
settings:
77-
filestore_tier: BASIC_SSD
78-
size_gb: 2560
79-
filestore_share_name: homeshare
80-
local_mount: /home
81-
82-
# - id: private_service_access
83-
# source: community/modules/network/private-service-access
84-
# use: [network]
85-
86-
# To use Managed Lustre as for the shared /home directory:
87-
# 1. Comment out the filestore block above and the `filestore_ip_range` line in the vars block.
88-
# 2. Uncomment the managed-lustre and private-service-access blocks.
89-
# 3. Ensure the instance_image being used has the Lustre client installed.
90-
# - id: homefs
91-
# source: modules/file-system/managed-lustre
92-
# use:
93-
# - network
94-
# - private_service_access
95-
# settings:
96-
# size_gib: 18000
97-
# name: lustre-instance1
98-
# local_mount: /home
99-
# remote_mount: lustrefs
100-
# outputs:
101-
# - network_storage
319+
enable_login_public_ips: true
102320

103321
- id: slurm_controller
104322
source: community/modules/scheduler/schedmd-slurm-gcp-v6-controller
105323
use:
106-
- network
107324
- g4_partition
108-
- slurm_login
325+
- net0
109326
- homefs
327+
- slurm_login
110328
settings:
111-
machine_type: e2-standard-2
112-
enable_controller_public_ips: true
113-
instance_image: $(vars.new_image)
329+
machine_type: n2-standard-4
330+
instance_image:
331+
project: $(vars.project_id)
332+
family: $(vars.built_image_family)
114333
instance_image_custom: true
334+
enable_controller_public_ips: true

0 commit comments

Comments
 (0)