Skip to content

Commit 1f28255

Browse files
ACW101ACW101
authored
Merge pull request GoogleCloudPlatform#4607 from ACW101/blueprint
Add slurm-gke blueprint
2 parents 721983b + 0859ea7 commit 1f28255

29 files changed

Lines changed: 1079 additions & 29 deletions

File tree

‎community/examples/slurm-gke/files/cgroup.conf.tpl‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
# cgroup.conf
2+
# https://slurm.schedmd.com/cgroup.conf.html
3+
4+
CgroupPlugin=autodetect
5+
IgnoreSystemd=yes
6+
# EnableControllers=yes
7+
ConstrainCores=yes
8+
ConstrainRamSpace=yes
9+
ConstrainSwapSpace=no
10+
ConstrainDevices=yes

‎community/examples/slurm-gke/files/slurm-namespace.yaml.tftpl‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
# Copyright 2025 "Google LLC"
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
apiVersion: v1
16+
kind: Namespace
17+
metadata:
18+
name: ${namespace}

‎community/examples/slurm-gke/slurm-gke.yaml‎

Lines changed: 192 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,192 @@
1+
# Copyright 2025 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
---
15+
16+
blueprint_name: slurm-gke
17+
18+
vars:
19+
# The following variables should be over-written in the deployment.yaml file.
20+
# Your GCP Project ID
21+
project_id: ## Set GCP Project ID Here ##
22+
23+
# This should be unique across all of your Cluster
24+
# Toolkit Deployments.
25+
deployment_name: slurmgke
26+
# The GCP Region used for this deployment.
27+
region:
28+
29+
# The GCP Zone used for this deployment.
30+
zone:
31+
32+
# Cidr block containing the IP of the machine calling terraform.
33+
# To allow all (IAM restrictions still enforced), use 0.0.0.0/0
34+
# To allow only your IP address, use <YOUR-IP-ADDRESS>/32
35+
authorized_cidr:
36+
37+
# The number of nodes to be created for the Slurm GKE NodeSet.
38+
gke_nodeset_replicas: 2
39+
40+
41+
# The pre-built Slinky Image for GKE Nodeset.
42+
# Follow instruction in ./images/containers to build this image.
43+
slinky_image: ghcr.io/slinkyproject/slurmd-pyxis:24.11-ubuntu24.04
44+
45+
# Namespace where Slurm GKE NodeSet will be created
46+
slurm_namespace: slurm
47+
48+
deployment_groups:
49+
- group: primary
50+
modules:
51+
52+
###### Common resources ######
53+
54+
- id: network
55+
source: modules/network/vpc
56+
settings:
57+
subnetwork_name: $(vars.deployment_name)-subnet
58+
secondary_ranges_list:
59+
- subnetwork_name: $(vars.deployment_name)-subnet
60+
ranges:
61+
- range_name: pods
62+
ip_cidr_range: 10.4.0.0/14
63+
- range_name: services
64+
ip_cidr_range: 10.0.32.0/20
65+
66+
- id: private_service_access
67+
source: community/modules/network/private-service-access
68+
use: [network]
69+
70+
- id: homefs
71+
source: modules/file-system/filestore
72+
use: [network, private_service_access]
73+
settings:
74+
local_mount: /home
75+
76+
###### GKE Setup ######
77+
78+
- id: gke_service_account
79+
source: community/modules/project/service-account
80+
settings:
81+
name: slinky-gke-sa
82+
project_roles:
83+
- logging.logWriter
84+
- monitoring.metricWriter
85+
- monitoring.viewer
86+
- stackdriver.resourceMetadata.writer
87+
- storage.objectAdmin
88+
- artifactregistry.reader
89+
90+
- id: gke_cluster
91+
source: modules/scheduler/gke-cluster
92+
use: [network, gke_service_account]
93+
settings:
94+
enable_private_endpoint: false
95+
enable_gcsfuse_csi: true
96+
enable_filestore_csi: true
97+
master_authorized_networks:
98+
- cidr_block: $(vars.authorized_cidr) # Allows your machine to run the kubectl command. Required for multi network setup.
99+
display_name: "kubectl-access-network"
100+
system_node_pool_enabled: false
101+
configure_workload_identity_sa: true
102+
enable_dcgm_monitoring: true
103+
outputs: [instructions]
104+
105+
- id: gke_base_pool
106+
source: modules/compute/gke-node-pool
107+
use: [gke_cluster, gke_service_account]
108+
settings:
109+
initial_node_count: 1
110+
disk_type: pd-balanced
111+
machine_type: e2-standard-4
112+
zones: [$(vars.zone)]
113+
114+
- id: gke_compute_pool
115+
source: modules/compute/gke-node-pool
116+
use: [gke_cluster, gke_service_account]
117+
settings:
118+
name: gke-compute-pool
119+
initial_node_count: $(vars.gke_nodeset_replicas)
120+
disk_type: pd-balanced
121+
machine_type: c2-standard-16
122+
zones: [$(vars.zone)]
123+
124+
- id: gke_ns_manifest
125+
source: modules/management/kubectl-apply
126+
use: [gke_cluster]
127+
settings:
128+
apply_manifests:
129+
- source: $(ghpc_stage("./files/slurm-namespace.yaml.tftpl"))
130+
template_vars:
131+
namespace: $(vars.slurm_namespace)
132+
133+
- id: slinky
134+
source: community/modules/scheduler/slinky
135+
use:
136+
- gke_cluster
137+
- gke_base_pool # Optionally specify nodepool(s) to avoid operator components running on HPC hardware
138+
settings:
139+
slurm_operator_namespace: $(vars.slurm_namespace)
140+
install_slurm_operator_chart: true
141+
install_slurm_chart: false
142+
143+
- id: gke_compute_nodeset
144+
source: community/modules/compute/gke-nodeset
145+
use: [gke_compute_pool, slinky, homefs, slurm_controller, network]
146+
settings:
147+
slurm_cluster_name: $(vars.deployment_name)
148+
image: $(vars.slinky_image)
149+
150+
- id: gke_compute_partition
151+
source: community/modules/compute/gke-partition
152+
use: [slurm_controller, gke_compute_nodeset]
153+
154+
###### GCE Setup ######
155+
156+
- id: debug_nodeset
157+
source: community/modules/compute/schedmd-slurm-gcp-v6-nodeset
158+
use: [network]
159+
settings:
160+
node_count_dynamic_max: 4
161+
machine_type: n2-standard-2
162+
allow_automatic_updates: false
163+
164+
- id: debug_partition
165+
source: community/modules/compute/schedmd-slurm-gcp-v6-partition
166+
use:
167+
- debug_nodeset
168+
settings:
169+
partition_name: debug
170+
exclusive: false # allows nodes to stay up after jobs are done
171+
is_default: true
172+
suspend_time: -1 # prevents nodes from suspending while it's idle
173+
174+
- id: slurm_login
175+
source: community/modules/scheduler/schedmd-slurm-gcp-v6-login
176+
use: [network]
177+
settings:
178+
machine_type: n2-standard-4
179+
enable_login_public_ips: true
180+
181+
- id: slurm_controller
182+
source: community/modules/scheduler/schedmd-slurm-gcp-v6-controller
183+
use:
184+
- network
185+
- slurm_login
186+
- debug_partition
187+
- homefs
188+
settings:
189+
slurm_cluster_name: $(vars.deployment_name)
190+
enable_slurm_auth: true
191+
cgroup_conf_tpl: $(ghpc_stage("./files/cgroup.conf.tpl"))
192+
enable_controller_public_ips: true

‎community/modules/compute/gke-nodeset/README.md‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
<!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
2+
## Requirements
3+
4+
| Name | Version |
5+
|------|---------|
6+
| <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >= 1.3 |
7+
| <a name="requirement_google"></a> [google](#requirement\_google) | >= 4.84 |
8+
9+
## Providers
10+
11+
| Name | Version |
12+
|------|---------|
13+
| <a name="provider_google"></a> [google](#provider\_google) | >= 4.84 |
14+
15+
## Modules
16+
17+
| Name | Source | Version |
18+
|------|--------|---------|
19+
| <a name="module_home_pv"></a> [home\_pv](#module\_home\_pv) | ../../../../modules/file-system/gke-persistent-volume | n/a |
20+
| <a name="module_kubectl_apply"></a> [kubectl\_apply](#module\_kubectl\_apply) | ../../../../modules/management/kubectl-apply | n/a |
21+
| <a name="module_slurm_key_pv"></a> [slurm\_key\_pv](#module\_slurm\_key\_pv) | ../../../../modules/file-system/gke-persistent-volume | n/a |
22+
23+
## Resources
24+
25+
| Name | Type |
26+
|------|------|
27+
| [google_storage_bucket_object.gke_nodeset_config](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource |
28+
| [google_storage_bucket.this](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/storage_bucket) | data source |
29+
30+
## Inputs
31+
32+
| Name | Description | Type | Default | Required |
33+
|------|-------------|------|---------|:--------:|
34+
| <a name="input_cluster_id"></a> [cluster\_id](#input\_cluster\_id) | projects/{{project}}/locations/{{location}}/clusters/{{cluster}} | `string` | n/a | yes |
35+
| <a name="input_filestore_id"></a> [filestore\_id](#input\_filestore\_id) | An array of identifier for a filestore with the format `projects/{{project}}/locations/{{location}}/instances/{{name}}`. | `list(string)` | n/a | yes |
36+
| <a name="input_image"></a> [image](#input\_image) | The image for slurm daemon | `string` | n/a | yes |
37+
| <a name="input_instance_templates"></a> [instance\_templates](#input\_instance\_templates) | The URLs of Instance Templates | `list(string)` | n/a | yes |
38+
| <a name="input_network_storage"></a> [network\_storage](#input\_network\_storage) | An array of network attached storage mounts to be configured on nodes. | <pre>list(object({<br/> server_ip = string,<br/> remote_mount = string,<br/> local_mount = string,<br/> fs_type = string,<br/> mount_options = string,<br/> client_install_runner = map(string)<br/> mount_runner = map(string)<br/> }))</pre> | n/a | yes |
39+
| <a name="input_node_count_static"></a> [node\_count\_static](#input\_node\_count\_static) | The number of static nodes in node-pool | `number` | n/a | yes |
40+
| <a name="input_node_pool_names"></a> [node\_pool\_names](#input\_node\_pool\_names) | If set to true. The node group VMs will have a random public IP assigned to it. Ignored if access\_config is set. | `list(string)` | n/a | yes |
41+
| <a name="input_nodeset_name"></a> [nodeset\_name](#input\_nodeset\_name) | The nodeset name | `string` | `"gkenodeset"` | no |
42+
| <a name="input_project_id"></a> [project\_id](#input\_project\_id) | The project ID to host the cluster in. | `string` | n/a | yes |
43+
| <a name="input_slurm_bucket"></a> [slurm\_bucket](#input\_slurm\_bucket) | GCS Bucket of Slurm cluster file storage. | `any` | n/a | yes |
44+
| <a name="input_slurm_bucket_dir"></a> [slurm\_bucket\_dir](#input\_slurm\_bucket\_dir) | Path directory within `bucket_name` for Slurm cluster file storage. | `string` | n/a | yes |
45+
| <a name="input_slurm_cluster_name"></a> [slurm\_cluster\_name](#input\_slurm\_cluster\_name) | Cluster name, used in slurm controller | `string` | n/a | yes |
46+
| <a name="input_slurm_controller_instance"></a> [slurm\_controller\_instance](#input\_slurm\_controller\_instance) | Slurm cluster controller instance | `any` | n/a | yes |
47+
| <a name="input_slurm_namespace"></a> [slurm\_namespace](#input\_slurm\_namespace) | slurm namespace for charts | `string` | `"slurm"` | no |
48+
| <a name="input_subnetwork"></a> [subnetwork](#input\_subnetwork) | Primary subnetwork object | `any` | n/a | yes |
49+
50+
## Outputs
51+
52+
| Name | Description |
53+
|------|-------------|
54+
| <a name="output_nodeset_name"></a> [nodeset\_name](#output\_nodeset\_name) | Name of the new Slinky nodset |
55+
<!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->

‎community/modules/compute/gke-nodeset/main.tf‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
# Copyright 2025 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
16+
### GKE NodeSet
17+
locals {
18+
manifest_path = "${path.module}/templates/nodeset-general.yaml.tftpl"
19+
}
20+
21+
module "kubectl_apply" {
22+
source = "../../../../modules/management/kubectl-apply"
23+
24+
cluster_id = var.cluster_id
25+
project_id = var.project_id
26+
27+
apply_manifests = [{
28+
source = local.manifest_path,
29+
template_vars = {
30+
slurm_namespace = var.slurm_namespace,
31+
nodeset_name = "${var.slurm_cluster_name}-${var.nodeset_name}",
32+
nodeset_cr_name = "${var.slurm_cluster_name}-${var.nodeset_name}",
33+
controller_name = "${var.slurm_cluster_name}-controller",
34+
node_pool_name = var.node_pool_names[0],
35+
node_count = var.node_count_static,
36+
image = var.image,
37+
home_pvc = module.home_pv.pvc_name
38+
slurm_key_pvc = module.slurm_key_pv.pvc_name
39+
}
40+
}]
41+
}
42+
43+
data "google_storage_bucket" "this" {
44+
name = var.slurm_bucket[0].name
45+
46+
depends_on = [var.slurm_bucket]
47+
}
48+
49+
### Slurm NodeSet
50+
locals {
51+
nodeset = {
52+
gke_nodepool = var.node_pool_names[0]
53+
nodeset_name = var.nodeset_name
54+
node_count_static = var.node_count_static
55+
subnetwork = "https://www.googleapis.com/compute/v1/projects/${var.project_id}/regions/${var.subnetwork.region}/subnetworks/${var.subnetwork.name}"
56+
instance_template = var.instance_templates[0]
57+
}
58+
}
59+
60+
resource "google_storage_bucket_object" "gke_nodeset_config" {
61+
bucket = data.google_storage_bucket.this.name
62+
name = "${var.slurm_bucket_dir}/nodeset_configs/${var.nodeset_name}.yaml"
63+
content = yamlencode(local.nodeset)
64+
}

‎community/modules/compute/gke-nodeset/metadata.yaml‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
# Copyright 2025 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
---
15+
16+
spec:
17+
requirements:
18+
services:
19+
- container.googleapis.com
20+
- storage.googleapis.com

‎community/modules/compute/gke-nodeset/output.tf‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
# Copyright 2025 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
output "nodeset_name" {
16+
description = "Name of the new Slinky nodset"
17+
value = local.nodeset.nodeset_name
18+
}

0 commit comments

Comments
 (0)