Skip to content

Commit 3eaf74e

Browse files
j0hnLj0hnL
committed
feat: add separate compute RKE2 cluster support
Add compute_rke2 as an independent Kubernetes cluster type that can coexist with service_k8s (kubeadm), service_rke2, and slurm clusters. Configuration: - software_config.json: add compute_rke2 subgroups (compute_kube_*) - omnia_config.yml: add compute_rke2_k8s_cluster block (commented) - omnia_config.json: add compute_rke2_k8s_cluster schema - config.py: add compute_kube_* to FUNCTIONAL_GROUP_LAYER_MAP Packages: - compute_rke2.json: RKE2 core packages without service-layer images Validation: - common_validation.py: x86_64 arch check, subgroup requirement, NFS matching, cluster_set deployment validation for compute_rke2 - validate_software_config_json.yml: detect compute_rke2_support - include_software_config.yml: set compute_rke2_support fact Pipeline: - provision.yml: add compute_rke2 to omnia_run_tags - image_package_collector.py: add compute_kube_* functional groups Cloud-init: - 3 new templates: control_plane_first, control_plane, node - configure_cloud_init_group.yml: route compute_kube_* to rke2/ NFS: - create_compute_rke2_config_nfs.yml: compute cluster NFS setup - k8s_config/tasks/main.yml: include compute RKE2 NFS task Signed-off-by: John Lockman <jlockman3@gmail.com>
1 parent 174c1cb commit 3eaf74e

18 files changed

Lines changed: 1626 additions & 16 deletions

File tree

‎common/library/module_utils/input_validation/common_utils/config.py‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@
3838
"x86_64": [
3939
"slurm_control_node", "slurm_node", "login_node", "login_compiler_node",
4040
"service_kube_control_plane", "service_kube_control_plane_first", "service_kube_node",
41+
"compute_kube_control_plane", "compute_kube_control_plane_first", "compute_kube_node",
4142
"os"
4243
],
4344
"aarch64": [
@@ -96,6 +97,11 @@
9697
files["storage_config"],
9798
files["high_availability_config"],
9899
],
100+
"compute_rke2": [
101+
files["omnia_config"],
102+
files["storage_config"],
103+
files["high_availability_config"],
104+
],
99105
"storage": [files["storage_config"]],
100106
"prepare_oim": [
101107
files["network_spec"],
@@ -191,6 +197,9 @@
191197
"service_kube_control_plane_first_x86_64": "management",
192198
"service_kube_control_plane_x86_64": "management",
193199
"service_kube_node_x86_64": "management",
200+
"compute_kube_control_plane_first_x86_64": "compute",
201+
"compute_kube_control_plane_x86_64": "compute",
202+
"compute_kube_node_x86_64": "compute",
194203
"login_node_x86_64": "management",
195204
"login_node_aarch64": "management",
196205
"login_compiler_node_x86_64": "management",

‎common/library/module_utils/input_validation/schema/omnia_config.json‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -249,6 +249,53 @@
249249
}
250250
]
251251
}
252+
},
253+
"compute_rke2_k8s_cluster": {
254+
"type": "array",
255+
"description": "List of RKE2-based compute Kubernetes cluster configurations.",
256+
"items": {
257+
"type": "object",
258+
"properties": {
259+
"cluster_name": {
260+
"type": "string",
261+
"minLength": 1,
262+
"description": "Unique name for the RKE2 compute cluster."
263+
},
264+
"deployment": {
265+
"type": "boolean"
266+
},
267+
"k8s_cni": {
268+
"enum": ["calico", "canal", "cilium", "flannel"],
269+
"description": "K8s CNI plugin for the compute RKE2 cluster."
270+
},
271+
"pod_external_ip_range": {
272+
"description": "IP range for LoadBalancer external IPs.",
273+
"type": "string",
274+
"pattern": "^(?:(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)-(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?))|(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)/[0-9]{1,2}$|^$"
275+
},
276+
"k8s_service_addresses": {
277+
"type": "string",
278+
"pattern": "^(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)/[0-9]{1,2}$",
279+
"description": "CIDR for K8s service IPs."
280+
},
281+
"k8s_pod_network_cidr": {
282+
"type": "string",
283+
"pattern": "^(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)/[0-9]{1,2}$",
284+
"description": "CIDR for K8s pod network."
285+
},
286+
"nfs_storage_name": {
287+
"type": "string",
288+
"description": "The NFS client server name mentioned in storage_config.yml"
289+
}
290+
},
291+
"required": [
292+
"cluster_name",
293+
"k8s_cni",
294+
"k8s_service_addresses",
295+
"k8s_pod_network_cidr",
296+
"pod_external_ip_range"
297+
]
298+
}
252299
}
253300
},
254301
"required": [

‎common/library/module_utils/input_validation/validation_flows/common_validation.py‎

Lines changed: 35 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -225,20 +225,21 @@ def validate_software_config(
225225
)
226226

227227
# RKE2 packages are only available for x86_64; reject any other arch explicitly
228-
if "service_rke2" in software_names:
229-
rke2_entry = next(
230-
(s for s in data.get("softwares", []) if s.get("name") == "service_rke2"), {}
231-
)
232-
rke2_arch = rke2_entry.get("arch", ["x86_64"])
233-
unsupported_archs = [a for a in rke2_arch if a != "x86_64"]
234-
if unsupported_archs:
235-
errors.append(
236-
create_error_msg(
237-
"Validation Error: ",
238-
"service_rke2",
239-
f"is only supported on x86_64. Unsupported arch(es): {', '.join(unsupported_archs)}."
240-
)
228+
for rke2_sw_name in ["service_rke2", "compute_rke2"]:
229+
if rke2_sw_name in software_names:
230+
rke2_entry = next(
231+
(s for s in data.get("softwares", []) if s.get("name") == rke2_sw_name), {}
241232
)
233+
rke2_arch = rke2_entry.get("arch", ["x86_64"])
234+
unsupported_archs = [a for a in rke2_arch if a != "x86_64"]
235+
if unsupported_archs:
236+
errors.append(
237+
create_error_msg(
238+
"Validation Error: ",
239+
rke2_sw_name,
240+
f"is only supported on x86_64. Unsupported arch(es): {', '.join(unsupported_archs)}."
241+
)
242+
)
242243

243244
# Ensure ldms is not configured without service_k8s or service_rke2 in softwares
244245
if "ldms" in software_names and "service_k8s" not in software_names and "service_rke2" not in software_names:
@@ -260,7 +261,7 @@ def validate_software_config(
260261
)
261262

262263
# Check for required subgroups when specific software names are present
263-
software_requiring_subgroups = ["additional_packages", "slurm_custom", "service_k8s", "service_rke2"]
264+
software_requiring_subgroups = ["additional_packages", "slurm_custom", "service_k8s", "service_rke2", "compute_rke2"]
264265
for software_name in software_requiring_subgroups:
265266
if software_name in software_names:
266267
if software_name not in data or not data[software_name]:
@@ -579,6 +580,22 @@ def get_matching_clusters_for_nfs(nfs_name, omnia_config):
579580
):
580581
matching_clusters["service_k8s_cluster"] = svc
581582

583+
# Service RKE2
584+
for svc in omnia_config.get("service_rke2_k8s_cluster", []):
585+
if (
586+
svc.get("nfs_storage_name") == nfs_name
587+
and svc.get("deployment") is True
588+
):
589+
matching_clusters["service_rke2_k8s_cluster"] = svc
590+
591+
# Compute RKE2
592+
for compute in omnia_config.get("compute_rke2_k8s_cluster", []):
593+
if (
594+
compute.get("nfs_storage_name") == nfs_name
595+
and compute.get("deployment") is True
596+
):
597+
matching_clusters["compute_rke2_k8s_cluster"] = compute
598+
582599
# Slurm
583600
for slurm in omnia_config.get("slurm_cluster", []):
584601
if slurm.get("nfs_storage_name") == nfs_name:
@@ -1493,6 +1510,10 @@ def validate_k8s(data, admin_networks, softwares, ha_config, tag_names, errors,
14931510
cluster_set["service_rke2_k8s_cluster"] = data.get(
14941511
"service_rke2_k8s_cluster", [])
14951512

1513+
if "compute_rke2" in softwares and "compute_rke2" in tag_names:
1514+
cluster_set["compute_rke2_k8s_cluster"] = data.get(
1515+
"compute_rke2_k8s_cluster", [])
1516+
14961517
for k8s_cluster_type, k8s_clusters in cluster_set.items():
14971518
deployments_list = [k.get('deployment', False) for k in k8s_clusters]
14981519
true_count = deployments_list.count(True)

‎common/library/modules/image_package_collector.py‎

Lines changed: 25 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -109,7 +109,11 @@ def collect_packages_from_json(sw_data, fg_name=None,
109109
elif service_k8s_defined:
110110
fg_name = fg_name.replace("_aarch64", "").replace("_x86_64", "")
111111

112-
k8s_top_key = "service_rke2" if "service_rke2" in sw_data else "service_k8s"
112+
k8s_top_key = (
113+
"compute_rke2" if "compute_rke2" in sw_data
114+
else "service_rke2" if "service_rke2" in sw_data
115+
else "service_k8s"
116+
)
113117

114118
if k8s_top_key in sw_data and "cluster" in sw_data[k8s_top_key]:
115119
for entry in sw_data[k8s_top_key]["cluster"]:
@@ -159,6 +163,7 @@ def process_functional_group(fg_name, arch, os_version, input_project_dir,
159163
# Remove version suffix for versioned files (e.g., service_k8s_v1.35.1 -> service_k8s)
160164
if sw_name.startswith("service_k8s_v"):
161165
sw_name = "service_k8s"
166+
# compute_rke2.json -> compute_rke2 (already handled by replace above)
162167
if sw_name not in allowed_softwares:
163168
continue
164169

@@ -177,7 +182,7 @@ def process_functional_group(fg_name, arch, os_version, input_project_dir,
177182
sw_data, fg_name=fg_name, slurm_defined=True
178183
)
179184
)
180-
elif json_file.startswith("service_k8s_v") or json_file == "service_rke2.json":
185+
elif json_file.startswith("service_k8s_v") or json_file == "service_rke2.json" or json_file == "compute_rke2.json":
181186
# Handle versioned service_k8s_v<version>.json or service_rke2.json
182187
packages.extend(
183188
collect_packages_from_json(
@@ -253,8 +258,16 @@ def run_module():
253258
"service_kube_control_plane_x86_64"
254259
}
255260

261+
# Compute RKE2 functional groups
262+
compute_k8s_functional_groups = {
263+
"compute_kube_node_x86_64",
264+
"compute_kube_control_plane_first_x86_64",
265+
"compute_kube_control_plane_x86_64"
266+
}
267+
256268
# Check if any k8s functional groups are being processed
257269
needs_service_k8s = any(fg in k8s_functional_groups for fg in functional_groups)
270+
needs_compute_rke2 = any(fg in compute_k8s_functional_groups for fg in functional_groups)
258271

259272
# Only validate service_k8s version if k8s functional groups are present
260273
service_k8s_json = None
@@ -272,6 +285,9 @@ def run_module():
272285
else:
273286
k8s_json = None
274287

288+
# Determine compute RKE2 package JSON
289+
compute_rke2_json = "compute_rke2.json" if "compute_rke2" in allowed_softwares else None
290+
275291
# Functional group → json files mapping
276292
software_map = {
277293
"os_x86_64": ["default_packages.json", "ldms.json"],
@@ -297,6 +313,13 @@ def run_module():
297313
"service_kube_control_plane_x86_64": [k8s_json],
298314
})
299315

316+
if compute_rke2_json:
317+
software_map.update({
318+
"compute_kube_node_x86_64": [compute_rke2_json],
319+
"compute_kube_control_plane_first_x86_64": [compute_rke2_json],
320+
"compute_kube_control_plane_x86_64": [compute_rke2_json],
321+
})
322+
300323
compute_images_dict = {}
301324

302325
for fg_name in functional_groups:

‎input/config/x86_64/rhel/10.0/compute_rke2.json‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
{
2+
"compute_rke2": {
3+
"cluster": [
4+
{ "package": "docker.io/library/busybox", "type": "image", "tag": "1.36" },
5+
{ "package": "firewalld", "type": "rpm", "repo_name": "x86_64_baseos" },
6+
{ "package": "python3-firewall", "type": "rpm", "repo_name": "x86_64_baseos" },
7+
{ "package": "git", "type": "rpm", "repo_name": "x86_64_appstream"},
8+
{ "package": "vim", "type": "rpm", "repo_name": "x86_64_appstream"},
9+
{ "package": "rke2-server", "type": "rpm", "repo_name": "rke2" },
10+
{ "package": "rke2-agent", "type": "rpm", "repo_name": "rke2" },
11+
{ "package": "rke2-common", "type": "rpm", "repo_name": "rke2" },
12+
{ "package": "rke2-selinux", "type": "rpm", "repo_name": "rke2-common" },
13+
{ "package": "rke2-images-core.linux-amd64", "type": "tarball", "url": "https://github.com/rancher/rke2/releases/download/v1.35.1+rke2r1/rke2-images-core.linux-amd64.tar.zst" },
14+
{ "package": "rke2-images-calico.linux-amd64", "type": "tarball", "url": "https://github.com/rancher/rke2/releases/download/v1.35.1+rke2r1/rke2-images-calico.linux-amd64.tar.zst" },
15+
{ "package": "rke2-images-canal.linux-amd64", "type": "tarball", "url": "https://github.com/rancher/rke2/releases/download/v1.35.1+rke2r1/rke2-images-canal.linux-amd64.tar.zst" },
16+
{ "package": "rke2-images-cilium.linux-amd64", "type": "tarball", "url": "https://github.com/rancher/rke2/releases/download/v1.35.1+rke2r1/rke2-images-cilium.linux-amd64.tar.zst" },
17+
{ "package": "rke2-images-flannel.linux-amd64", "type": "tarball", "url": "https://github.com/rancher/rke2/releases/download/v1.35.1+rke2r1/rke2-images-flannel.linux-amd64.tar.zst" },
18+
{ "package": "apptainer", "type": "rpm", "repo_name": "epel" },
19+
{ "package": "doca-ofed", "type": "iso", "url": "https://www.mellanox.com/downloads/DOCA/DOCA_v3.2.1/host/doca-host-3.2.1-044000_25.10_rhel10.x86_64.rpm"}
20+
]
21+
},
22+
"compute_kube_control_plane": {
23+
"cluster": [
24+
{ "package": "ghcr.io/kube-vip/kube-vip", "tag": "v0.8.9", "type": "image" },
25+
{ "package": "quay.io/metallb/speaker", "tag": "v0.15.2", "type": "image" },
26+
{ "package": "prettytable==3.14.0", "type": "pip_module" },
27+
{ "package": "python3.12", "type": "rpm", "repo_name": "x86_64_appstream" },
28+
{ "package": "git", "type": "rpm", "repo_name": "x86_64_appstream"},
29+
{ "package": "kubernetes==33.1.0", "type": "pip_module" }
30+
]
31+
},
32+
"compute_kube_control_plane_first": {
33+
"cluster": [
34+
{ "package": "ghcr.io/kube-vip/kube-vip", "tag": "v0.8.9", "type": "image" },
35+
{ "package": "quay.io/metallb/speaker", "tag": "v0.15.2", "type": "image" },
36+
{
37+
"package": "metallb-native-v0.15.2",
38+
"type": "manifest",
39+
"url": "https://raw.githubusercontent.com/metallb/metallb/v0.15.2/config/manifests/metallb-native.yaml"
40+
},
41+
{ "package": "helm-v3.19.0-amd64", "type": "tarball", "url": "https://get.helm.sh/helm-v3.19.0-linux-amd64.tar.gz" },
42+
{ "package": "nfs-subdir-external-provisioner-4.0.18", "type": "tarball", "url": "https://github.com/kubernetes-sigs/nfs-subdir-external-provisioner/releases/download/nfs-subdir-external-provisioner-4.0.18/nfs-subdir-external-provisioner-4.0.18.tgz" },
43+
{ "package": "prettytable==3.14.0", "type": "pip_module" },
44+
{ "package": "python3.12", "type": "rpm", "repo_name": "x86_64_appstream" },
45+
{ "package": "git", "type": "rpm", "repo_name": "x86_64_appstream"},
46+
{ "package": "kubernetes==33.1.0", "type": "pip_module" }
47+
]
48+
},
49+
"compute_kube_node": {
50+
"cluster": [
51+
{ "package": "registry.k8s.io/sig-storage/nfs-subdir-external-provisioner", "tag": "v4.0.2", "type": "image" },
52+
{ "package": "quay.io/metallb/speaker", "tag": "v0.15.2", "type": "image" },
53+
{ "package": "quay.io/metallb/controller", "tag": "v0.15.2", "type": "image" }
54+
]
55+
}
56+
}

‎input/omnia_config.yml‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -220,3 +220,21 @@ service_k8s_cluster:
220220
# nfs_storage_name: "nfs_k8s"
221221
# csi_powerscale_driver_secret_file_path: ""
222222
# csi_powerscale_driver_values_file_path: ""
223+
224+
# ----------------------------COMPUTE RKE2 K8S-------------------------------------------------
225+
# To deploy a separate RKE2-based compute Kubernetes cluster, configure this block
226+
# and add "compute_rke2" to software_config.json softwares list.
227+
# The compute RKE2 cluster is independent of the service cluster and can coexist
228+
# with service_k8s (kubeadm), service_rke2, and slurm clusters.
229+
# k8s_cni supports "calico", "canal", "cilium", and "flannel".
230+
# Please ensure a corresponding cluster entry is added to high_availability_config.yml.
231+
# CIDRs must not overlap with the service cluster CIDRs.
232+
233+
# compute_rke2_k8s_cluster:
234+
# - cluster_name: compute_cluster
235+
# deployment: false
236+
# k8s_cni: "calico"
237+
# pod_external_ip_range: ""
238+
# k8s_service_addresses: "10.234.0.0/18"
239+
# k8s_pod_network_cidr: "10.234.64.0/18"
240+
# nfs_storage_name: "nfs_compute_k8s"

‎input/software_config.json‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,10 +23,18 @@
2323
{"name": "service_kube_control_plane"},
2424
{"name": "service_kube_node"}
2525
],
26+
"compute_rke2": [
27+
{"name": "compute_kube_control_plane_first"},
28+
{"name": "compute_kube_control_plane"},
29+
{"name": "compute_kube_node"}
30+
],
2631
"additional_packages":[
2732
{"name": "service_kube_control_plane_first"},
2833
{"name": "service_kube_control_plane"},
2934
{"name": "service_kube_node"},
35+
{"name": "compute_kube_control_plane_first"},
36+
{"name": "compute_kube_control_plane"},
37+
{"name": "compute_kube_node"},
3038
{"name": "slurm_control_node"},
3139
{"name": "slurm_node"},
3240
{"name": "login_node"},

‎local_repo/roles/validation/tasks/validate_software_config_json.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@
3232
ansible.builtin.set_fact:
3333
service_k8s_support: "{{ software_config.softwares | selectattr('name', 'equalto', 'service_k8s') | list | length > 0 }}"
3434
service_rke2_support: "{{ software_config.softwares | selectattr('name', 'equalto', 'service_rke2') | list | length > 0 }}"
35+
compute_rke2_support: "{{ software_config.softwares | selectattr('name', 'equalto', 'compute_rke2') | list | length > 0 }}"
3536
software_names: "{{ software_config.softwares | map(attribute='name') | select('defined') | list }}"
3637
software_json_list: "{{ software_names | map('regex_replace', '$', '.json') | list }}"
3738

‎provision/provision.yml‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,13 @@
7373
lookup('file', hostvars['localhost']['input_project_dir'] ~ '/software_config.json')
7474
| from_json).softwares | map(attribute='name') | list )
7575
else []
76+
) +
77+
(
78+
['compute_rke2']
79+
if 'compute_rke2' in (
80+
lookup('file', hostvars['localhost']['input_project_dir'] ~ '/software_config.json')
81+
| from_json).softwares | map(attribute='name') | list
82+
else []
7683
)
7784
) | unique
7885
}}

‎provision/roles/configure_ochami/tasks/configure_cloud_init_group.yml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,8 @@
3232
{%- if k8s_distro | default('kubeadm') == 'rke2' and
3333
'service_kube' in functional_group_name -%}
3434
cloud_init/rke2/ci-group-{{ functional_group_name }}.yaml.j2
35+
{%- elif 'compute_kube' in functional_group_name -%}
36+
cloud_init/rke2/ci-group-{{ functional_group_name }}.yaml.j2
3537
{%- else -%}
3638
cloud_init/ci-group-{{ functional_group_name }}.yaml.j2
3739
{%- endif -%}

0 commit comments

Comments
 (0)