feat: Add RKE2 as alternative Kubernetes distribution for service cluster

Add support for deploying RKE2 (Rancher Kubernetes Engine 2) as an alternative
to vanilla Kubernetes (kubeadm) for the service cluster. Users can choose the
distribution via the new k8s_distro field in omnia_config.yml.

Configuration changes:
- Add k8s_distro field to service_k8s_cluster in omnia_config.yml (default: kubeadm)
- Update omnia_config.json schema to validate k8s_distro enum (kubeadm/rke2)
- Expand k8s_cni enum to include canal, cilium (RKE2-supported CNIs)

Pipeline integration:
- discovery.yml: Enable service_k8s tag when service_rke2 is in software_config.json
- include_software_config.yml: Detect service_rke2 and set service_rke2_support fact
- validate_software_config_json.yml: Handle service_rke2 arch and version detection
- common_validation.py: Accept service_rke2 for cluster validation
- image_package_collector.py: Dynamically select service_rke2.json for image builds
- k8s_config/main.yml: Branch NFS setup based on k8s_distro
- configure_cloud_init_group.yml: Select RKE2-specific templates when k8s_distro=rke2

New files:
- 3 RKE2 cloud-init templates (first server, additional server, agent)
- create_rke2_config_nfs.yml for RKE2-specific NFS directory setup
- service_rke2.json package definitions for RHEL 10.0 x86_64

Key differences from kubeadm path:
- RKE2 uses built-in containerd (no CRI-O)
- RKE2 manages CNI lifecycle (calico/canal/cilium/flannel)
- Token-based cluster join instead of kubeadm certificates
- kube-vip deployed as RKE2 static pod manifest
- RKE2 registries.yaml for Pulp mirror integration
- Port 9345 for RKE2 supervisor API

Existing kubeadm deployment is completely unaffected when k8s_distro=kubeadm (default).

Signed-off-by: John Lockman <j.lockman@dell.com>

Author: j0hnL

common/library/module_utils/input_validation/schema/omnia_config.json

@@ -176,6 +176,77 @@
         }
       ]
       }
+    },
+    "service_rke2_k8s_cluster": {
+      "type": "array",
+      "description": "List of RKE2 Kubernetes cluster configurations.",
+      "items": {
+        "type": "object",
+        "properties": {
+          "cluster_name": {
+            "type": "string",
+            "minLength": 1,
+            "description": "Unique name for the RKE2 service cluster."
+          },
+          "deployment": {
+            "type": "boolean"
+          },
+          "k8s_cni": {
+            "enum": ["calico", "canal", "cilium", "flannel"],
+            "description": "K8s CNI plugin for the RKE2 cluster."
+          },
+          "pod_external_ip_range": {
+            "description": "IP range for LoadBalancer external IPs.",
+            "type": "string",
+            "pattern": "^(?:(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)-(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?))|(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)/[0-9]{1,2}$|^$"
+          },
+          "k8s_service_addresses": {
+            "type": "string",
+            "pattern": "^(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)/[0-9]{1,2}$",
+            "description": "CIDR for K8s service IPs."
+          },
+          "k8s_pod_network_cidr": {
+            "type": "string",
+            "pattern": "^(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)/[0-9]{1,2}$",
+            "description": "CIDR for K8s pod network."
+          },
+          "nfs_storage_name": {
+            "type": "string",
+            "description": "The NFS client server name mentioned in storage_config.yml"
+          },
+          "csi_powerscale_driver_secret_file_path": {
+            "description": "Absolute file path for the secret.yaml file.",
+            "type": "string",
+            "pattern": "^(|/?([a-zA-Z0-9._-]+/)*[a-zA-Z0-9._-]+\\.yaml)$"
+          },
+          "csi_powerscale_driver_values_file_path": {
+            "description": "File path for the values.yaml file.",
+            "type": "string",
+            "pattern": "^(|/?([a-zA-Z0-9._-]+/)*[a-zA-Z0-9._-]+\\.yaml)$"
+          }
+        },
+        "required": [
+          "cluster_name",
+          "k8s_cni",
+          "k8s_service_addresses"
+        ],
+        "allOf": [
+        {
+          "if": {
+            "properties": {
+              "csi_powerscale_driver_secret_file_path": {
+                "type": "string",
+                "minLength": 1
+              }
+            },
+            "required": ["csi_powerscale_driver_secret_file_path"]
+          },
+          "then": {
+            "required": ["csi_powerscale_driver_values_file_path"]
+          }
+        }
+      ]
+      }
     }
 },
   "required": [

common/library/module_utils/input_validation/validation_flows/common_validation.py

@@ -1463,6 +1463,10 @@ def validate_k8s(data, admin_networks, softwares, ha_config, tag_names, errors,
         cluster_set["service_k8s_cluster"] = data.get(
             "service_k8s_cluster", [])
 
+    if "service_rke2" in softwares and "service_k8s" in tag_names:
+        cluster_set["service_rke2_k8s_cluster"] = data.get(
+            "service_rke2_k8s_cluster", [])
+
     for k8s_cluster_type, k8s_clusters in cluster_set.items():
         deployments_list = [k.get('deployment', False) for k in k8s_clusters]
         true_count = deployments_list.count(True)

common/library/modules/image_package_collector.py

@@ -109,8 +109,11 @@ def collect_packages_from_json(sw_data, fg_name=None,
     elif service_k8s_defined:
         fg_name = fg_name.replace("_aarch64", "").replace("_x86_64", "")
 
-        if "service_k8s" in sw_data and "cluster" in sw_data["service_k8s"]:
-            for entry in sw_data["service_k8s"]["cluster"]:
+        # Determine the top-level key (service_k8s or service_rke2)
+        k8s_top_key = "service_rke2" if "service_rke2" in sw_data else "service_k8s"
+
+        if k8s_top_key in sw_data and "cluster" in sw_data[k8s_top_key]:
+            for entry in sw_data[k8s_top_key]["cluster"]:
                 if entry.get("type") == "rpm" and "package" in entry:
                     packages.append(entry["package"])
 
@@ -175,8 +178,8 @@ def process_functional_group(fg_name, arch, os_version, input_project_dir,
                     sw_data, fg_name=fg_name, slurm_defined=True
                 )
             )
-        elif json_file.startswith("service_k8s_v"):
-            # Handle versioned service_k8s_v<version>.json files
+        elif json_file.startswith("service_k8s_v") or json_file == "service_rke2.json":
+            # Handle versioned service_k8s_v<version>.json or service_rke2.json
             packages.extend(
                 collect_packages_from_json(
                     sw_data, fg_name=fg_name, service_k8s_defined=True
@@ -256,12 +259,20 @@ def run_module():
 
     # Only validate service_k8s version if k8s functional groups are present
     service_k8s_json = None
-    if needs_service_k8s:
+    if needs_service_k8s and "service_rke2" not in allowed_softwares:
         if not service_k8s_version:
             module.fail_json(msg="service_k8s version not found in software_config.json")
         service_k8s_json = f"service_k8s_v{service_k8s_version}.json"
 
     # pylint: disable=line-too-long
+    # Determine which K8s package JSON to use based on software_config.json
+    if "service_rke2" in allowed_softwares:
+        k8s_json = "service_rke2.json"
+    elif service_k8s_json:
+        k8s_json = service_k8s_json
+    else:
+        k8s_json = None
+
     # Functional group → json files mapping
     software_map = {
         "os_x86_64": ["default_packages.json", "ldms.json"],
@@ -280,12 +291,12 @@ def run_module():
         ],
     }
 
-    # Add k8s functional groups to software_map only if service_k8s_json is available
-    if service_k8s_json:
+    # Add k8s functional groups to software_map if k8s json is available
+    if k8s_json:
         software_map.update({
-            "service_kube_node_x86_64": [service_k8s_json],
-            "service_kube_control_plane_first_x86_64": [service_k8s_json],
-            "service_kube_control_plane_x86_64": [service_k8s_json],
+            "service_kube_node_x86_64": [k8s_json],
+            "service_kube_control_plane_first_x86_64": [k8s_json],
+            "service_kube_control_plane_x86_64": [k8s_json],
         })
 
     compute_images_dict = {}

input/config/x86_64/rhel/10.0/service_rke2.json

@@ -0,0 +1,73 @@
+{
+  "service_rke2": {
+    "cluster": [
+      { "package": "docker.io/library/busybox", "type": "image", "tag": "1.36" },
+      { "package": "firewalld", "type": "rpm", "repo_name": "x86_64_baseos" },
+      { "package": "python3-firewall", "type": "rpm", "repo_name": "x86_64_baseos" },
+      { "package": "git", "type": "rpm", "repo_name": "x86_64_appstream"},
+      { "package": "vim", "type": "rpm", "repo_name": "x86_64_appstream"},
+      { "package": "rke2-server", "type": "rpm", "repo_name": "rke2" },
+      { "package": "rke2-agent", "type": "rpm", "repo_name": "rke2" },
+      { "package": "rke2-common", "type": "rpm", "repo_name": "rke2" },
+      { "package": "rke2-selinux", "type": "rpm", "repo_name": "rke2" },
+      { "package": "docker.io/victoriametrics/victoria-metrics", "type": "image", "tag": "v1.128.0" },
+      { "package": "docker.io/victoriametrics/vmagent", "type": "image", "tag": "v1.128.0" },
+      { "package": "docker.io/victoriametrics/vmstorage", "type": "image", "tag": "v1.128.0-cluster" },
+      { "package": "docker.io/victoriametrics/vminsert", "type": "image", "tag": "v1.128.0-cluster" },
+      { "package": "docker.io/victoriametrics/vmselect", "type": "image", "tag": "v1.128.0-cluster" },
+      { "package": "docker.io/curlimages/curl", "type": "image", "tag": "8.17.0" },
+      { "package": "docker.io/rmohr/activemq", "type": "image", "tag": "5.15.9" },
+      { "package": "docker.io/library/mysql", "type": "image", "tag": "9.3.0" },
+      { "package": "docker.io/dellhpcomniaaisolution/idrac_telemetry_receiver", "type": "image", "tag": "1.0" },
+      { "package": "docker.io/dellhpcomniaaisolution/kafkapump", "type": "image", "tag": "1.0" },
+      { "package": "docker.io/dellhpcomniaaisolution/victoriapump", "type": "image", "tag": "1.0" },
+      { "package": "cryptography==45.0.7", "type": "pip_module" },
+      { "package": "omsdk==1.2.518", "type": "pip_module" },
+      { "package": "cffi==1.17.1", "type": "pip_module" },
+      { "package": "quay.io/strimzi/operator", "tag": "0.48.0", "type": "image" },
+      { "package": "quay.io/strimzi/kafka", "tag": "0.48.0-kafka-4.1.0", "type": "image" },
+      { "package": "docker.io/dellhpcomniaaisolution/ubuntu-ldms", "tag": "1.0", "type": "image" },
+      { "package": "strimzi-kafka-operator-helm-3-chart-0.48.0", "type": "tarball", "url": "https://github.com/strimzi/strimzi-kafka-operator/releases/download/0.48.0/strimzi-kafka-operator-helm-3-chart-0.48.0.tgz" },
+      { "package": "quay.io/strimzi/kafka-bridge", "tag": "0.33.1", "type": "image" },
+      { "package": "apptainer", "type": "rpm", "repo_name": "epel" },
+      { "package": "doca-ofed", "type": "iso", "url": "https://www.mellanox.com/downloads/DOCA/DOCA_v3.2.1/host/doca-host-3.2.1-044000_25.10_rhel10.x86_64.rpm"}
+    ]
+  },
+  "service_kube_control_plane": {
+    "cluster": [
+      { "package": "ghcr.io/kube-vip/kube-vip", "tag": "v0.8.9", "type": "image" },
+      { "package": "quay.io/metallb/speaker", "tag": "v0.15.2", "type": "image" },
+      { "package": "prettytable==3.14.0", "type": "pip_module" },
+      { "package": "python3.12", "type": "rpm", "repo_name": "x86_64_appstream" },
+      { "package": "git", "type": "rpm", "repo_name": "x86_64_appstream"},
+      { "package": "kubernetes==33.1.0", "type": "pip_module" },
+      { "package": "PyMySQL==1.1.2", "type": "pip_module" }
+    ]
+  },
+  "service_kube_control_plane_first": {
+    "cluster": [
+      { "package": "ghcr.io/kube-vip/kube-vip", "tag": "v0.8.9", "type": "image" },
+      { "package": "quay.io/metallb/speaker", "tag": "v0.15.2", "type": "image" },
+      {
+        "package": "metallb-native-v0.15.2",
+        "type": "manifest",
+        "url": "https://raw.githubusercontent.com/metallb/metallb/v0.15.2/config/manifests/metallb-native.yaml"
+      },
+      { "package": "helm-v3.19.0-amd64", "type": "tarball", "url": "https://get.helm.sh/helm-v3.19.0-linux-amd64.tar.gz" },
+      { "package": "nfs-subdir-external-provisioner-4.0.18", "type": "tarball", "url": "https://github.com/kubernetes-sigs/nfs-subdir-external-provisioner/releases/download/nfs-subdir-external-provisioner-4.0.18/nfs-subdir-external-provisioner-4.0.18.tgz" },
+      { "package": "prettytable==3.14.0", "type": "pip_module" },
+      { "package": "python3.12", "type": "rpm", "repo_name": "x86_64_appstream" },
+      { "package": "git", "type": "rpm", "repo_name": "x86_64_appstream"},
+      { "package": "kubernetes==33.1.0", "type": "pip_module" },
+      { "package": "PyMySQL==1.1.2", "type": "pip_module" }
+    ]
+  },
+
+  "service_kube_node": {
+    "cluster": [
+      { "package": "registry.k8s.io/sig-storage/nfs-subdir-external-provisioner", "tag": "v4.0.2", "type": "image" },
+      { "package": "quay.io/metallb/speaker", "tag": "v0.15.2", "type": "image" },
+      { "package": "quay.io/metallb/controller", "tag": "v0.15.2", "type": "image" }
+    ]
+  }
+}

input/omnia_config.yml

@@ -201,3 +201,22 @@ service_k8s_cluster:
     k8s_crio_storage_size: "20G"
     csi_powerscale_driver_secret_file_path: ""
     csi_powerscale_driver_values_file_path: ""
+
+# ----------------------------SERVICE RKE2 K8S--------------------------------------------------
+# To deploy RKE2 instead of vanilla Kubernetes (kubeadm), configure this block
+# and add "service_rke2" to software_config.json softwares list instead of "service_k8s".
+# RKE2 bundles its own container runtime (containerd) and manages the CNI lifecycle.
+# k8s_cni supports "calico", "canal", "cilium", and "flannel".
+# Only one of service_k8s_cluster or service_rke2_k8s_cluster should have deployment: true.
+# Please ensure corresponding cluster entry is added to high_availability_config.yml.
+
+# service_rke2_k8s_cluster:
+#   - cluster_name: service_cluster
+#     deployment: false
+#     k8s_cni: "calico"
+#     pod_external_ip_range: "172.16.107.170-172.16.107.200"
+#     k8s_service_addresses: "10.233.0.0/18"
+#     k8s_pod_network_cidr: "10.233.64.0/18"
+#     nfs_storage_name: "nfs_k8s"
+#     csi_powerscale_driver_secret_file_path: ""
+#     csi_powerscale_driver_values_file_path: ""

local_repo/roles/validation/tasks/validate_software_config_json.yml

@@ -31,13 +31,24 @@
 - name: Check if service k8s support is true
   ansible.builtin.set_fact:
     service_k8s_support: "{{ software_config.softwares | selectattr('name', 'equalto', 'service_k8s') | list | length > 0 }}"
+    service_rke2_support: "{{ software_config.softwares | selectattr('name', 'equalto', 'service_rke2') | list | length > 0 }}"
     software_names: "{{ software_config.softwares | map(attribute='name') | select('defined') | list }}"
     software_json_list: "{{ software_names | map('regex_replace', '$', '.json') | list }}"
 
+- name: Set service_k8s_support true when rke2 is selected
+  ansible.builtin.set_fact:
+    service_k8s_support: true
+  when: service_rke2_support | bool
+
 - name: Get k8s archs
   ansible.builtin.set_fact:
     service_k8s_arch: "{{ (software_config.softwares | selectattr('name', 'equalto', 'service_k8s') | first).get('arch', default_archs) }}"
-  when: service_k8s_support
+  when: service_k8s_support and not (service_rke2_support | bool)
+
+- name: Get rke2 archs
+  ansible.builtin.set_fact:
+    service_k8s_arch: "{{ (software_config.softwares | selectattr('name', 'equalto', 'service_rke2') | first).get('arch', default_archs) }}"
+  when: service_rke2_support | bool
 
 - name: Get k8s archs
   ansible.builtin.set_fact:
@@ -91,4 +102,5 @@
     msg: "{{ fail_msg }}"
   when:
     - service_k8s_support
+    - not (service_rke2_support | default(false) | bool)
     - service_k8s_version not in supported_k8s_versions

provision/provision.yml

@@ -67,6 +67,9 @@
               (
                 ['service_k8s']
                 if ( 'service_k8s' in (
+                    lookup('file', hostvars['localhost']['input_project_dir'] ~ '/software_config.json')
+                    | from_json).softwares | map(attribute='name') | list
+                  or 'service_rke2' in (
                     lookup('file', hostvars['localhost']['input_project_dir'] ~ '/software_config.json')
                     | from_json).softwares | map(attribute='name') | list )
                 else []

provision/roles/configure_ochami/tasks/configure_cloud_init_group.yml

@@ -26,11 +26,21 @@
   failed_when: false
   when: not (hostvars['localhost']['upgrade_mode'] | default(false) | bool)
 
+- name: Determine cloud-init template path based on k8s_distro # noqa: yaml[line-length]
+  ansible.builtin.set_fact:
+    ci_group_template_src: >-
+      {%- if k8s_distro | default('kubeadm') == 'rke2' and
+          'service_kube' in functional_group_name -%}
+        cloud_init/rke2/ci-group-{{ functional_group_name }}.yaml.j2
+      {%- else -%}
+        cloud_init/ci-group-{{ functional_group_name }}.yaml.j2
+      {%- endif -%}
+
 - name: Load ci group template
   block:
     - name: Load ci group template - {{ functional_group_name }}
       ansible.builtin.template:
-        src: "cloud_init/ci-group-{{ functional_group_name }}.yaml.j2"
+        src: "{{ ci_group_template_src | trim }}"
         dest: "{{ cloud_init_dir }}/ci-group-{{ functional_group_name }}.yaml"
         mode: "{{ hostvars['localhost']['file_permissions_644'] }}"
   rescue: