From 0c1a5e44a5c221e46011a027e5088ceb693af486 Mon Sep 17 00:00:00 2001 From: ffais Date: Wed, 22 Apr 2026 18:18:40 +0200 Subject: [PATCH 1/2] move kubelet --system-reserved deprecated flag to kubelet-config file Signed-off-by: ffais --- .../kubernetes/kubelet-resource-sizing.sh | 16 +++++----------- .../capi/ansible/roles/kubernetes/tasks/main.yml | 13 +++++++++++++ .../system/kubelet.service.d/10-kubeadm.conf | 6 +++++- 3 files changed, 23 insertions(+), 12 deletions(-) diff --git a/images/capi/ansible/roles/kubernetes/files/usr/libexec/kubernetes/kubelet-resource-sizing.sh b/images/capi/ansible/roles/kubernetes/files/usr/libexec/kubernetes/kubelet-resource-sizing.sh index 864b0aa76d..f196a37ebc 100755 --- a/images/capi/ansible/roles/kubernetes/files/usr/libexec/kubernetes/kubelet-resource-sizing.sh +++ b/images/capi/ansible/roles/kubernetes/files/usr/libexec/kubernetes/kubelet-resource-sizing.sh @@ -5,6 +5,7 @@ #RPM and DEB systems kubelet sysconfig PATH KUBELET_SYSCONFIG_FILES=( "/etc/sysconfig/kubelet" "/etc/default/kubelet" ) +KUBELET_CONFIG="/var/lib/kubelet/kubelet.conf.d/kubelet-resource-sizing.json" for KUBELET_SYSCONFIG in "${KUBELET_SYSCONFIG_FILES[@]}" do @@ -13,7 +14,7 @@ do # shellcheck source=/dev/null . "${KUBELET_SYSCONFIG}" # If system-reserved is already set by user, ignore - if grep -q 'KUBELET_EXTRA_ARGS=.*--system-reserved' "${KUBELET_SYSCONFIG}"; then + if grep -q 'systemReserved' "${KUBELET_SYSCONFIG}"; then exit 0 fi fi @@ -113,13 +114,6 @@ cpu_milicores_to_reserve() { echo "$cpu_microcores_reserved" | awk '{result = $1 / 10; if (result != int(result)) result++; printf "%d\n", result}' } -mkdir -p /run/kubelet -# Check if system-reserved already exists -if grep '.*--system-reserved' <<< "${KUBELET_EXTRA_ARGS}"; then - # If system-reserved is already set by a previous run, replace old value with new one and write to /run/kubelet/extra-args.env - system_reserved=$(sed -E "s|--system-reserved=cpu=[0-9]+m,memory=[0-9]+Mi|--system-reserved=cpu=$(cpu_milicores_to_reserve)m,memory=$(memory_reservation_mebibytes)Mi|" <<< "${KUBELET_EXTRA_ARGS}") - echo "KUBELET_EXTRA_ARGS=${system_reserved} >/run/kubelet/extra-args.env" -else - # If not append system-reserved to KUBELET_EXTRA_ARGS and write to /run/kubelet/extra-args.env - echo "KUBELET_EXTRA_ARGS=${KUBELET_EXTRA_ARGS} --system-reserved=cpu=$(cpu_milicores_to_reserve)m,memory=$(memory_reservation_mebibytes)Mi" >/run/kubelet/extra-args.env -fi +mkdir -p /var/lib/kubelet/kubelet.conf.d +echo "$(jq --arg mebibytes_to_reserve "${mebibytes_to_reserve}Mi" --arg cpu_millicores_to_reserve "${cpu_millicores_to_reserve}m" \ + '. += {systemReserved: {"cpu": $cpu_millicores_to_reserve, "ephemeral-storage": "1Gi", "memory": $mebibytes_to_reserve}}' $KUBELET_CONFIG)" > $KUBELET_CONFIG diff --git a/images/capi/ansible/roles/kubernetes/tasks/main.yml b/images/capi/ansible/roles/kubernetes/tasks/main.yml index c2e563d0fe..3f3d7f1274 100644 --- a/images/capi/ansible/roles/kubernetes/tasks/main.yml +++ b/images/capi/ansible/roles/kubernetes/tasks/main.yml @@ -84,6 +84,19 @@ mode: "0644" when: kubernetes_enable_automatic_resource_sizing | bool +- name: Set Kubelet Configuration Drop-in Directory argument + ansible.builtin.lineinfile: + path: "{{ systemd_prefix }}/system/kubelet.service.d/10-kubeadm.conf" + regexp: '^Environment="KUBELET_CONFIG_ARGS=' + line: Environment="KUBELET_CONFIG_ARGS=--config=/var/lib/kubelet/config.yaml --config-dir /var/lib/kubelet/kubelet.conf.d" + when: kubernetes_enable_automatic_resource_sizing | bool + +- name: Add KUBELET_CONFIG_DROPIN_DIR_ALPHA environment variable for Kubernetes v1.29 and below + ansible.builtin.lineinfile: + path: "{{ systemd_prefix }}/system/kubelet.service.d/10-kubeadm.conf" + line: Environment="KUBELET_CONFIG_DROPIN_DIR_ALPHA" + when: kubernetes_semver is version('v1.29.0', '<=') and kubernetes_enable_automatic_resource_sizing | bool + - name: Generate kubectl bash completion ansible.builtin.shell: cmd: "{{ sysusr_prefix }}/bin/kubectl completion bash > {{ sysusr_prefix }}/share/bash-completion/completions/kubectl" diff --git a/images/capi/ansible/roles/kubernetes/templates/usr/lib/systemd/system/kubelet.service.d/10-kubeadm.conf b/images/capi/ansible/roles/kubernetes/templates/usr/lib/systemd/system/kubelet.service.d/10-kubeadm.conf index 4f1b8156b9..9e6804a638 100644 --- a/images/capi/ansible/roles/kubernetes/templates/usr/lib/systemd/system/kubelet.service.d/10-kubeadm.conf +++ b/images/capi/ansible/roles/kubernetes/templates/usr/lib/systemd/system/kubelet.service.d/10-kubeadm.conf @@ -1,7 +1,11 @@ # Note: This dropin only works with kubeadm and kubelet v1.11+ [Service] Environment="KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf" -Environment="KUBELET_CONFIG_ARGS=--config=/var/lib/kubelet/config.yaml" +Environment="KUBELET_CONFIG_ARGS=--config=/var/lib/kubelet/config.yaml {% if kubernetes_enable_automatic_resource_sizing | bool %}--config-dir /var/lib/kubelet/kubelet.conf.d{% endif %}" +# For Kubernetes v1.28 to v1.29, you can only specify --config-dir if you also set the environment variable KUBELET_CONFIG_DROPIN_DIR_ALPHA for the kubelet process (the value of that variable does not matter). +{% if kubernetes_semver is version('v1.29.0', '<=') %} +Environment="KUBELET_CONFIG_DROPIN_DIR_ALPHA" +{% endif %} # This is a file that "kubeadm init" and "kubeadm join" generates at runtime, populating the KUBELET_KUBEADM_ARGS variable dynamically EnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env # This is a file that the user can use for overrides of the kubelet args as a last resort. Preferably, the user should use From b6b93c995e43165e9691713185aea181c4babfd6 Mon Sep 17 00:00:00 2001 From: ffais Date: Fri, 1 May 2026 12:28:40 +0200 Subject: [PATCH 2/2] fix resource sizing script & add version check on main.yaml and 10-kubeadm.conf Signed-off-by: ffais --- .../kubernetes/kubelet-resource-sizing.sh | 16 +++++++++++++--- .../capi/ansible/roles/kubernetes/tasks/main.yml | 4 ++-- .../system/kubelet.service.d/10-kubeadm.conf | 4 ++-- 3 files changed, 17 insertions(+), 7 deletions(-) diff --git a/images/capi/ansible/roles/kubernetes/files/usr/libexec/kubernetes/kubelet-resource-sizing.sh b/images/capi/ansible/roles/kubernetes/files/usr/libexec/kubernetes/kubelet-resource-sizing.sh index f196a37ebc..4535510634 100755 --- a/images/capi/ansible/roles/kubernetes/files/usr/libexec/kubernetes/kubelet-resource-sizing.sh +++ b/images/capi/ansible/roles/kubernetes/files/usr/libexec/kubernetes/kubelet-resource-sizing.sh @@ -99,7 +99,7 @@ CPU_CORE_RESERVATION_MICROCORES=( ) # Calculate the CPU reservation -cpu_milicores_to_reserve() { +cpu_millicores_to_reserve() { local cpu_microcores_reserved=0 for ((i = 0; i < schedulable_cores_no; i++)); do @@ -115,5 +115,15 @@ cpu_milicores_to_reserve() { } mkdir -p /var/lib/kubelet/kubelet.conf.d -echo "$(jq --arg mebibytes_to_reserve "${mebibytes_to_reserve}Mi" --arg cpu_millicores_to_reserve "${cpu_millicores_to_reserve}m" \ - '. += {systemReserved: {"cpu": $cpu_millicores_to_reserve, "ephemeral-storage": "1Gi", "memory": $mebibytes_to_reserve}}' $KUBELET_CONFIG)" > $KUBELET_CONFIG + +# Initialize config file if it doesn't exist +if [ ! -f "$KUBELET_CONFIG" ]; then + echo "{}" > "$KUBELET_CONFIG" +fi + +# Get the computed values from the functions +memory_reservation_mebibytes=$(memory_reservation_mebibytes) +cpu_millicores_to_reserve=$(cpu_millicores_to_reserve) + +echo "$(jq --arg memory_reservation_mebibytes "${memory_reservation_mebibytes}Mi" --arg cpu_millicores_to_reserve "${cpu_millicores_to_reserve}m" \ + '. += {"apiVersion": "kubelet.config.k8s.io/v1beta1","kind": "KubeletConfiguration", "systemReserved": {"cpu": $cpu_millicores_to_reserve, "ephemeral-storage": "1Gi", "memory": $memory_reservation_mebibytes}}' "$KUBELET_CONFIG")" > "$KUBELET_CONFIG" diff --git a/images/capi/ansible/roles/kubernetes/tasks/main.yml b/images/capi/ansible/roles/kubernetes/tasks/main.yml index 3f3d7f1274..60d259dd36 100644 --- a/images/capi/ansible/roles/kubernetes/tasks/main.yml +++ b/images/capi/ansible/roles/kubernetes/tasks/main.yml @@ -89,13 +89,13 @@ path: "{{ systemd_prefix }}/system/kubelet.service.d/10-kubeadm.conf" regexp: '^Environment="KUBELET_CONFIG_ARGS=' line: Environment="KUBELET_CONFIG_ARGS=--config=/var/lib/kubelet/config.yaml --config-dir /var/lib/kubelet/kubelet.conf.d" - when: kubernetes_enable_automatic_resource_sizing | bool + when: kubernetes_semver is version('v1.28.0', '>=') and kubernetes_enable_automatic_resource_sizing | bool - name: Add KUBELET_CONFIG_DROPIN_DIR_ALPHA environment variable for Kubernetes v1.29 and below ansible.builtin.lineinfile: path: "{{ systemd_prefix }}/system/kubelet.service.d/10-kubeadm.conf" line: Environment="KUBELET_CONFIG_DROPIN_DIR_ALPHA" - when: kubernetes_semver is version('v1.29.0', '<=') and kubernetes_enable_automatic_resource_sizing | bool + when: kubernetes_semver is version('v1.28.0', '>=') and kubernetes_semver is version('v1.29.0', '<') and kubernetes_enable_automatic_resource_sizing | bool - name: Generate kubectl bash completion ansible.builtin.shell: diff --git a/images/capi/ansible/roles/kubernetes/templates/usr/lib/systemd/system/kubelet.service.d/10-kubeadm.conf b/images/capi/ansible/roles/kubernetes/templates/usr/lib/systemd/system/kubelet.service.d/10-kubeadm.conf index 9e6804a638..95712ca4df 100644 --- a/images/capi/ansible/roles/kubernetes/templates/usr/lib/systemd/system/kubelet.service.d/10-kubeadm.conf +++ b/images/capi/ansible/roles/kubernetes/templates/usr/lib/systemd/system/kubelet.service.d/10-kubeadm.conf @@ -1,9 +1,9 @@ # Note: This dropin only works with kubeadm and kubelet v1.11+ [Service] Environment="KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf" -Environment="KUBELET_CONFIG_ARGS=--config=/var/lib/kubelet/config.yaml {% if kubernetes_enable_automatic_resource_sizing | bool %}--config-dir /var/lib/kubelet/kubelet.conf.d{% endif %}" +Environment="KUBELET_CONFIG_ARGS=--config=/var/lib/kubelet/config.yaml {% if kubernetes_enable_automatic_resource_sizing | bool and kubernetes_semver is version('v1.28.0', '>=') %}--config-dir /var/lib/kubelet/kubelet.conf.d{% endif %}" # For Kubernetes v1.28 to v1.29, you can only specify --config-dir if you also set the environment variable KUBELET_CONFIG_DROPIN_DIR_ALPHA for the kubelet process (the value of that variable does not matter). -{% if kubernetes_semver is version('v1.29.0', '<=') %} +{% if kubernetes_semver is version('v1.28.0', '>=') and kubernetes_semver is version('v1.29.0', '<') %} Environment="KUBELET_CONFIG_DROPIN_DIR_ALPHA" {% endif %} # This is a file that "kubeadm init" and "kubeadm join" generates at runtime, populating the KUBELET_KUBEADM_ARGS variable dynamically