diff --git a/images/capi/ansible/roles/kubernetes/files/usr/libexec/kubernetes/kubelet-resource-sizing.sh b/images/capi/ansible/roles/kubernetes/files/usr/libexec/kubernetes/kubelet-resource-sizing.sh index 864b0aa76d..4535510634 100755 --- a/images/capi/ansible/roles/kubernetes/files/usr/libexec/kubernetes/kubelet-resource-sizing.sh +++ b/images/capi/ansible/roles/kubernetes/files/usr/libexec/kubernetes/kubelet-resource-sizing.sh @@ -5,6 +5,7 @@ #RPM and DEB systems kubelet sysconfig PATH KUBELET_SYSCONFIG_FILES=( "/etc/sysconfig/kubelet" "/etc/default/kubelet" ) +KUBELET_CONFIG="/var/lib/kubelet/kubelet.conf.d/kubelet-resource-sizing.json" for KUBELET_SYSCONFIG in "${KUBELET_SYSCONFIG_FILES[@]}" do @@ -13,7 +14,7 @@ do # shellcheck source=/dev/null . "${KUBELET_SYSCONFIG}" # If system-reserved is already set by user, ignore - if grep -q 'KUBELET_EXTRA_ARGS=.*--system-reserved' "${KUBELET_SYSCONFIG}"; then + if grep -q 'systemReserved' "${KUBELET_SYSCONFIG}"; then exit 0 fi fi @@ -98,7 +99,7 @@ CPU_CORE_RESERVATION_MICROCORES=( ) # Calculate the CPU reservation -cpu_milicores_to_reserve() { +cpu_millicores_to_reserve() { local cpu_microcores_reserved=0 for ((i = 0; i < schedulable_cores_no; i++)); do @@ -113,13 +114,16 @@ cpu_milicores_to_reserve() { echo "$cpu_microcores_reserved" | awk '{result = $1 / 10; if (result != int(result)) result++; printf "%d\n", result}' } -mkdir -p /run/kubelet -# Check if system-reserved already exists -if grep '.*--system-reserved' <<< "${KUBELET_EXTRA_ARGS}"; then - # If system-reserved is already set by a previous run, replace old value with new one and write to /run/kubelet/extra-args.env - system_reserved=$(sed -E "s|--system-reserved=cpu=[0-9]+m,memory=[0-9]+Mi|--system-reserved=cpu=$(cpu_milicores_to_reserve)m,memory=$(memory_reservation_mebibytes)Mi|" <<< "${KUBELET_EXTRA_ARGS}") - echo "KUBELET_EXTRA_ARGS=${system_reserved} >/run/kubelet/extra-args.env" -else - # If not append system-reserved to KUBELET_EXTRA_ARGS and write to /run/kubelet/extra-args.env - echo "KUBELET_EXTRA_ARGS=${KUBELET_EXTRA_ARGS} --system-reserved=cpu=$(cpu_milicores_to_reserve)m,memory=$(memory_reservation_mebibytes)Mi" >/run/kubelet/extra-args.env +mkdir -p /var/lib/kubelet/kubelet.conf.d + +# Initialize config file if it doesn't exist +if [ ! -f "$KUBELET_CONFIG" ]; then + echo "{}" > "$KUBELET_CONFIG" fi + +# Get the computed values from the functions +memory_reservation_mebibytes=$(memory_reservation_mebibytes) +cpu_millicores_to_reserve=$(cpu_millicores_to_reserve) + +echo "$(jq --arg memory_reservation_mebibytes "${memory_reservation_mebibytes}Mi" --arg cpu_millicores_to_reserve "${cpu_millicores_to_reserve}m" \ + '. += {"apiVersion": "kubelet.config.k8s.io/v1beta1","kind": "KubeletConfiguration", "systemReserved": {"cpu": $cpu_millicores_to_reserve, "ephemeral-storage": "1Gi", "memory": $memory_reservation_mebibytes}}' "$KUBELET_CONFIG")" > "$KUBELET_CONFIG" diff --git a/images/capi/ansible/roles/kubernetes/tasks/main.yml b/images/capi/ansible/roles/kubernetes/tasks/main.yml index c2e563d0fe..60d259dd36 100644 --- a/images/capi/ansible/roles/kubernetes/tasks/main.yml +++ b/images/capi/ansible/roles/kubernetes/tasks/main.yml @@ -84,6 +84,19 @@ mode: "0644" when: kubernetes_enable_automatic_resource_sizing | bool +- name: Set Kubelet Configuration Drop-in Directory argument + ansible.builtin.lineinfile: + path: "{{ systemd_prefix }}/system/kubelet.service.d/10-kubeadm.conf" + regexp: '^Environment="KUBELET_CONFIG_ARGS=' + line: Environment="KUBELET_CONFIG_ARGS=--config=/var/lib/kubelet/config.yaml --config-dir /var/lib/kubelet/kubelet.conf.d" + when: kubernetes_semver is version('v1.28.0', '>=') and kubernetes_enable_automatic_resource_sizing | bool + +- name: Add KUBELET_CONFIG_DROPIN_DIR_ALPHA environment variable for Kubernetes v1.29 and below + ansible.builtin.lineinfile: + path: "{{ systemd_prefix }}/system/kubelet.service.d/10-kubeadm.conf" + line: Environment="KUBELET_CONFIG_DROPIN_DIR_ALPHA" + when: kubernetes_semver is version('v1.28.0', '>=') and kubernetes_semver is version('v1.29.0', '<') and kubernetes_enable_automatic_resource_sizing | bool + - name: Generate kubectl bash completion ansible.builtin.shell: cmd: "{{ sysusr_prefix }}/bin/kubectl completion bash > {{ sysusr_prefix }}/share/bash-completion/completions/kubectl" diff --git a/images/capi/ansible/roles/kubernetes/templates/usr/lib/systemd/system/kubelet.service.d/10-kubeadm.conf b/images/capi/ansible/roles/kubernetes/templates/usr/lib/systemd/system/kubelet.service.d/10-kubeadm.conf index 4f1b8156b9..95712ca4df 100644 --- a/images/capi/ansible/roles/kubernetes/templates/usr/lib/systemd/system/kubelet.service.d/10-kubeadm.conf +++ b/images/capi/ansible/roles/kubernetes/templates/usr/lib/systemd/system/kubelet.service.d/10-kubeadm.conf @@ -1,7 +1,11 @@ # Note: This dropin only works with kubeadm and kubelet v1.11+ [Service] Environment="KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf" -Environment="KUBELET_CONFIG_ARGS=--config=/var/lib/kubelet/config.yaml" +Environment="KUBELET_CONFIG_ARGS=--config=/var/lib/kubelet/config.yaml {% if kubernetes_enable_automatic_resource_sizing | bool and kubernetes_semver is version('v1.28.0', '>=') %}--config-dir /var/lib/kubelet/kubelet.conf.d{% endif %}" +# For Kubernetes v1.28 to v1.29, you can only specify --config-dir if you also set the environment variable KUBELET_CONFIG_DROPIN_DIR_ALPHA for the kubelet process (the value of that variable does not matter). +{% if kubernetes_semver is version('v1.28.0', '>=') and kubernetes_semver is version('v1.29.0', '<') %} +Environment="KUBELET_CONFIG_DROPIN_DIR_ALPHA" +{% endif %} # This is a file that "kubeadm init" and "kubeadm join" generates at runtime, populating the KUBELET_KUBEADM_ARGS variable dynamically EnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env # This is a file that the user can use for overrides of the kubelet args as a last resort. Preferably, the user should use