feat: install Kubernetes binaries and symlinks to /opt/bin

/usr/local/bin is generally read-only on immutable distributions, so
these binaries cannot be installed there.

The packages published to PMC and the system extensions built from those
install their binaries to /usr/bin. These were previously moved to
/usr/local/bin, but since this is no longer possible, symlinks are now
created in /opt/bin instead.

Symlinks are not used when installing from a tarball because the
versioned binaries (e.g. kubelet-*) are removed, leading to dangling
links.

The `install` command has been used to write these binaries because it
removes the existing file first (rather than following a symlink) and
takes care of making the new file executable.

Author: chewi

e2e/kubelet/generate-kubelet-flags.sh

@@ -29,7 +29,7 @@ for KUBE_BINARY_VERSION in $KUBE_BINARY_VERSIONS; do
     K8S_TGZ_TMP=${KUBE_BINARY_URL##*/}
     retrycmd_get_tarball 120 5 "$K8S_DOWNLOADS_DIR/${K8S_TGZ_TMP}" ${KUBE_BINARY_URL} || exit 120
     tar --transform="s|.*|&-${KUBE_BINARY_VERSION}|" --show-transformed-names -xzvf "$K8S_DOWNLOADS_DIR/${K8S_TGZ_TMP}" \
-        --strip-components=3 -C /usr/local/bin kubernetes/node/bin/kubelet kubernetes/node/bin/kubectl
+        --strip-components=3 -C /opt/bin kubernetes/node/bin/kubelet kubernetes/node/bin/kubectl
     rm -f "$K8S_DOWNLOADS_DIR/${K8S_TGZ_TMP}"
     export KUBE_BINARY_VERSION
     pushd e2e || exit 1

e2e/kubelet/main.go

@@ -26,7 +26,7 @@ func run() error {
 	}
 
 	fmt.Println("k8s version is:", k8sVersion)
-	binaryPath := fmt.Sprintf("/usr/local/bin/kubelet-%s", k8sVersion)
+	binaryPath := fmt.Sprintf("/opt/bin/kubelet-%s", k8sVersion)
 
 	r, w := io.Pipe()
 

parts/linux/cloud-init/artifacts/azlosguard/cse_install_osguard.sh

@@ -13,7 +13,7 @@ installKubeletKubectlPkgFromPMC() {
 installRPMPackageFromFile() {
     local packageName="${1}"
     local desiredVersion="${2}"
-    local targetBinDir="${3:-"/usr/local/bin"}"
+    local targetBinDir="${3:-"/opt/bin"}"
 
     echo "installing ${packageName} version ${desiredVersion} by manually unpacking the RPM"
     if [ "${packageName}" != "kubelet" ] && [ "${packageName}" != "kubectl" ] && [ "${packageName}" != "azure-acr-credential-provider" ]; then
@@ -52,19 +52,8 @@ installRPMPackageFromFile() {
     fi
 
     echo "Unpacking usr/bin/${rpmBinaryName} from ${downloadDir}/${packageName}-${desiredVersion}*"
-    pushd ${downloadDir} || exit 1
-    rpm2cpio "${rpmFile}" | cpio -idmv
-    mkdir -p "${targetBinDir}"
-    if [ -f "usr/bin/${rpmBinaryName}" ]; then
-        mv "usr/bin/${rpmBinaryName}" "${targetBinDir}/${targetBinaryName}"
-    elif [ -f "usr/local/bin/${rpmBinaryName}" ]; then
-        mv "usr/local/bin/${rpmBinaryName}" "${targetBinDir}/${targetBinaryName}"
-    else
-        popd || exit 1
-        rm -rf ${downloadDir}
-        return 1
-    fi
-    popd || exit 1
+    # This assumes that the binary will either be in /usr/bin or /usr/local/bin, but not both.
+    rpm2cpio "${rpmFile}" | cpio -i --to-stdout "./usr/bin/${rpmBinaryName}" "./usr/local/bin/${rpmBinaryName}" | install -m0755 /dev/stdin "${targetBinDir}/${targetBinaryName}"
 	rm -rf ${downloadDir}
 }
 

parts/linux/cloud-init/artifacts/cse_helpers.sh

@@ -171,7 +171,7 @@ AZURELINUX_KATA_OS_NAME="AZURELINUXKATA"
 AZURELINUX_OS_NAME="AZURELINUX"
 FLATCAR_OS_NAME="FLATCAR"
 AZURELINUX_OSGUARD_OS_VARIANT="OSGUARD"
-KUBECTL=/usr/local/bin/kubectl
+KUBECTL=/opt/bin/kubectl
 DOCKER=/usr/bin/docker
 # this will be empty during VHD build
 # but vhd build runs with `set -o nounset`
@@ -1234,6 +1234,7 @@ extract_tarball() {
     local tarball="$1"
     local dest="$2"
     shift 2
+    mkdir -p "$dest"
     # Use tar options if provided, otherwise default to -xzf
     case "$tarball" in
         *.tar.gz|*.tgz)

parts/linux/cloud-init/artifacts/cse_install.sh

@@ -459,15 +459,15 @@ installAzureCNI() {
 }
 
 # extract the cached or downloaded kube package and remove
-extractKubeBinariesToUsrLocalBin() {
+extractKubeBinariesToOptBin() {
     local k8s_tgz_tmp=$1
     local k8s_version=$2
     local is_private_url=$3
 
-    extract_tarball "${k8s_tgz_tmp}" "/usr/local/bin" \
+    extract_tarball "${k8s_tgz_tmp}" "/opt/bin" \
         --transform="s|.*|&-${k8s_version}|" --show-transformed-names --strip-components=3 \
         kubernetes/node/bin/kubelet kubernetes/node/bin/kubectl || exit $ERR_K8S_INSTALL_ERR
-    if [ ! -f "/usr/local/bin/kubectl-${k8s_version}" ] || [ ! -f "/usr/local/bin/kubelet-${k8s_version}" ]; then
+    if [ ! -f "/opt/bin/kubectl-${k8s_version}" ] || [ ! -f "/opt/bin/kubelet-${k8s_version}" ]; then
         exit $ERR_K8S_INSTALL_ERR
     fi
     if [ "$is_private_url" = "false" ]; then
@@ -500,7 +500,7 @@ extractKubeBinaries() {
 
         echo "cached package ${k8s_tgz_tmp} found, will extract that"
         # remove the current kubelet and kubectl binaries before extracting new binaries from the cached package
-        rm -rf /usr/local/bin/kubelet-* /usr/local/bin/kubectl-*
+        rm -rf /opt/bin/kubelet-* /opt/bin/kubectl-*
     else
         k8s_tgz_tmp="${k8s_downloads_dir}/${k8s_tgz_tmp_filename}"
         mkdir -p ${k8s_downloads_dir}
@@ -523,7 +523,7 @@ extractKubeBinaries() {
         fi
     fi
 
-    extractKubeBinariesToUsrLocalBin "${k8s_tgz_tmp}" "${k8s_version}" "${is_private_url}"
+    extractKubeBinariesToOptBin "${k8s_tgz_tmp}" "${k8s_version}" "${is_private_url}"
 }
 
 installToolFromBootstrapProfileRegistry() {
@@ -579,7 +579,7 @@ installKubeletKubectlFromBootstrapProfileRegistry() {
     local registry_server=$1
     local kubernetes_version=$2
     for tool_name in $(get_kubernetes_tools); do
-        install_path="/usr/local/bin/${tool_name}"
+        install_path="/opt/bin/${tool_name}"
         if ! installToolFromBootstrapProfileRegistry "${tool_name}" "${registry_server}" "${kubernetes_version}" "${install_path}"; then
             # SHOULD_ENFORCE_KUBE_PMC_INSTALL will only be set for e2e tests, which should not fallback to reflect result of package installation behavior
             # TODO: remove SHOULD_ENFORCE_KUBE_PMC_INSTALL check when the test cluster supports > 1.34.0 case
@@ -604,7 +604,7 @@ installKubeletKubectlFromURL() {
 
     if [ ! -z "${CUSTOM_KUBE_BINARY_DOWNLOAD_URL}" ]; then
         # remove the kubelet and kubectl binaries to make sure the only binary left is from the CUSTOM_KUBE_BINARY_DOWNLOAD_URL
-        rm -rf /usr/local/bin/kubelet-* /usr/local/bin/kubectl-*
+        rm -rf /opt/bin/kubelet-* /opt/bin/kubectl-*
 
         # NOTE(mainred): we expect kubelet binary to be under `kubernetes/node/bin`. This suits the current setting of
         # kube binaries used by AKS and Kubernetes upstream.
@@ -617,7 +617,7 @@ installKubeletKubectlFromURL() {
     fi
 
     # if the custom url is not specified and the required kubectl/kubelet-version via private url is not installed, install using the default url/package
-    if [ ! -f "/usr/local/bin/kubectl-${KUBERNETES_VERSION}" ] || [ ! -f "/usr/local/bin/kubelet-${KUBERNETES_VERSION}" ]; then
+    if [ ! -f "/opt/bin/kubectl-${KUBERNETES_VERSION}" ] || [ ! -f "/opt/bin/kubelet-${KUBERNETES_VERSION}" ]; then
         if [ "$install_default_if_missing" = "true" ]; then
             if [ -n "${BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER}" ]; then
                 # network isolated cluster
@@ -635,11 +635,10 @@ installKubeletKubectlFromURL() {
             fi
         fi
     fi
-    mv "/usr/local/bin/kubelet-${KUBERNETES_VERSION}" "/usr/local/bin/kubelet"
-    mv "/usr/local/bin/kubectl-${KUBERNETES_VERSION}" "/usr/local/bin/kubectl"
+    install -m0755 "/opt/bin/kubelet-${KUBERNETES_VERSION}" /opt/bin/kubelet
+    install -m0755 "/opt/bin/kubectl-${KUBERNETES_VERSION}" /opt/bin/kubectl
 
-    chmod a+x /usr/local/bin/kubelet /usr/local/bin/kubectl
-    rm -rf /usr/local/bin/kubelet-* /usr/local/bin/kubectl-* /home/hyperkube-downloads &
+    rm -rf /opt/bin/kubelet-* /opt/bin/kubectl-* /home/hyperkube-downloads &
 }
 
 pullContainerImage() {

parts/linux/cloud-init/artifacts/kubelet.service

@@ -1,6 +1,6 @@
 [Unit]
 Description=Kubelet
-ConditionPathExists=/usr/local/bin/kubelet
+ConditionPathExists=/opt/bin/kubelet
 Wants=network-online.target containerd.service
 After=network-online.target containerd.service
 
@@ -22,7 +22,7 @@ ExecStartPre=-/sbin/iptables -t nat --numeric --list
 
 ExecStartPre=/bin/bash /opt/azure/containers/validate-kubelet-credentials.sh
 
-ExecStart=/usr/local/bin/kubelet \
+ExecStart=/opt/bin/kubelet \
         --enable-server \
         --node-labels="${KUBELET_NODE_LABELS}" \
         --v=2 \

parts/linux/cloud-init/artifacts/mariner/cse_install_mariner.sh

@@ -189,7 +189,7 @@ installCredentialProviderFromPMC() {
     mkdir -p "${CREDENTIAL_PROVIDER_BIN_DIR}"
     chown -R root:root "${CREDENTIAL_PROVIDER_BIN_DIR}"
     installRPMPackageFromFile "azure-acr-credential-provider" "${packageVersion}" || exit $ERR_CREDENTIAL_PROVIDER_DOWNLOAD_TIMEOUT
-    mv "/usr/local/bin/azure-acr-credential-provider" "$CREDENTIAL_PROVIDER_BIN_DIR/acr-credential-provider"
+    ln -snf /usr/bin/azure-acr-credential-provider "$CREDENTIAL_PROVIDER_BIN_DIR/acr-credential-provider"
 }
 
 installKubeletKubectlPkgFromPMC() {
@@ -395,7 +395,8 @@ installRPMPackageFromFile() {
     if ! dnf_install 30 1 600 ${rpmFile}; then
         exit $ERR_APT_INSTALL_TIMEOUT
     fi
-    mv "/usr/bin/${packageName}" "/usr/local/bin/${packageName}"
+    mkdir -p /opt/bin
+    ln -snf "/usr/bin/${packageName}" "/opt/bin/${packageName}"
 	rm -rf ${downloadDir}
 }
 

parts/linux/cloud-init/artifacts/mariner/mariner-package-update.sh

@@ -8,7 +8,7 @@ set -e
 OS_RELEASE_FILE="/etc/os-release"
 SECURITY_PATCH_REPO_DIR="/etc/yum.repos.d"
 KUBECONFIG="/var/lib/kubelet/kubeconfig"
-KUBECTL="/usr/local/bin/kubectl --kubeconfig ${KUBECONFIG}"
+KUBECTL="/opt/bin/kubectl --kubeconfig ${KUBECONFIG}"
 KUBELET_EXECUTABLE="/usr/local/bin/kubelet"
 SECURITY_PATCH_TMP_DIR="/tmp/security-patch"
 

parts/linux/cloud-init/artifacts/ubuntu/cse_install_ubuntu.sh

@@ -202,7 +202,7 @@ installCredentialProviderFromPMC() {
     mkdir -p "${CREDENTIAL_PROVIDER_BIN_DIR}"
     chown -R root:root "${CREDENTIAL_PROVIDER_BIN_DIR}"
     installPkgWithAptGet "azure-acr-credential-provider" "${packageVersion}" || exit $ERR_CREDENTIAL_PROVIDER_DOWNLOAD_TIMEOUT
-    mv "/usr/local/bin/azure-acr-credential-provider" "$CREDENTIAL_PROVIDER_BIN_DIR/acr-credential-provider"
+    ln -snf /usr/bin/azure-acr-credential-provider "$CREDENTIAL_PROVIDER_BIN_DIR/acr-credential-provider"
 }
 
 installKubeletKubectlPkgFromPMC() {
@@ -307,7 +307,8 @@ installPkgWithAptGet() {
 
     logs_to_events "AKS.CSE.install${packageName}.installDebPackageFromFile" "installDebPackageFromFile ${debFile}" || exit $ERR_APT_INSTALL_TIMEOUT
 
-    mv "/usr/bin/${packageName}" "/usr/local/bin/${packageName}"
+    mkdir -p /opt/bin
+    ln -snf "/usr/bin/${packageName}" "/opt/bin/${packageName}"
     rm -rf ${downloadDir}
 }
 

parts/linux/cloud-init/artifacts/ubuntu/ubuntu-snapshot-update.sh

@@ -7,7 +7,7 @@ set -e
 # -------------------------------------------------------------------------------------------------
 SECURITY_PATCH_CONFIG_DIR=/var/lib/security-patch
 KUBECONFIG="/var/lib/kubelet/kubeconfig"
-KUBECTL="/usr/local/bin/kubectl --kubeconfig ${KUBECONFIG}"
+KUBECTL="/opt/bin/kubectl --kubeconfig ${KUBECONFIG}"
 DEFAULT_ENDPOINT="snapshot.ubuntu.com"
 
 # Function definitions used in this file.