feat: install Kubernetes binaries and symlinks to /opt/bin

/usr/local/bin is generally read-only on immutable distributions, so
these binaries cannot be installed there.

The packages published to PMC and the system extensions built from those
install their binaries to /usr/bin. These were previously moved to
/usr/local/bin, but since this is no longer possible, symlinks are now
created in /opt/bin instead.

Symlinks are not used when installing from a tarball because the
versioned binaries (e.g. kubelet-*) are removed, leading to dangling
links.

The `install` command has been used to write these binaries because it
removes the existing file first (rather than following a symlink) and
takes care of making the new file executable.

Author: chewi

e2e/kubelet/generate-kubelet-flags.sh

@@ -29,7 +29,7 @@ for KUBE_BINARY_VERSION in $KUBE_BINARY_VERSIONS; do
     K8S_TGZ_TMP=${KUBE_BINARY_URL##*/}
     retrycmd_get_tarball 120 5 "$K8S_DOWNLOADS_DIR/${K8S_TGZ_TMP}" ${KUBE_BINARY_URL} || exit 120
     tar --transform="s|.*|&-${KUBE_BINARY_VERSION}|" --show-transformed-names -xzvf "$K8S_DOWNLOADS_DIR/${K8S_TGZ_TMP}" \
-        --strip-components=3 -C /usr/local/bin kubernetes/node/bin/kubelet kubernetes/node/bin/kubectl
+        --strip-components=3 -C /opt/bin kubernetes/node/bin/kubelet kubernetes/node/bin/kubectl
     rm -f "$K8S_DOWNLOADS_DIR/${K8S_TGZ_TMP}"
     export KUBE_BINARY_VERSION
     pushd e2e || exit 1

e2e/kubelet/main.go

@@ -26,7 +26,7 @@ func run() error {
 	}
 
 	fmt.Println("k8s version is:", k8sVersion)
-	binaryPath := fmt.Sprintf("/usr/local/bin/kubelet-%s", k8sVersion)
+	binaryPath := fmt.Sprintf("/opt/bin/kubelet-%s", k8sVersion)
 
 	r, w := io.Pipe()
 

parts/linux/cloud-init/artifacts/azlosguard/cse_install_osguard.sh

@@ -41,10 +41,7 @@ installRPMPackageFromFile() {
     fi
 
     echo "Unpacking usr/bin/${packageName} from ${downloadDir}/${packageName}-${desiredVersion}*"
-    pushd ${downloadDir} || exit 1
-    rpm2cpio "${rpmFile}" | cpio -idmv
-    mv "usr/bin/${packageName}" "/usr/local/bin/${packageName}"
-    popd || exit 1
+    rpm2cpio "${rpmFile}" | cpio -i --to-stdout "./usr/bin/${packageName}" | install -m0755 /dev/stdin "/opt/bin/${packageName}"
 	rm -rf ${downloadDir}
 }
 

parts/linux/cloud-init/artifacts/cse_helpers.sh

@@ -161,7 +161,7 @@ AZURELINUX_KATA_OS_NAME="AZURELINUXKATA"
 AZURELINUX_OS_NAME="AZURELINUX"
 FLATCAR_OS_NAME="FLATCAR"
 AZURELINUX_OSGUARD_OS_VARIANT="OSGUARD"
-KUBECTL=/usr/local/bin/kubectl
+KUBECTL=/opt/bin/kubectl
 DOCKER=/usr/bin/docker
 # this will be empty during VHD build
 # but vhd build runs with `set -o nounset`
@@ -1096,6 +1096,7 @@ extract_tarball() {
     local tarball="$1"
     local dest="$2"
     shift 2
+    mkdir -p "$dest"
     # Use tar options if provided, otherwise default to -xzf
     case "$tarball" in
         *.tar.gz|*.tgz)

parts/linux/cloud-init/artifacts/cse_install.sh

@@ -462,15 +462,15 @@ installAzureCNI() {
 }
 
 # extract the cached or downloaded kube package and remove
-extractKubeBinariesToUsrLocalBin() {
+extractKubeBinariesToOptBin() {
     local k8s_tgz_tmp=$1
     local k8s_version=$2
     local is_private_url=$3
 
-    extract_tarball "${k8s_tgz_tmp}" "/usr/local/bin" \
+    extract_tarball "${k8s_tgz_tmp}" "/opt/bin" \
         --transform="s|.*|&-${k8s_version}|" --show-transformed-names --strip-components=3 \
         kubernetes/node/bin/kubelet kubernetes/node/bin/kubectl || exit $ERR_K8S_INSTALL_ERR
-    if [ ! -f "/usr/local/bin/kubectl-${k8s_version}" ] || [ ! -f "/usr/local/bin/kubelet-${k8s_version}" ]; then
+    if [ ! -f "/opt/bin/kubectl-${k8s_version}" ] || [ ! -f "/opt/bin/kubelet-${k8s_version}" ]; then
         exit $ERR_K8S_INSTALL_ERR
     fi
     if [ "$is_private_url" = "false" ]; then
@@ -503,7 +503,7 @@ extractKubeBinaries() {
 
         echo "cached package ${k8s_tgz_tmp} found, will extract that"
         # remove the current kubelet and kubectl binaries before extracting new binaries from the cached package
-        rm -rf /usr/local/bin/kubelet-* /usr/local/bin/kubectl-*
+        rm -rf /opt/bin/kubelet-* /opt/bin/kubectl-*
     else
         k8s_tgz_tmp="${k8s_downloads_dir}/${k8s_tgz_tmp_filename}"
         mkdir -p ${k8s_downloads_dir}
@@ -526,7 +526,7 @@ extractKubeBinaries() {
         fi
     fi
 
-    extractKubeBinariesToUsrLocalBin "${k8s_tgz_tmp}" "${k8s_version}" "${is_private_url}"
+    extractKubeBinariesToOptBin "${k8s_tgz_tmp}" "${k8s_version}" "${is_private_url}"
 }
 
 installKubeletKubectlFromURL() {
@@ -538,7 +538,7 @@ installKubeletKubectlFromURL() {
 
     if [ ! -z "${CUSTOM_KUBE_BINARY_DOWNLOAD_URL}" ]; then
         # remove the kubelet and kubectl binaries to make sure the only binary left is from the CUSTOM_KUBE_BINARY_DOWNLOAD_URL
-        rm -rf /usr/local/bin/kubelet-* /usr/local/bin/kubectl-*
+        rm -rf /opt/bin/kubelet-* /opt/bin/kubectl-*
 
         # NOTE(mainred): we expect kubelet binary to be under `kubernetes/node/bin`. This suits the current setting of
         # kube binaries used by AKS and Kubernetes upstream.
@@ -551,7 +551,7 @@ installKubeletKubectlFromURL() {
     fi
 
     # if the custom url is not specified and the required kubectl/kubelet-version via private url is not installed, install using the default url/package
-    if [ ! -f "/usr/local/bin/kubectl-${KUBERNETES_VERSION}" ] || [ ! -f "/usr/local/bin/kubelet-${KUBERNETES_VERSION}" ]; then
+    if [ ! -f "/opt/bin/kubectl-${KUBERNETES_VERSION}" ] || [ ! -f "/opt/bin/kubelet-${KUBERNETES_VERSION}" ]; then
         if [ "$install_default_if_missing" = "true" ]; then
             if [ -n "${BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER}" ]; then 
                 # network isolated cluster
@@ -569,11 +569,10 @@ installKubeletKubectlFromURL() {
             fi
         fi
     fi
-    mv "/usr/local/bin/kubelet-${KUBERNETES_VERSION}" "/usr/local/bin/kubelet"
-    mv "/usr/local/bin/kubectl-${KUBERNETES_VERSION}" "/usr/local/bin/kubectl"
+    install -m0755 "/opt/bin/kubelet-${KUBERNETES_VERSION}" /opt/bin/kubelet
+    install -m0755 "/opt/bin/kubectl-${KUBERNETES_VERSION}" /opt/bin/kubectl
 
-    chmod a+x /usr/local/bin/kubelet /usr/local/bin/kubectl
-    rm -rf /usr/local/bin/kubelet-* /usr/local/bin/kubectl-* /home/hyperkube-downloads &
+    rm -rf /opt/bin/kubelet-* /opt/bin/kubectl-* /home/hyperkube-downloads &
 }
 
 pullContainerImage() {

parts/linux/cloud-init/artifacts/cse_main.sh

@@ -421,7 +421,7 @@ function nodePrep {
             tee "/etc/systemd/system/nvidia-device-plugin.service.d/10-binary-path.conf" > /dev/null <<'EOF'
 [Service]
 ExecStart=
-ExecStart=/usr/local/bin/nvidia-device-plugin
+ExecStart=/opt/bin/nvidia-device-plugin
 EOF
             # Reload systemd to pick up the base path override
             systemctl daemon-reload
@@ -431,7 +431,7 @@ EOF
 [Service]
 Environment="MIG_STRATEGY=--mig-strategy single"
 ExecStart=
-ExecStart=/usr/local/bin/nvidia-device-plugin $MIG_STRATEGY
+ExecStart=/opt/bin/nvidia-device-plugin $MIG_STRATEGY
 EOF
                 # Reload systemd to pick up drop-ins
                 systemctl daemon-reload

parts/linux/cloud-init/artifacts/kubelet.service

@@ -1,6 +1,6 @@
 [Unit]
 Description=Kubelet
-ConditionPathExists=/usr/local/bin/kubelet
+ConditionPathExists=/opt/bin/kubelet
 Wants=network-online.target containerd.service
 After=network-online.target containerd.service
 
@@ -22,7 +22,7 @@ ExecStartPre=-/sbin/iptables -t nat --numeric --list
 
 ExecStartPre=/bin/bash /opt/azure/containers/validate-kubelet-credentials.sh
 
-ExecStart=/usr/local/bin/kubelet \
+ExecStart=/opt/bin/kubelet \
         --enable-server \
         --node-labels="${KUBELET_NODE_LABELS}" \
         --v=2 \

parts/linux/cloud-init/artifacts/mariner/cse_install_mariner.sh

@@ -168,7 +168,7 @@ installCredentialProviderFromPMC() {
     mkdir -p "${CREDENTIAL_PROVIDER_BIN_DIR}"
     chown -R root:root "${CREDENTIAL_PROVIDER_BIN_DIR}"
     installRPMPackageFromFile "azure-acr-credential-provider" "${packageVersion}" || exit $ERR_CREDENTIAL_PROVIDER_DOWNLOAD_TIMEOUT
-    mv "/usr/local/bin/azure-acr-credential-provider" "$CREDENTIAL_PROVIDER_BIN_DIR/acr-credential-provider"
+    ln -snf /usr/bin/azure-acr-credential-provider "$CREDENTIAL_PROVIDER_BIN_DIR/acr-credential-provider"
 }
 
 installKubeletKubectlPkgFromPMC() {
@@ -204,7 +204,7 @@ installRPMPackageFromFile() {
     if ! dnf_install 30 1 600 ${rpmFile}; then
         exit $ERR_APT_INSTALL_TIMEOUT
     fi
-    mv "/usr/bin/${packageName}" "/usr/local/bin/${packageName}"
+    ln -snf "/usr/bin/${packageName}" "/opt/bin/${packageName}"
 	rm -rf ${downloadDir}
 }
 

parts/linux/cloud-init/artifacts/mariner/mariner-package-update.sh

@@ -8,7 +8,7 @@ set -e
 OS_RELEASE_FILE="/etc/os-release"
 SECURITY_PATCH_REPO_DIR="/etc/yum.repos.d"
 KUBECONFIG="/var/lib/kubelet/kubeconfig"
-KUBECTL="/usr/local/bin/kubectl --kubeconfig ${KUBECONFIG}"
+KUBECTL="/opt/bin/kubectl --kubeconfig ${KUBECONFIG}"
 
 # Function definitions used in this file. 
 # functions defined until "${__SOURCED__:+return}" are sourced and tested in -

parts/linux/cloud-init/artifacts/ubuntu/cse_install_ubuntu.sh

@@ -139,7 +139,7 @@ installCredentialProviderFromPMC() {
     mkdir -p "${CREDENTIAL_PROVIDER_BIN_DIR}"
     chown -R root:root "${CREDENTIAL_PROVIDER_BIN_DIR}"
     installPkgWithAptGet "azure-acr-credential-provider" "${packageVersion}" || exit $ERR_CREDENTIAL_PROVIDER_DOWNLOAD_TIMEOUT
-    mv "/usr/local/bin/azure-acr-credential-provider" "$CREDENTIAL_PROVIDER_BIN_DIR/acr-credential-provider"
+    ln -snf /usr/bin/azure-acr-credential-provider "$CREDENTIAL_PROVIDER_BIN_DIR/acr-credential-provider"
 }
 
 installKubeletKubectlPkgFromPMC() {
@@ -175,7 +175,7 @@ installPkgWithAptGet() {
 
     logs_to_events "AKS.CSE.install${packageName}.installDebPackageFromFile" "installDebPackageFromFile ${debFile}" || exit $ERR_APT_INSTALL_TIMEOUT
 
-    mv "/usr/bin/${packageName}" "/usr/local/bin/${packageName}"
+    ln -snf "/usr/bin/${packageName}" "/opt/bin/${packageName}"
     rm -rf ${downloadDir}
 }