feat: install most components to /opt/bin instead of /usr/local/bin

e2e/kubelet/generate-kubelet-flags.sh

@@ -29,7 +29,7 @@ for KUBE_BINARY_VERSION in $KUBE_BINARY_VERSIONS; do
     K8S_TGZ_TMP=${KUBE_BINARY_URL##*/}
     retrycmd_get_tarball 120 5 "$K8S_DOWNLOADS_DIR/${K8S_TGZ_TMP}" ${KUBE_BINARY_URL} || exit 120
     tar --transform="s|.*|&-${KUBE_BINARY_VERSION}|" --show-transformed-names -xzvf "$K8S_DOWNLOADS_DIR/${K8S_TGZ_TMP}" \
-        --strip-components=3 -C /usr/local/bin kubernetes/node/bin/kubelet kubernetes/node/bin/kubectl
+        --strip-components=3 -C /opt/bin kubernetes/node/bin/kubelet kubernetes/node/bin/kubectl
     rm -f "$K8S_DOWNLOADS_DIR/${K8S_TGZ_TMP}"
     export KUBE_BINARY_VERSION
     pushd e2e || exit 1

e2e/kubelet/main.go

@@ -26,7 +26,7 @@ func run() error {
 	}
 
 	fmt.Println("k8s version is:", k8sVersion)
-	binaryPath := fmt.Sprintf("/usr/local/bin/kubelet-%s", k8sVersion)
+	binaryPath := fmt.Sprintf("/opt/bin/kubelet-%s", k8sVersion)
 
 	r, w := io.Pipe()
 

parts/common/components.json

@@ -934,7 +934,7 @@
     },
     {
       "name": "oras",
-      "downloadLocation": "/usr/local/bin",
+      "downloadLocation": "/opt/bin",
       "downloadURIs": {
         "default": {
           "current": {
@@ -951,7 +951,7 @@
     },
     {
       "name": "aks-secure-tls-bootstrap-client",
-      "downloadLocation": "/usr/local/bin",
+      "downloadLocation": "/opt/bin",
       "windowsDownloadLocation": "c:\\akse-cache\\aks-secure-tls-bootstrap-client\\",
       "downloadURIs": {
         "default": {
@@ -1610,7 +1610,6 @@
     },
     {
       "name": "nvidia-container-toolkit",
-      "downloadLocation": "/usr/local/bin",
       "downloadURIs": {
         "windows": {
           "default": {

parts/linux/cloud-init/artifacts/aks-logrotate.service

@@ -1,4 +1,4 @@
 [Unit]
 Description=runs the logrotate utility for log rotation with a custom configuration
 [Service]
-ExecStart=/usr/local/bin/logrotate.sh
+ExecStart=/opt/bin/logrotate.sh

parts/linux/cloud-init/artifacts/aks-logrotate.sh

@@ -1,6 +1,6 @@
 #!/bin/sh
 # This script was originally generated by logrotate automatically and placed in /etc/cron.daily/logrotate
-# This will be saved on the target VM within /usr/local/bin/logrotate.sh and invoked by logrotate.service
+# This will be saved on the target VM within /opt/bin/logrotate.sh and invoked by logrotate.service
 
 # Clean non existent log file entries from status file
 test -d /var/lib/logrotate || mkdir -p /var/lib/logrotate

parts/linux/cloud-init/artifacts/azlosguard/cse_install_osguard.sh

@@ -13,7 +13,7 @@ installKubeletKubectlPkgFromPMC() {
 installRPMPackageFromFile() {
     local packageName="${1}"
     local desiredVersion="${2}"
-    local targetBinDir="${3:-"/usr/local/bin"}"
+    local targetBinDir="${3:-"/opt/bin"}"
 
     echo "installing ${packageName} version ${desiredVersion} by manually unpacking the RPM"
     if [ "${packageName}" != "kubelet" ] && [ "${packageName}" != "kubectl" ] && [ "${packageName}" != "azure-acr-credential-provider" ]; then
@@ -52,19 +52,8 @@ installRPMPackageFromFile() {
     fi
 
     echo "Unpacking usr/bin/${rpmBinaryName} from ${downloadDir}/${packageName}-${desiredVersion}*"
-    pushd ${downloadDir} || exit 1
-    rpm2cpio "${rpmFile}" | cpio -idmv
-    mkdir -p "${targetBinDir}"
-    if [ -f "usr/bin/${rpmBinaryName}" ]; then
-        mv "usr/bin/${rpmBinaryName}" "${targetBinDir}/${targetBinaryName}"
-    elif [ -f "usr/local/bin/${rpmBinaryName}" ]; then
-        mv "usr/local/bin/${rpmBinaryName}" "${targetBinDir}/${targetBinaryName}"
-    else
-        popd || exit 1
-        rm -rf ${downloadDir}
-        return 1
-    fi
-    popd || exit 1
+    # This assumes that the binary will either be in /usr/bin or /usr/local/bin, but not both.
+    rpm2cpio "${rpmFile}" | cpio -i --to-stdout "./usr/bin/${rpmBinaryName}" "./usr/local/bin/${rpmBinaryName}" | install -m0755 /dev/stdin "${targetBinDir}/${targetBinaryName}"
 	rm -rf ${downloadDir}
 }
 

parts/linux/cloud-init/artifacts/ci-syslog-watcher.service

@@ -3,7 +3,7 @@ Description=Update syslog config based on ContainerInsights syslog status change
 
 [Service]
 Type=oneshot
-ExecStart=/usr/local/bin/ci-syslog-watcher.sh
+ExecStart=/opt/bin/ci-syslog-watcher.sh
 
 [Install]
 WantedBy=multi-user.target

parts/linux/cloud-init/artifacts/cse_helpers.sh

@@ -153,6 +153,9 @@ ERR_PULL_POD_INFRA_CONTAINER_IMAGE=225 # Error pulling pause image
 ERR_AKS_NODE_CONTROLLER_ERROR=240 # Generic error in AKS Node Controller
 # -----------------------------------------------------------------------------
 
+# This probably wasn't launched via a login shell, so ensure the PATH is correct.
+[ -f /etc/profile.d/path.sh ] && . /etc/profile.d/path.sh
+
 # For both Ubuntu and Mariner, /etc/*-release should exist.
 # For unit tests, the OS and OS_VERSION will be set in the unit test script.
 # So whether it's if or else actually doesn't matter to our unit test.
@@ -172,7 +175,7 @@ AZURELINUX_KATA_OS_NAME="AZURELINUXKATA"
 AZURELINUX_OS_NAME="AZURELINUX"
 FLATCAR_OS_NAME="FLATCAR"
 AZURELINUX_OSGUARD_OS_VARIANT="OSGUARD"
-KUBECTL=/usr/local/bin/kubectl
+KUBECTL=/opt/bin/kubectl
 DOCKER=/usr/bin/docker
 # this will be empty during VHD build
 # but vhd build runs with `set -o nounset`
@@ -1235,6 +1238,7 @@ extract_tarball() {
     local tarball="$1"
     local dest="$2"
     shift 2
+    mkdir -p "$dest"
     # Use tar options if provided, otherwise default to -xzf
     case "$tarball" in
         *.tar.gz|*.tgz)

parts/linux/cloud-init/artifacts/cse_install.sh

@@ -7,7 +7,7 @@ CNI_BIN_DIR="/opt/cni/bin"
 #TODO pull this out of componetns.json too?
 CNI_DOWNLOADS_DIR="/opt/cni/downloads"
 CRICTL_DOWNLOAD_DIR="/opt/crictl/downloads"
-CRICTL_BIN_DIR="/usr/local/bin"
+CRICTL_BIN_DIR="/opt/bin"
 CONTAINERD_DOWNLOADS_DIR="/opt/containerd/downloads"
 RUNC_DOWNLOADS_DIR="/opt/runc/downloads"
 K8S_DOWNLOADS_DIR="/opt/kubernetes/downloads"
@@ -18,11 +18,11 @@ UBUNTU_RELEASE=$(lsb_release -r -s 2>/dev/null || echo "")
 OS=$(if ls /etc/*-release 1> /dev/null 2>&1; then sort -r /etc/*-release | gawk 'match($0, /^(ID=(.*))$/, a) { print toupper(a[2]); exit }'; fi)
 OS_VARIANT=$(if ls /etc/*-release 1> /dev/null 2>&1; then sort -r /etc/*-release | gawk 'match($0, /^(VARIANT_ID=(.*))$/, a) { print toupper(a[2]); exit }' | tr -d '"'; fi)
 SECURE_TLS_BOOTSTRAP_CLIENT_DOWNLOAD_DIR="/opt/aks-secure-tls-bootstrap-client/downloads"
-SECURE_TLS_BOOTSTRAP_CLIENT_BIN_DIR="/usr/local/bin"
+SECURE_TLS_BOOTSTRAP_CLIENT_BIN_DIR="/opt/bin"
 TELEPORTD_PLUGIN_DOWNLOAD_DIR="/opt/teleportd/downloads"
 CREDENTIAL_PROVIDER_DOWNLOAD_DIR="/opt/credentialprovider/downloads"
 CREDENTIAL_PROVIDER_BIN_DIR="/var/lib/kubelet/credential-provider"
-TELEPORTD_PLUGIN_BIN_DIR="/usr/local/bin"
+TELEPORTD_PLUGIN_BIN_DIR="/opt/bin"
 MANIFEST_FILEPATH="/opt/azure/manifest.json"
 COMPONENTS_FILEPATH="/opt/azure/components.json"
 VHD_LOGS_FILEPATH="/opt/azure/vhd-install.complete"
@@ -201,7 +201,7 @@ installCredentialProviderFromUrl() {
 # TODO (alburgess) have oras version managed by dependant or Renovate
 installOras() {
     ORAS_DOWNLOAD_DIR="/opt/oras/downloads"
-    ORAS_EXTRACTED_DIR=${1} # Use components.json var for /usr/local/bin for linux-vhd-content-test.sh binary file checks.
+    ORAS_EXTRACTED_DIR=${1} # Use components.json var for /opt/bin for linux-vhd-content-test.sh binary file checks.
     ORAS_DOWNLOAD_URL=${2}
     ORAS_VERSION=${3}
 
@@ -459,15 +459,15 @@ installAzureCNI() {
 }
 
 # extract the cached or downloaded kube package and remove
-extractKubeBinariesToUsrLocalBin() {
+extractKubeBinariesToOptBin() {
     local k8s_tgz_tmp=$1
     local k8s_version=$2
     local is_private_url=$3
 
-    extract_tarball "${k8s_tgz_tmp}" "/usr/local/bin" \
+    extract_tarball "${k8s_tgz_tmp}" "/opt/bin" \
         --transform="s|.*|&-${k8s_version}|" --show-transformed-names --strip-components=3 \
         kubernetes/node/bin/kubelet kubernetes/node/bin/kubectl || exit $ERR_K8S_INSTALL_ERR
-    if [ ! -f "/usr/local/bin/kubectl-${k8s_version}" ] || [ ! -f "/usr/local/bin/kubelet-${k8s_version}" ]; then
+    if [ ! -f "/opt/bin/kubectl-${k8s_version}" ] || [ ! -f "/opt/bin/kubelet-${k8s_version}" ]; then
         exit $ERR_K8S_INSTALL_ERR
     fi
     if [ "$is_private_url" = "false" ]; then
@@ -500,7 +500,7 @@ extractKubeBinaries() {
 
         echo "cached package ${k8s_tgz_tmp} found, will extract that"
         # remove the current kubelet and kubectl binaries before extracting new binaries from the cached package
-        rm -rf /usr/local/bin/kubelet-* /usr/local/bin/kubectl-*
+        rm -rf /opt/bin/kubelet-* /opt/bin/kubectl-*
     else
         k8s_tgz_tmp="${k8s_downloads_dir}/${k8s_tgz_tmp_filename}"
         mkdir -p ${k8s_downloads_dir}
@@ -523,7 +523,7 @@ extractKubeBinaries() {
         fi
     fi
 
-    extractKubeBinariesToUsrLocalBin "${k8s_tgz_tmp}" "${k8s_version}" "${is_private_url}"
+    extractKubeBinariesToOptBin "${k8s_tgz_tmp}" "${k8s_version}" "${is_private_url}"
 }
 
 installToolFromBootstrapProfileRegistry() {
@@ -579,7 +579,7 @@ installKubeletKubectlFromBootstrapProfileRegistry() {
     local registry_server=$1
     local kubernetes_version=$2
     for tool_name in $(get_kubernetes_tools); do
-        install_path="/usr/local/bin/${tool_name}"
+        install_path="/opt/bin/${tool_name}"
         if ! installToolFromBootstrapProfileRegistry "${tool_name}" "${registry_server}" "${kubernetes_version}" "${install_path}"; then
             # SHOULD_ENFORCE_KUBE_PMC_INSTALL will only be set for e2e tests, which should not fallback to reflect result of package installation behavior
             # TODO: remove SHOULD_ENFORCE_KUBE_PMC_INSTALL check when the test cluster supports > 1.34.0 case
@@ -604,7 +604,7 @@ installKubeletKubectlFromURL() {
 
     if [ ! -z "${CUSTOM_KUBE_BINARY_DOWNLOAD_URL}" ]; then
         # remove the kubelet and kubectl binaries to make sure the only binary left is from the CUSTOM_KUBE_BINARY_DOWNLOAD_URL
-        rm -rf /usr/local/bin/kubelet-* /usr/local/bin/kubectl-*
+        rm -rf /opt/bin/kubelet-* /opt/bin/kubectl-*
 
         # NOTE(mainred): we expect kubelet binary to be under `kubernetes/node/bin`. This suits the current setting of
         # kube binaries used by AKS and Kubernetes upstream.
@@ -617,7 +617,7 @@ installKubeletKubectlFromURL() {
     fi
 
     # if the custom url is not specified and the required kubectl/kubelet-version via private url is not installed, install using the default url/package
-    if [ ! -f "/usr/local/bin/kubectl-${KUBERNETES_VERSION}" ] || [ ! -f "/usr/local/bin/kubelet-${KUBERNETES_VERSION}" ]; then
+    if [ ! -f "/opt/bin/kubectl-${KUBERNETES_VERSION}" ] || [ ! -f "/opt/bin/kubelet-${KUBERNETES_VERSION}" ]; then
         if [ "$install_default_if_missing" = "true" ]; then
             if [ -n "${BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER}" ]; then
                 # network isolated cluster
@@ -635,11 +635,10 @@ installKubeletKubectlFromURL() {
             fi
         fi
     fi
-    mv "/usr/local/bin/kubelet-${KUBERNETES_VERSION}" "/usr/local/bin/kubelet"
-    mv "/usr/local/bin/kubectl-${KUBERNETES_VERSION}" "/usr/local/bin/kubectl"
+    install -m0755 "/opt/bin/kubelet-${KUBERNETES_VERSION}" /opt/bin/kubelet
+    install -m0755 "/opt/bin/kubectl-${KUBERNETES_VERSION}" /opt/bin/kubectl
 
-    chmod a+x /usr/local/bin/kubelet /usr/local/bin/kubectl
-    rm -rf /usr/local/bin/kubelet-* /usr/local/bin/kubectl-* /home/hyperkube-downloads &
+    rm -rf /opt/bin/kubelet-* /opt/bin/kubectl-* /home/hyperkube-downloads &
 }
 
 pullContainerImage() {

parts/linux/cloud-init/artifacts/kubelet.service

@@ -1,6 +1,6 @@
 [Unit]
 Description=Kubelet
-ConditionPathExists=/usr/local/bin/kubelet
+ConditionPathExists=/opt/bin/kubelet
 Wants=network-online.target containerd.service
 After=network-online.target containerd.service
 
@@ -22,7 +22,7 @@ ExecStartPre=-/sbin/iptables -t nat --numeric --list
 
 ExecStartPre=/bin/bash /opt/azure/containers/validate-kubelet-credentials.sh
 
-ExecStart=/usr/local/bin/kubelet \
+ExecStart=/opt/bin/kubelet \
         --enable-server \
         --node-labels="${KUBELET_NODE_LABELS}" \
         --v=2 \

parts/linux/cloud-init/artifacts/mariner/cse_install_mariner.sh

@@ -189,7 +189,7 @@ installCredentialProviderFromPMC() {
     mkdir -p "${CREDENTIAL_PROVIDER_BIN_DIR}"
     chown -R root:root "${CREDENTIAL_PROVIDER_BIN_DIR}"
     installRPMPackageFromFile "azure-acr-credential-provider" "${packageVersion}" || exit $ERR_CREDENTIAL_PROVIDER_DOWNLOAD_TIMEOUT
-    mv "/usr/local/bin/azure-acr-credential-provider" "$CREDENTIAL_PROVIDER_BIN_DIR/acr-credential-provider"
+    ln -snf /usr/bin/azure-acr-credential-provider "$CREDENTIAL_PROVIDER_BIN_DIR/acr-credential-provider"
 }
 
   getPackageCacheRoot() {
@@ -445,7 +445,8 @@ installRPMPackageFromFile() {
     if ! dnf_install 30 1 600 "${rpmArgs[@]}"; then
         exit $ERR_APT_INSTALL_TIMEOUT
     fi
-    mv "/usr/bin/${packageName}" "/usr/local/bin/${packageName}"
+    mkdir -p /opt/bin
+    ln -snf "/usr/bin/${packageName}" "/opt/bin/${packageName}"
     rm -rf "${downloadDir}"
 }
 

parts/linux/cloud-init/artifacts/mariner/mariner-package-update.sh

@@ -8,7 +8,7 @@ set -e
 OS_RELEASE_FILE="/etc/os-release"
 SECURITY_PATCH_REPO_DIR="/etc/yum.repos.d"
 KUBECONFIG="/var/lib/kubelet/kubeconfig"
-KUBECTL="/usr/local/bin/kubectl --kubeconfig ${KUBECONFIG}"
+KUBECTL="/opt/bin/kubectl --kubeconfig ${KUBECONFIG}"
 KUBELET_EXECUTABLE="/usr/local/bin/kubelet"
 SECURITY_PATCH_TMP_DIR="/tmp/security-patch"
 

parts/linux/cloud-init/artifacts/profile-d-path.sh

@@ -0,0 +1,6 @@
+#!/bin/sh
+
+case "${PATH}" in
+    /opt/bin:*) : ;;
+    *) PATH=/opt/bin:${PATH} ;;
+esac

parts/linux/cloud-init/artifacts/secure-tls-bootstrap.service

@@ -1,14 +1,14 @@
 [Unit]
 Description=AKS Secure TLS Bootstrap Client
-ConditionPathExists=/usr/local/bin/aks-secure-tls-bootstrap-client
+ConditionPathExists=/opt/bin/aks-secure-tls-bootstrap-client
 Wants=network-online.target
 After=network-online.target
 
 [Service]
 Type=oneshot
 RemainAfterExit=yes
 
-ExecStart=/usr/local/bin/aks-secure-tls-bootstrap-client \
+ExecStart=/opt/bin/aks-secure-tls-bootstrap-client \
     --verbose \
     --ensure-authorized \
     --next-proto=aks-tls-bootstrap \

parts/linux/cloud-init/artifacts/teleportd.service

@@ -2,7 +2,7 @@
 Description=teleportd teleport runtime
 After=network.target
 [Service]
-ExecStart=/usr/local/bin/teleportd --metrics --aksConfig /etc/kubernetes/azure.json
+ExecStart=/opt/bin/teleportd --metrics --aksConfig /etc/kubernetes/azure.json
 Delegate=yes
 KillMode=process
 Restart=always

parts/linux/cloud-init/artifacts/ubuntu/cse_install_ubuntu.sh

@@ -202,7 +202,7 @@ installCredentialProviderFromPMC() {
     mkdir -p "${CREDENTIAL_PROVIDER_BIN_DIR}"
     chown -R root:root "${CREDENTIAL_PROVIDER_BIN_DIR}"
     installPkgWithAptGet "azure-acr-credential-provider" "${packageVersion}" || exit $ERR_CREDENTIAL_PROVIDER_DOWNLOAD_TIMEOUT
-    mv "/usr/local/bin/azure-acr-credential-provider" "$CREDENTIAL_PROVIDER_BIN_DIR/acr-credential-provider"
+    ln -snf /usr/bin/azure-acr-credential-provider "$CREDENTIAL_PROVIDER_BIN_DIR/acr-credential-provider"
 }
 
 installKubeletKubectlPkgFromPMC() {
@@ -307,7 +307,8 @@ installPkgWithAptGet() {
 
     logs_to_events "AKS.CSE.install${packageName}.installDebPackageFromFile" "installDebPackageFromFile ${debFile}" || exit $ERR_APT_INSTALL_TIMEOUT
 
-    mv "/usr/bin/${packageName}" "/usr/local/bin/${packageName}"
+    mkdir -p /opt/bin
+    ln -snf "/usr/bin/${packageName}" "/opt/bin/${packageName}"
     rm -rf ${downloadDir}
 }
 

parts/linux/cloud-init/artifacts/ubuntu/ubuntu-snapshot-update.sh

@@ -7,7 +7,7 @@ set -e
 # -------------------------------------------------------------------------------------------------
 SECURITY_PATCH_CONFIG_DIR=/var/lib/security-patch
 KUBECONFIG="/var/lib/kubelet/kubeconfig"
-KUBECTL="/usr/local/bin/kubectl --kubeconfig ${KUBECONFIG}"
+KUBECTL="/opt/bin/kubectl --kubeconfig ${KUBECONFIG}"
 DEFAULT_ENDPOINT="snapshot.ubuntu.com"
 
 # Function definitions used in this file. 

parts/linux/cloud-init/flatcar.yml

@@ -1,2 +1,20 @@
 variant: flatcar
 version: 1.1.0
+systemd:
+  units:
+    - name: ignition-bootcmds.service
+      enabled: true
+      contents: |
+        [Unit]
+        Description=Ignition Early Boot Commands
+        DefaultDependencies=no
+        After=local-fs.target
+        Before=sysinit.target
+        ConditionPathExists=/etc/ignition-bootcmds.sh
+
+        [Service]
+        Type=oneshot
+        ExecStart=-/etc/ignition-bootcmds.sh
+
+        [Install]
+        WantedBy=sysinit.target

parts/linux/cloud-init/nodecustomdata.yml

@@ -1,5 +1,14 @@
 #cloud-config
 
+bootcmd:
+- |
+  mkdir -p /opt/bin
+  for bin in aks-secure-tls-bootstrap-client ci-syslog-watcher.sh logrotate.sh teleportd; do
+    [ -e /opt/bin/${bin} ] && continue
+    [ -e /usr/local/bin/${bin} ] || continue
+    ln -s /usr/local/bin/${bin} /opt/bin/
+  done
+
 write_files:
 - path: {{GetCSEHelpersScriptFilepath}}
   permissions: "0744"