refactor(stls): augment secure TLS bootstrapping configuration options in NodeBootstrappingConfiguration/AKSNodeConfig (#7112)

Author: cameronmeissner

aks-node-controller/parser/helper.go

@@ -202,19 +202,11 @@ func getCustomCACertsStatus(customCACerts []string) bool {
 	return len(customCACerts) > 0
 }
 
-func getEnableSecureTLSBootstrap(bootstrapConfig *aksnodeconfigv1.BootstrappingConfig) bool {
-	// TODO: Change logic to default to false once Secure TLS Bootstrapping is complete
+func getEnableSecureTLSBootstrapping(bootstrapConfig *aksnodeconfigv1.BootstrappingConfig) bool {
+	// TODO: Change logic to default to true once Secure TLS Bootstrapping is complete
 	return bootstrapConfig.GetBootstrappingAuthMethod() == aksnodeconfigv1.BootstrappingAuthMethod_BOOTSTRAPPING_AUTH_METHOD_SECURE_TLS_BOOTSTRAPPING
 }
 
-func getTLSBootstrapToken(bootstrapConfig *aksnodeconfigv1.BootstrappingConfig) string {
-	return bootstrapConfig.GetTlsBootstrappingToken()
-}
-
-func getCustomSecureTLSBootstrapAADServerAppID(bootstrapConfig *aksnodeconfigv1.BootstrappingConfig) string {
-	return bootstrapConfig.GetCustomAadResource()
-}
-
 func getEnsureNoDupePromiscuousBridge(nc *aksnodeconfigv1.NetworkConfig) bool {
 	return nc.GetNetworkPlugin() == aksnodeconfigv1.NetworkPlugin_NETWORK_PLUGIN_KUBENET && nc.GetNetworkPolicy() != aksnodeconfigv1.NetworkPolicy_NETWORK_POLICY_CALICO
 }

aks-node-controller/parser/parser.go

@@ -436,6 +436,12 @@ func TestAKSNodeConfigCompatibilityFromJsonToCSECommand(t *testing.T) {
 				assert.Equal(t, "", vars["NO_PROXY"])
 				assert.Equal(t, "", vars["PROXY_TRUSTED_CA"])
 				assert.Equal(t, helpers.DefaultCloudName, vars["TARGET_ENVIRONMENT"])
+				assert.Equal(t, "", vars["TLS_BOOTSTRAP_TOKEN"])
+				assert.Equal(t, "false", vars["ENABLE_SECURE_TLS_BOOTSTRAPPING"])
+				assert.Equal(t, "", vars["SECURE_TLS_BOOTSTRAPPING_DEADLINE"])
+				assert.Equal(t, "", vars["SECURE_TLS_BOOTSTRAPPING_AAD_RESOURCE"])
+				assert.Equal(t, "", vars["SECURE_TLS_BOOTSTRAPPING_USER_ASSIGNED_IDENTITY_ID"])
+				assert.Equal(t, "", vars["CUSTOM_SECURE_TLS_BOOTSTRAPPING_CLIENT_URL"])
 			},
 		},
 	}

aks-node-controller/parser/parser_test.go

@@ -41,9 +41,22 @@ message BootstrappingConfig {
   // Only required until Secure TLS bootstrapping in place. Would use kubelet identity after that.
   optional string tls_bootstrapping_token = 3;
 
-  // Only used when secure TLS bootstrapping is enabled or one of the Azure/Arc methods. This is the appserver appid that the node will use to bootstrap.
-  optional string custom_aad_resource = 4;
+  reserved 4;
+  reserved "custom_aad_resource";
+  reserved 5;
+  reserved "custom_aad_client_id";
 
-  // Only used when one of the Azure/Arc methods is enabled. This is the client ID of the MSI that the node will use to bootstrap.
-  optional string custom_aad_client_id = 5;
+  // Only used when secure TLS bootstrapping is enabled. This is the AAD resource used to request access tokens from Entra ID.
+  optional string secure_tls_bootstrapping_aad_resource = 6;
+
+  // Only used when secure TLS bootstrapping is enabled. This is the client ID of the user-assigned identity ID the node will use to perform secure TLS bootstrapping.
+  optional string secure_tls_bootstrapping_user_assigned_identity_id = 7;
+
+  // Only used when secure TLS bootstrapping is enabled. If specified, the bootstrap client installation will be replaced with the client version downloaded from this URL.
+  optional string secure_tls_bootstrapping_custom_client_download_url = 8;
+
+  // Only used when secure TLS bootstrapping is enabled. This is the bootstrapping deadline used to perform secure TLS bootstrapping.
+  // If the deadline is hit, the TLS bootstrap token will be used as a fall-back.
+  // This field will be deprecated once TLS bootstrap tokens are no longer used.
+  optional string secure_tls_bootstrapping_deadline = 9;
 }

aks-node-controller/pkg/gen/aksnodeconfig/v1/bootstrapping_config.pb.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"testing"
+	"time"
 
 	aksnodeconfigv1 "github.com/Azure/agentbaker/aks-node-controller/pkg/gen/aksnodeconfig/v1"
 	"github.com/Azure/agentbaker/e2e/components"
@@ -139,8 +140,10 @@ func Test_AzureLinuxV2_SecureTLSBootstrapping_BootstrapToken_Fallback(t *testing
 			Cluster: ClusterKubenet,
 			VHD:     config.VHDAzureLinuxV2Gen2,
 			BootstrapConfigMutator: func(nbc *datamodel.NodeBootstrappingConfiguration) {
-				// secure TLS bootstrapping is not yet enabled in e2e regions, thus this will test the bootstrap token fallback case
-				nbc.EnableSecureTLSBootstrapping = true
+				nbc.SecureTLSBootstrappingConfig = &datamodel.SecureTLSBootstrappingConfig{
+					Enabled:  true,
+					Deadline: (30 * time.Second).String(),
+				}
 			},
 		},
 	})
@@ -1745,8 +1748,10 @@ func Test_Ubuntu2404Gen2_SecureTLSBootstrapping_BootstrapToken_Fallback(t *testi
 			Cluster: ClusterKubenet,
 			VHD:     config.VHDUbuntu2404Gen2Containerd,
 			BootstrapConfigMutator: func(nbc *datamodel.NodeBootstrappingConfiguration) {
-				// secure TLS bootstrapping is not yet enabled in e2e regions, thus this will test the bootstrap token fallback case
-				nbc.EnableSecureTLSBootstrapping = true
+				nbc.SecureTLSBootstrappingConfig = &datamodel.SecureTLSBootstrappingConfig{
+					Enabled:  true,
+					Deadline: (30 * time.Second).String(),
+				}
 			},
 		},
 	})

aks-node-controller/proto/aksnodeconfig/v1/bootstrapping_config.proto

@@ -117,8 +117,10 @@ HTTPS_PROXY_URLS="{{GetHTTPSProxy}}"
 NO_PROXY_URLS="{{GetNoProxy}}"
 PROXY_VARS="{{GetProxyVariables}}"
 ENABLE_SECURE_TLS_BOOTSTRAPPING="{{EnableSecureTLSBootstrapping}}"
-CUSTOM_SECURE_TLS_BOOTSTRAP_AAD_SERVER_APP_ID="{{GetCustomSecureTLSBootstrapAADServerAppID}}"
-CUSTOM_SECURE_TLS_BOOTSTRAP_CLIENT_URL="{{GetCustomSecureTLSBootstrapClientURL}}"
+SECURE_TLS_BOOTSTRAPPING_DEADLINE="{{GetSecureTLSBootstrappingDeadline}}"
+SECURE_TLS_BOOTSTRAPPING_AAD_RESOURCE="{{GetSecureTLSBootstrappingAADResource}}"
+SECURE_TLS_BOOTSTRAPPING_USER_ASSIGNED_IDENTITY_ID="{{GetSecureTLSBootstrappingUserAssignedIdentityID}}"
+CUSTOM_SECURE_TLS_BOOTSTRAPPING_CLIENT_DOWNLOAD_URL="{{GetCustomSecureTLSBootstrappingClientDownloadURL}}"
 ENABLE_KUBELET_SERVING_CERTIFICATE_ROTATION="{{EnableKubeletServingCertificateRotation}}"
 DHCPV6_SERVICE_FILEPATH="{{GetDHCPv6ServiceCSEScriptFilepath}}"
 DHCPV6_CONFIG_FILEPATH="{{GetDHCPv6ConfigCSEScriptFilepath}}"
@@ -166,4 +168,4 @@ MCR_REPOSITORY_BASE="{{GetMCRRepositoryBase}}"
 ENABLE_IMDS_RESTRICTION="{{EnableIMDSRestriction}}"
 INSERT_IMDS_RESTRICTION_RULE_TO_MANGLE_TABLE="{{InsertIMDSRestrictionRuleToMangleTable}}"
 PRE_PROVISION_ONLY="{{GetPreProvisionOnly}}"
-/usr/bin/nohup /bin/bash -c "/bin/bash /opt/azure/containers/provision_start.sh"
+/usr/bin/nohup /bin/bash -c "/bin/bash /opt/azure/containers/provision_start.sh"

e2e/scenario_test.go

@@ -534,14 +534,19 @@ ensureKubeCACert() {
 # drop-in path defined outside so configureAndStartSecureTLSBootstrapping can be unit tested
 SECURE_TLS_BOOTSTRAPPING_DROP_IN="/etc/systemd/system/secure-tls-bootstrap.service.d/10-securetlsbootstrap.conf"
 configureAndStartSecureTLSBootstrapping() {
+    BOOTSTRAP_CLIENT_FLAGS="--deadline=${SECURE_TLS_BOOTSTRAPPING_DEADLINE:-"2m0s"} --aad-resource=${SECURE_TLS_BOOTSTRAPPING_AAD_RESOURCE:-$AKS_AAD_SERVER_APP_ID} --apiserver-fqdn=${API_SERVER_NAME} --cloud-provider-config=${AZURE_JSON_PATH}"
+    if [ -n "${SECURE_TLS_BOOTSTRAPPING_USER_ASSIGNED_IDENTITY_ID}" ]; then
+        BOOTSTRAP_CLIENT_FLAGS="${BOOTSTRAP_CLIENT_FLAGS} --user-assigned-identity-id=$SECURE_TLS_BOOTSTRAPPING_USER_ASSIGNED_IDENTITY_ID"
+    fi
+
     mkdir -p "$(dirname "${SECURE_TLS_BOOTSTRAPPING_DROP_IN}")"
     touch "${SECURE_TLS_BOOTSTRAPPING_DROP_IN}"
     chmod 0600 "${SECURE_TLS_BOOTSTRAPPING_DROP_IN}"
     cat > "${SECURE_TLS_BOOTSTRAPPING_DROP_IN}" <<EOF
 [Unit]
 Before=kubelet.service
 [Service]
-Environment="BOOTSTRAP_FLAGS=--aad-resource=${CUSTOM_SECURE_TLS_BOOTSTRAP_AAD_SERVER_APP_ID:-$AKS_AAD_SERVER_APP_ID} --apiserver-fqdn=${API_SERVER_NAME} --cloud-provider-config=${AZURE_JSON_PATH}"
+Environment="BOOTSTRAP_FLAGS=${BOOTSTRAP_CLIENT_FLAGS}"
 [Install]
 # once bootstrap tokens are no longer a fallback, kubelet.service needs to be a RequiredBy=
 WantedBy=kubelet.service

parts/linux/cloud-init/artifacts/cse_cmd.sh

@@ -57,7 +57,7 @@ installContainerdWithComponentsJson() {
     else
         os_version="${UBUNTU_RELEASE}"
     fi
-    
+
     containerdPackage=$(jq ".Packages" "$COMPONENTS_FILEPATH" | jq ".[] | select(.name == \"containerd\")") || exit $ERR_CONTAINERD_VERSION_INVALID
     PACKAGE_VERSIONS=()
     if isMariner "${OS}" && [ "${IS_KATA}" = "true" ]; then
@@ -67,7 +67,7 @@ installContainerdWithComponentsJson() {
         os=${AZURELINUX_KATA_OS_NAME}
     fi
     updatePackageVersions "${containerdPackage}" "${os}" "${os_version}"
-    
+
     #Containerd's versions array is expected to have only one element.
     #If it has more than one element, we will install the last element in the array.
     # shellcheck disable=SC3010
@@ -103,7 +103,7 @@ installContainerdWithComponentsJson() {
 }
 
 # containerd versions definitions are only available in the manifest file before the centralized packages changes, before around early July 2024.
-# After the centralized packages changes, the containerd versions are only available in the components.json. 
+# After the centralized packages changes, the containerd versions are only available in the components.json.
 installContainerdWithManifestJson() {
     local containerd_version
     if [ -f "$MANIFEST_FILEPATH" ]; then
@@ -144,16 +144,16 @@ installNetworkPlugin() {
         installAzureCNI
     fi
     installCNI #reference plugins. Mostly for kubenet but loopback plugin is used by containerd until containerd 2
-    rm -rf $CNI_DOWNLOADS_DIR & 
+    rm -rf $CNI_DOWNLOADS_DIR &
 }
 
-# downloadCredentialProvider is always called during build time by install-dependencies.sh. 
+# downloadCredentialProvider is always called during build time by install-dependencies.sh.
 # It can also be called during node provisioning by cse_config.sh, meaning CREDENTIAL_PROVIDER_DOWNLOAD_URL is set by a passed in linuxCredentialProviderURL.
 downloadCredentialProvider() {
     CREDENTIAL_PROVIDER_DOWNLOAD_URL="${CREDENTIAL_PROVIDER_DOWNLOAD_URL:=}"
     if [ -n "${CREDENTIAL_PROVIDER_DOWNLOAD_URL}" ]; then
         # CREDENTIAL_PROVIDER_DOWNLOAD_URL is set by linuxCredentialProviderURL
-        # The version in the URL is unknown. An acs-mirror or registry URL could be passed meaning the version must be extracted from the URL. 
+        # The version in the URL is unknown. An acs-mirror or registry URL could be passed meaning the version must be extracted from the URL.
         cred_version_for_oras=$(echo "$CREDENTIAL_PROVIDER_DOWNLOAD_URL" | grep -oP 'v\d+(\.\d+)*' | sed 's/^v//' | head -n 1)
     fi
 
@@ -175,7 +175,7 @@ downloadCredentialProvider() {
         local credential_provider_download_url_for_oras="${BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER}/${K8S_REGISTRY_REPO}/azure-acr-credential-provider:v${cred_version_for_oras}-linux-${CPU_ARCH}"
         CREDENTIAL_PROVIDER_TGZ_TMP="${CREDENTIAL_PROVIDER_DOWNLOAD_URL##*/}" # Use bash builtin ## to remove all chars ("*") up to the final "/"
         retrycmd_get_tarball_from_registry_with_oras 120 5 "$CREDENTIAL_PROVIDER_DOWNLOAD_DIR/$CREDENTIAL_PROVIDER_TGZ_TMP" "${credential_provider_download_url_for_oras}" || exit $ERR_ORAS_PULL_CREDENTIAL_PROVIDER
-        return 
+        return
     elif isRegistryUrl "${CREDENTIAL_PROVIDER_DOWNLOAD_URL}"; then
         # if the URL is a registry URL, then download the credential provider using oras
         # extract version v1.30.0 from format like mcr.microsoft.com/oss/binaries/kubernetes/azure-acr-credential-provider:v1.30.0-linux-amd64
@@ -242,12 +242,12 @@ installSecureTLSBootstrapClient() {
     # without having to tag new versions of AgentBaker, in the end we probably won't honor custom URLs specified
     # by the bootstrapper for this particular binary. In the end, if we do decide to support this, we will need
     # to make sure to use oras to download the client binary and ensure the binary itself is hosted within MCR.
-    if [ -z "${CUSTOM_SECURE_TLS_BOOTSTRAP_CLIENT_URL}" ]; then
-        echo "secure TLS bootstrapping is enabled but no custom client URL was provided, nothing to download"
+    if [ -z "${CUSTOM_SECURE_TLS_BOOTSTRAPPING_CLIENT_DOWNLOAD_URL}" ]; then
+        echo "secure TLS bootstrapping is enabled but no custom client download URL was provided, nothing to download"
         return 0
     fi
 
-    downloadSecureTLSBootstrapClient "${SECURE_TLS_BOOTSTRAP_CLIENT_BIN_DIR}" "${CUSTOM_SECURE_TLS_BOOTSTRAP_CLIENT_URL}" || exit $ERR_SECURE_TLS_BOOTSTRAP_CLIENT_DOWNLOAD_ERROR
+    downloadSecureTLSBootstrapClient "${SECURE_TLS_BOOTSTRAP_CLIENT_BIN_DIR}" "${CUSTOM_SECURE_TLS_BOOTSTRAPPING_CLIENT_DOWNLOAD_URL}" || exit $ERR_SECURE_TLS_BOOTSTRAP_CLIENT_DOWNLOAD_ERROR
 }
 
 downloadSecureTLSBootstrapClient() {
@@ -290,7 +290,7 @@ evalPackageDownloadURL() {
 
 downloadAzureCNI() {
     mkdir -p ${1-$:CNI_DOWNLOADS_DIR}
-    # At VHD build time, the VNET_CNI_PLUGINS_URL is usually not set. 
+    # At VHD build time, the VNET_CNI_PLUGINS_URL is usually not set.
     # So, we will get the URL passed from install-depenencies.sh which is actually from components.json
     # At node provisioning time, if AKS-RP sets the VNET_CNI_PLUGINS_URL, then we will use that.
     VNET_CNI_PLUGINS_URL=${2:-$VNET_CNI_PLUGINS_URL}
@@ -389,24 +389,24 @@ setupCNIDirs() {
 
 # Reference CNI plugins is used by kubenet and the loopback plugin used by containerd 1.0 (dependency gone in 2.0)
 # The version used to be deteremined by RP/toggle but are now just hadcoded in vhd as they rarely change and require a node image upgrade anyways
-# Latest VHD should have the untar, older should have the tgz. And who knows will have neither. 
+# Latest VHD should have the untar, older should have the tgz. And who knows will have neither.
 installCNI() {
     # Old versions of VHDs will not have components.json. If it does not exist, we will fall back to the hardcoded download for CNI.
     # Network Isolated Cluster / Bring Your Own ACR will not work with a vhd that requres a hardcoded CNI download.
     if [ ! -f "$COMPONENTS_FILEPATH" ] || ! jq '.Packages[] | select(.name == "cni-plugins")' < $COMPONENTS_FILEPATH > /dev/null; then
-        echo "WARNING: no cni-plugins components present falling back to hard coded download of 1.4.1. This should error eventually" 
+        echo "WARNING: no cni-plugins components present falling back to hard coded download of 1.4.1. This should error eventually"
         # could we fail if not Ubuntu2204Gen2ContainerdPrivateKubePkg vhd? Are there others?
         # definitely not handling arm here.
         retrycmd_get_tarball 120 5 "${CNI_DOWNLOADS_DIR}/refcni.tar.gz" "https://${PACKAGE_DOWNLOAD_BASE_URL}/cni-plugins/v1.4.1/binaries/cni-plugins-linux-amd64-v1.4.1.tgz" || exit $ERR_CNI_DOWNLOAD_TIMEOUT
         extract_tarball "${CNI_DOWNLOADS_DIR}/refcni.tar.gz" "$CNI_BIN_DIR"
-        return 
+        return
     fi
 
     #always just use what is listed in components.json so we don't have to sync.
     cniPackage=$(jq ".Packages" "$COMPONENTS_FILEPATH" | jq ".[] | select(.name == \"cni-plugins\")") || exit $ERR_CNI_VERSION_INVALID
-    
+
     #CNI doesn't really care about this but wanted to reuse updatePackageVersions which requires it.
-    os=${UBUNTU_OS_NAME} 
+    os=${UBUNTU_OS_NAME}
     if [ -z "$UBUNTU_RELEASE" ]; then
         os=${OS}
         os_version="current"
@@ -417,7 +417,7 @@ installCNI() {
     fi
     PACKAGE_VERSIONS=()
     updatePackageVersions "${cniPackage}" "${os}" "${os_version}"
-    
+
     #should change to ne
     # shellcheck disable=SC3010
     if [[ ${#PACKAGE_VERSIONS[@]} -gt 1 ]]; then
@@ -427,15 +427,15 @@ installCNI() {
     packageVersion=${PACKAGE_VERSIONS[0]}
 
     # Is there a ${arch} variable I can use instead of the iff
-    if [ "$(isARM64)" -eq 1 ]; then 
+    if [ "$(isARM64)" -eq 1 ]; then
         CNI_DIR_TMP="cni-plugins-linux-arm64-v${packageVersion}"
-    else 
+    else
         CNI_DIR_TMP="cni-plugins-linux-amd64-v${packageVersion}"
     fi
-    
+
     if [ -d "$CNI_DOWNLOADS_DIR/${CNI_DIR_TMP}" ]; then
-        #not clear to me when this would ever happen. assume its related to the line above Latest VHD should have the untar, older should have the tgz. 
-        mv ${CNI_DOWNLOADS_DIR}/${CNI_DIR_TMP}/* $CNI_BIN_DIR 
+        #not clear to me when this would ever happen. assume its related to the line above Latest VHD should have the untar, older should have the tgz.
+        mv ${CNI_DOWNLOADS_DIR}/${CNI_DIR_TMP}/* $CNI_BIN_DIR
     else
         echo "CNI tarball should already be unzipped by components.json"
         exit $ERR_CNI_VERSION_INVALID
@@ -507,7 +507,7 @@ extractKubeBinaries() {
     else
         k8s_tgz_tmp="${k8s_downloads_dir}/${k8s_tgz_tmp_filename}"
         mkdir -p ${k8s_downloads_dir}
-        
+
         # if the url is a registry url, use oras to pull the artifact instead of curl
         if isRegistryUrl "${kube_binary_url}"; then
             echo "detect kube_binary_url, ${kube_binary_url}, as registry url, will use oras to pull artifact binary"
@@ -553,11 +553,11 @@ installKubeletKubectlFromURL() {
     # if the custom url is not specified and the required kubectl/kubelet-version via private url is not installed, install using the default url/package
     if [ ! -f "/usr/local/bin/kubectl-${KUBERNETES_VERSION}" ] || [ ! -f "/usr/local/bin/kubelet-${KUBERNETES_VERSION}" ]; then
         if [ "$install_default_if_missing" = "true" ]; then
-            if [ -n "${BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER}" ]; then 
+            if [ -n "${BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER}" ]; then
                 # network isolated cluster
                 echo "Detect Bootstrap profile artifact is Cache, will use oras to pull artifact binary"
                 updateKubeBinaryRegistryURL
-                
+
                 K8S_DOWNLOADS_TEMP_DIR_FROM_REGISTRY="/tmp/kubernetes/downloads" # /opt folder will return permission error
                 logs_to_events "AKS.CSE.installKubeletKubectlFromURL.extractKubeBinaries" extractKubeBinaries ${KUBERNETES_VERSION} "${KUBE_BINARY_REGISTRY_URL:-}" false ${K8S_DOWNLOADS_TEMP_DIR_FROM_REGISTRY}
                 # no egress traffic, default install will fail
@@ -610,7 +610,7 @@ pullContainerImage() {
             return $ERR_CONTAINERD_DOCKER_IMG_PULL_TIMEOUT
         fi
     fi
-    
+
     echo "successfully pulled image ${CONTAINER_IMAGE_URL} using ${CLI_TOOL}"
 }
 
@@ -770,10 +770,10 @@ getInstallModeAndCleanupContainerImages() {
         echo "detected golden image pre-install"
         logs_to_events "AKS.CSE.cleanUpContainerImages" cleanUpContainerImages
         FULL_INSTALL_REQUIRED=false
-    else 
+    else
         echo "the file $VHD_LOGS_FILEPATH does not exist and IS_VHD is "${IS_VHD,,}", full install requred"
     fi
- 
+
     echo "${FULL_INSTALL_REQUIRED,,}"
 }
 

parts/linux/cloud-init/artifacts/cse_config.sh

@@ -16,7 +16,6 @@ ExecStart=/usr/local/bin/aks-secure-tls-bootstrap-client \
     --cert-dir=/var/lib/kubelet/pki \
     --cluster-ca-file=/etc/kubernetes/certs/ca.crt \
     --log-file=/var/log/azure/aks/secure-tls-bootstrap.log \
-    --deadline=120s \
     $BOOTSTRAP_FLAGS
 
 [Install]