refactor(linux): only start secure-tls-bootstrap.service via kubelet WantedBy=

parts/linux/cloud-init/artifacts/cse_config.sh

@@ -518,10 +518,10 @@ ensureKubeCACert() {
     chmod 0600 "${KUBE_CA_FILE}"
 }
 
-# file paths defined outside so configureAndStartSecureTLSBootstrapping can be unit tested
+# file paths defined outside so configureAndEnableSecureTLSBootstrapping can be unit tested
 SECURE_TLS_BOOTSTRAPPING_DEFAULT_FILE="/etc/default/secure-tls-bootstrap"
 SECURE_TLS_BOOTSTRAPPING_DROP_IN="/etc/systemd/system/secure-tls-bootstrap.service.d/10-securetlsbootstrap.conf"
-configureAndStartSecureTLSBootstrapping() {
+configureAndEnableSecureTLSBootstrapping() {
     BOOTSTRAP_CLIENT_FLAGS="--aad-resource=${SECURE_TLS_BOOTSTRAPPING_AAD_RESOURCE:-$AKS_AAD_SERVER_APP_ID} --apiserver-fqdn=${API_SERVER_NAME} --cloud-provider-config=${AZURE_JSON_PATH}"
     if [ -n "${SECURE_TLS_BOOTSTRAPPING_USER_ASSIGNED_IDENTITY_ID}" ]; then
         BOOTSTRAP_CLIENT_FLAGS="${BOOTSTRAP_CLIENT_FLAGS} --user-assigned-identity-id=$SECURE_TLS_BOOTSTRAPPING_USER_ASSIGNED_IDENTITY_ID"
@@ -565,12 +565,16 @@ Before=kubelet.service
 [Service]
 EnvironmentFile=${SECURE_TLS_BOOTSTRAPPING_DEFAULT_FILE}
 [Install]
+# this configuration has secure-tls-bootstrap.service only start when kubelet.service is started
 # once bootstrap tokens are no longer a fallback, kubelet.service needs to be a RequiredBy=
 WantedBy=kubelet.service
 EOF
 
-    # explicitly start secure TLS bootstrapping ahead of kubelet
-    systemctlEnableAndStartNoBlock secure-tls-bootstrap 30 || exit $ERR_SECURE_TLS_BOOTSTRAP_START_FAILURE
+    # enable the service so it runs ahead of kubelet on next boot; do not start it now
+    if ! retrycmd_if_failure 120 5 25 systemctl enable secure-tls-bootstrap; then
+        echo "secure-tls-bootstrap could not be enabled by systemctl"
+        exit $ERR_SECURE_TLS_BOOTSTRAP_ENABLE_FAILURE
+    fi
 
     # once bootstrap tokens are no longer a fallback, we can unset TLS_BOOTSTRAP_TOKEN here if needed
 }

parts/linux/cloud-init/artifacts/cse_helpers.sh

@@ -140,7 +140,7 @@ ERR_LOCALDNS_SLICEFILE_NOTFOUND=218 # Localdns slicefile not found.
 ERR_LOCALDNS_BINARY_ERR=219 # Localdns binary not found or not executable.
 # ----------------------------------------------------------------------------------
 
-ERR_SECURE_TLS_BOOTSTRAP_START_FAILURE=220 # Error starting the secure TLS bootstrap systemd service
+ERR_SECURE_TLS_BOOTSTRAP_ENABLE_FAILURE=220 # Error enabling the secure TLS bootstrap systemd service
 
 ERR_CLOUD_INIT_FAILED=223 # Error indicating that cloud-init returned exit code 1 in cse_cmd.sh
 ERR_NVIDIA_DRIVER_INSTALL=224 # Error determining if nvidia driver install should be skipped

parts/linux/cloud-init/artifacts/cse_main.sh

@@ -387,7 +387,7 @@ function nodePrep {
 
     if [ "${ENABLE_SECURE_TLS_BOOTSTRAPPING}" = "true" ]; then
         # Depends on configureK8s, ensureKubeCACert, and installSecureTLSBootstrapClient
-        logs_to_events "AKS.CSE.configureAndStartSecureTLSBootstrapping" configureAndStartSecureTLSBootstrapping
+        logs_to_events "AKS.CSE.configureAndEnableSecureTLSBootstrapping" configureAndEnableSecureTLSBootstrapping
     fi
 
     if [ -n "${OUTBOUND_COMMAND}" ]; then

spec/parts/linux/cloud-init/artifacts/cse_config_spec.sh

@@ -1215,7 +1215,7 @@ SETUP_EOF
         End
     End
 
-    Describe 'configureAndStartSecureTLSBootstrapping'
+    Describe 'configureAndEnableSecureTLSBootstrapping'
         SECURE_TLS_BOOTSTRAPPING_DROP_IN_DIR="secure-tls-bootstrap.service.d"
         SECURE_TLS_BOOTSTRAPPING_DROP_IN="${SECURE_TLS_BOOTSTRAPPING_DROP_IN_DIR}/10-securetlsbootstrap.conf"
         SECURE_TLS_BOOTSTRAPPING_DEFAULT_FILE_DIR="default"
@@ -1227,21 +1227,24 @@ SETUP_EOF
             echo "chmod $@"
         }
 
+        retrycmd_if_failure() {
+            shift 3
+            echo "$@"
+        }
+
         cleanup() {
             rm -rf "$SECURE_TLS_BOOTSTRAPPING_DROP_IN_DIR"
             rm -rf "$SECURE_TLS_BOOTSTRAPPING_DEFAULT_FILE_DIR"
         }
 
         AfterEach 'cleanup'
 
-        It 'should configure and start secure TLS bootstrapping'
-            systemctlEnableAndStartNoBlock() {
-                echo "systemctlEnableAndStartNoBlock $@"
-            }
-            When call configureAndStartSecureTLSBootstrapping
+        It 'should configure and enable secure TLS bootstrapping'
+            When call configureAndEnableSecureTLSBootstrapping
             The output should include "chmod 0600 secure-tls-bootstrap.service.d/10-securetlsbootstrap.conf"
             The output should include "chmod 0600 default/secure-tls-bootstrap"
-            The output should include "systemctlEnableAndStartNoBlock secure-tls-bootstrap 30"
+            The output should include "systemctl enable secure-tls-bootstrap"
+            The output should not include "systemctlEnableAndStartNoBlock"
             The contents of file "secure-tls-bootstrap.service.d/10-securetlsbootstrap.conf" should include "[Unit]"
             The contents of file "secure-tls-bootstrap.service.d/10-securetlsbootstrap.conf" should include "Before=kubelet.service"
             The contents of file "secure-tls-bootstrap.service.d/10-securetlsbootstrap.conf" should include "[Service]"
@@ -1254,21 +1257,15 @@ SETUP_EOF
         End
 
         It 'should include AZURE_ENVIRONMENT_FILEPATH in the default file when set'
-            systemctlEnableAndStartNoBlock() {
-                echo "systemctlEnableAndStartNoBlock $@"
-            }
             AZURE_ENVIRONMENT_FILEPATH="/etc/kubernetes/akscustom.json"
-            When call configureAndStartSecureTLSBootstrapping
-            The output should include "systemctlEnableAndStartNoBlock secure-tls-bootstrap 30"
+            When call configureAndEnableSecureTLSBootstrapping
+            The output should include "systemctl enable secure-tls-bootstrap"
             The contents of file "default/secure-tls-bootstrap" should include 'BOOTSTRAP_FLAGS=--aad-resource=6dae42f8-4368-4678-94ff-3960e28e3630 --apiserver-fqdn=fqdn --cloud-provider-config=/etc/kubernetes/azure.json'
             The contents of file "default/secure-tls-bootstrap" should include 'AZURE_ENVIRONMENT_FILEPATH=/etc/kubernetes/akscustom.json'
             The status should be success
         End
 
-        It 'should configure and start secure TLS bootstrapping using provided overrides'
-            systemctlEnableAndStartNoBlock() {
-                echo "systemctlEnableAndStartNoBlock $@"
-            }
+        It 'should configure and enable secure TLS bootstrapping using provided overrides'
             SECURE_TLS_BOOTSTRAPPING_VALIDATE_KUBECONFIG_TIMEOUT="custom-validate-kubeconfig-timeout"
             SECURE_TLS_BOOTSTRAPPING_GET_ACCESS_TOKEN_TIMEOUT="custom-get-access-token-timeout"
             SECURE_TLS_BOOTSTRAPPING_GET_INSTANCE_DATA_TIMEOUT="custom-get-instance-data-timeout"
@@ -1278,10 +1275,11 @@ SETUP_EOF
             SECURE_TLS_BOOTSTRAPPING_DEADLINE="custom-deadline"
             SECURE_TLS_BOOTSTRAPPING_AAD_RESOURCE="custom-resource"
             SECURE_TLS_BOOTSTRAPPING_USER_ASSIGNED_IDENTITY_ID="custom-identity-id"
-            When call configureAndStartSecureTLSBootstrapping
+            When call configureAndEnableSecureTLSBootstrapping
             The output should include "chmod 0600 secure-tls-bootstrap.service.d/10-securetlsbootstrap.conf"
             The output should include "chmod 0600 default/secure-tls-bootstrap"
-            The output should include "systemctlEnableAndStartNoBlock secure-tls-bootstrap 30"
+            The output should include "systemctl enable secure-tls-bootstrap"
+            The output should not include "systemctlEnableAndStartNoBlock"
             The contents of file "secure-tls-bootstrap.service.d/10-securetlsbootstrap.conf" should include "[Unit]"
             The contents of file "secure-tls-bootstrap.service.d/10-securetlsbootstrap.conf" should include "Before=kubelet.service"
             The contents of file "secure-tls-bootstrap.service.d/10-securetlsbootstrap.conf" should include "[Service]"