diff --git a/parts/linux/cloud-init/artifacts/cse_config.sh b/parts/linux/cloud-init/artifacts/cse_config.sh index 1ff1ee3b708..7ff251f2d0b 100755 --- a/parts/linux/cloud-init/artifacts/cse_config.sh +++ b/parts/linux/cloud-init/artifacts/cse_config.sh @@ -518,10 +518,10 @@ ensureKubeCACert() { chmod 0600 "${KUBE_CA_FILE}" } -# file paths defined outside so configureAndStartSecureTLSBootstrapping can be unit tested +# file paths defined outside so configureAndEnableSecureTLSBootstrapping can be unit tested SECURE_TLS_BOOTSTRAPPING_DEFAULT_FILE="/etc/default/secure-tls-bootstrap" SECURE_TLS_BOOTSTRAPPING_DROP_IN="/etc/systemd/system/secure-tls-bootstrap.service.d/10-securetlsbootstrap.conf" -configureAndStartSecureTLSBootstrapping() { +configureAndEnableSecureTLSBootstrapping() { BOOTSTRAP_CLIENT_FLAGS="--aad-resource=${SECURE_TLS_BOOTSTRAPPING_AAD_RESOURCE:-$AKS_AAD_SERVER_APP_ID} --apiserver-fqdn=${API_SERVER_NAME} --cloud-provider-config=${AZURE_JSON_PATH}" if [ -n "${SECURE_TLS_BOOTSTRAPPING_USER_ASSIGNED_IDENTITY_ID}" ]; then BOOTSTRAP_CLIENT_FLAGS="${BOOTSTRAP_CLIENT_FLAGS} --user-assigned-identity-id=$SECURE_TLS_BOOTSTRAPPING_USER_ASSIGNED_IDENTITY_ID" @@ -565,12 +565,16 @@ Before=kubelet.service [Service] EnvironmentFile=${SECURE_TLS_BOOTSTRAPPING_DEFAULT_FILE} [Install] +# this configuration has secure-tls-bootstrap.service only start when kubelet.service is started # once bootstrap tokens are no longer a fallback, kubelet.service needs to be a RequiredBy= WantedBy=kubelet.service EOF - # explicitly start secure TLS bootstrapping ahead of kubelet - systemctlEnableAndStartNoBlock secure-tls-bootstrap 30 || exit $ERR_SECURE_TLS_BOOTSTRAP_START_FAILURE + # enable the service so it runs ahead of kubelet on next boot; do not start it now + if ! retrycmd_if_failure 120 5 25 systemctl enable secure-tls-bootstrap; then + echo "secure-tls-bootstrap could not be enabled by systemctl" + exit $ERR_SECURE_TLS_BOOTSTRAP_ENABLE_FAILURE + fi # once bootstrap tokens are no longer a fallback, we can unset TLS_BOOTSTRAP_TOKEN here if needed } diff --git a/parts/linux/cloud-init/artifacts/cse_helpers.sh b/parts/linux/cloud-init/artifacts/cse_helpers.sh index 7242d158a6b..9fa031990ae 100755 --- a/parts/linux/cloud-init/artifacts/cse_helpers.sh +++ b/parts/linux/cloud-init/artifacts/cse_helpers.sh @@ -140,7 +140,7 @@ ERR_LOCALDNS_SLICEFILE_NOTFOUND=218 # Localdns slicefile not found. ERR_LOCALDNS_BINARY_ERR=219 # Localdns binary not found or not executable. # ---------------------------------------------------------------------------------- -ERR_SECURE_TLS_BOOTSTRAP_START_FAILURE=220 # Error starting the secure TLS bootstrap systemd service +ERR_SECURE_TLS_BOOTSTRAP_ENABLE_FAILURE=220 # Error enabling the secure TLS bootstrap systemd service ERR_CLOUD_INIT_FAILED=223 # Error indicating that cloud-init returned exit code 1 in cse_cmd.sh ERR_NVIDIA_DRIVER_INSTALL=224 # Error determining if nvidia driver install should be skipped diff --git a/parts/linux/cloud-init/artifacts/cse_main.sh b/parts/linux/cloud-init/artifacts/cse_main.sh index 0ab9de70e2d..8d29383fe1e 100755 --- a/parts/linux/cloud-init/artifacts/cse_main.sh +++ b/parts/linux/cloud-init/artifacts/cse_main.sh @@ -387,7 +387,7 @@ function nodePrep { if [ "${ENABLE_SECURE_TLS_BOOTSTRAPPING}" = "true" ]; then # Depends on configureK8s, ensureKubeCACert, and installSecureTLSBootstrapClient - logs_to_events "AKS.CSE.configureAndStartSecureTLSBootstrapping" configureAndStartSecureTLSBootstrapping + logs_to_events "AKS.CSE.configureAndEnableSecureTLSBootstrapping" configureAndEnableSecureTLSBootstrapping fi if [ -n "${OUTBOUND_COMMAND}" ]; then diff --git a/spec/parts/linux/cloud-init/artifacts/cse_config_spec.sh b/spec/parts/linux/cloud-init/artifacts/cse_config_spec.sh index 5051528c554..8dccf24d3cf 100755 --- a/spec/parts/linux/cloud-init/artifacts/cse_config_spec.sh +++ b/spec/parts/linux/cloud-init/artifacts/cse_config_spec.sh @@ -1215,7 +1215,7 @@ SETUP_EOF End End - Describe 'configureAndStartSecureTLSBootstrapping' + Describe 'configureAndEnableSecureTLSBootstrapping' SECURE_TLS_BOOTSTRAPPING_DROP_IN_DIR="secure-tls-bootstrap.service.d" SECURE_TLS_BOOTSTRAPPING_DROP_IN="${SECURE_TLS_BOOTSTRAPPING_DROP_IN_DIR}/10-securetlsbootstrap.conf" SECURE_TLS_BOOTSTRAPPING_DEFAULT_FILE_DIR="default" @@ -1227,6 +1227,11 @@ SETUP_EOF echo "chmod $@" } + retrycmd_if_failure() { + shift 3 + echo "$@" + } + cleanup() { rm -rf "$SECURE_TLS_BOOTSTRAPPING_DROP_IN_DIR" rm -rf "$SECURE_TLS_BOOTSTRAPPING_DEFAULT_FILE_DIR" @@ -1234,14 +1239,12 @@ SETUP_EOF AfterEach 'cleanup' - It 'should configure and start secure TLS bootstrapping' - systemctlEnableAndStartNoBlock() { - echo "systemctlEnableAndStartNoBlock $@" - } - When call configureAndStartSecureTLSBootstrapping + It 'should configure and enable secure TLS bootstrapping' + When call configureAndEnableSecureTLSBootstrapping The output should include "chmod 0600 secure-tls-bootstrap.service.d/10-securetlsbootstrap.conf" The output should include "chmod 0600 default/secure-tls-bootstrap" - The output should include "systemctlEnableAndStartNoBlock secure-tls-bootstrap 30" + The output should include "systemctl enable secure-tls-bootstrap" + The output should not include "systemctlEnableAndStartNoBlock" The contents of file "secure-tls-bootstrap.service.d/10-securetlsbootstrap.conf" should include "[Unit]" The contents of file "secure-tls-bootstrap.service.d/10-securetlsbootstrap.conf" should include "Before=kubelet.service" The contents of file "secure-tls-bootstrap.service.d/10-securetlsbootstrap.conf" should include "[Service]" @@ -1254,21 +1257,15 @@ SETUP_EOF End It 'should include AZURE_ENVIRONMENT_FILEPATH in the default file when set' - systemctlEnableAndStartNoBlock() { - echo "systemctlEnableAndStartNoBlock $@" - } AZURE_ENVIRONMENT_FILEPATH="/etc/kubernetes/akscustom.json" - When call configureAndStartSecureTLSBootstrapping - The output should include "systemctlEnableAndStartNoBlock secure-tls-bootstrap 30" + When call configureAndEnableSecureTLSBootstrapping + The output should include "systemctl enable secure-tls-bootstrap" The contents of file "default/secure-tls-bootstrap" should include 'BOOTSTRAP_FLAGS=--aad-resource=6dae42f8-4368-4678-94ff-3960e28e3630 --apiserver-fqdn=fqdn --cloud-provider-config=/etc/kubernetes/azure.json' The contents of file "default/secure-tls-bootstrap" should include 'AZURE_ENVIRONMENT_FILEPATH=/etc/kubernetes/akscustom.json' The status should be success End - It 'should configure and start secure TLS bootstrapping using provided overrides' - systemctlEnableAndStartNoBlock() { - echo "systemctlEnableAndStartNoBlock $@" - } + It 'should configure and enable secure TLS bootstrapping using provided overrides' SECURE_TLS_BOOTSTRAPPING_VALIDATE_KUBECONFIG_TIMEOUT="custom-validate-kubeconfig-timeout" SECURE_TLS_BOOTSTRAPPING_GET_ACCESS_TOKEN_TIMEOUT="custom-get-access-token-timeout" SECURE_TLS_BOOTSTRAPPING_GET_INSTANCE_DATA_TIMEOUT="custom-get-instance-data-timeout" @@ -1278,10 +1275,11 @@ SETUP_EOF SECURE_TLS_BOOTSTRAPPING_DEADLINE="custom-deadline" SECURE_TLS_BOOTSTRAPPING_AAD_RESOURCE="custom-resource" SECURE_TLS_BOOTSTRAPPING_USER_ASSIGNED_IDENTITY_ID="custom-identity-id" - When call configureAndStartSecureTLSBootstrapping + When call configureAndEnableSecureTLSBootstrapping The output should include "chmod 0600 secure-tls-bootstrap.service.d/10-securetlsbootstrap.conf" The output should include "chmod 0600 default/secure-tls-bootstrap" - The output should include "systemctlEnableAndStartNoBlock secure-tls-bootstrap 30" + The output should include "systemctl enable secure-tls-bootstrap" + The output should not include "systemctlEnableAndStartNoBlock" The contents of file "secure-tls-bootstrap.service.d/10-securetlsbootstrap.conf" should include "[Unit]" The contents of file "secure-tls-bootstrap.service.d/10-securetlsbootstrap.conf" should include "Before=kubelet.service" The contents of file "secure-tls-bootstrap.service.d/10-securetlsbootstrap.conf" should include "[Service]"