Skip to content

Cap per-VM STLS retry attempts (AB#38327355) #8653

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
djsly wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Cap per-VM STLS retry attempts (AB#38327355) #8653

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions aks-node-controller/parser/parser.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -139,6 +139,10 @@ func getCSEEnv(config *aksnodeconfigv1.Configuration) map[string]string {
"SECURE_TLS_BOOTSTRAPPING_GET_CREDENTIAL_TIMEOUT": config.GetBootstrappingConfig().GetSecureTlsBootstrappingGetCredentialTimeout(),
//nolint:staticcheck // keeping for now for backwards compatibility - will soon be removed
"SECURE_TLS_BOOTSTRAPPING_DEADLINE": config.GetBootstrappingConfig().GetSecureTlsBootstrappingDeadline(),
"SECURE_TLS_BOOTSTRAPPING_MAX_ATTEMPTS": config.GetBootstrappingConfig().GetSecureTlsBootstrappingMaxAttempts(),
"SECURE_TLS_BOOTSTRAPPING_MAX_TOTAL_SECONDS": config.GetBootstrappingConfig().GetSecureTlsBootstrappingMaxTotalSeconds(),
"SECURE_TLS_BOOTSTRAPPING_INITIAL_BACKOFF_SECONDS": config.GetBootstrappingConfig().GetSecureTlsBootstrappingInitialBackoffSeconds(),
"SECURE_TLS_BOOTSTRAPPING_MAX_BACKOFF_SECONDS": config.GetBootstrappingConfig().GetSecureTlsBootstrappingMaxBackoffSeconds(),
"CUSTOM_SECURE_TLS_BOOTSTRAPPING_CLIENT_DOWNLOAD_URL": config.GetBootstrappingConfig().GetSecureTlsBootstrappingCustomClientDownloadUrl(),
"ENABLE_KUBELET_SERVING_CERTIFICATE_ROTATION": fmt.Sprintf("%v", config.GetKubeletConfig().GetKubeletConfigFileConfig().GetServerTlsBootstrap()),
"DHCPV6_SERVICE_FILEPATH": getDHCPV6ServiceFilepath(),
Expand Down
4 changes: 4 additions & 0 deletions aks-node-controller/parser/parser_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -475,6 +475,10 @@ func TestAKSNodeConfigCompatibilityFromJsonToCSECommand(t *testing.T) {
assertHasKeyWithValue(t, vars, "SECURE_TLS_BOOTSTRAPPING_GET_ATTESTED_DATA_TIMEOUT", "")
assertHasKeyWithValue(t, vars, "SECURE_TLS_BOOTSTRAPPING_GET_CREDENTIAL_TIMEOUT", "")
assertHasKeyWithValue(t, vars, "SECURE_TLS_BOOTSTRAPPING_DEADLINE", "")
assertHasKeyWithValue(t, vars, "SECURE_TLS_BOOTSTRAPPING_MAX_ATTEMPTS", "")
assertHasKeyWithValue(t, vars, "SECURE_TLS_BOOTSTRAPPING_MAX_TOTAL_SECONDS", "")
assertHasKeyWithValue(t, vars, "SECURE_TLS_BOOTSTRAPPING_INITIAL_BACKOFF_SECONDS", "")
assertHasKeyWithValue(t, vars, "SECURE_TLS_BOOTSTRAPPING_MAX_BACKOFF_SECONDS", "")
},
},
}
Expand Down
210 changes: 145 additions & 65 deletions aks-node-controller/pkg/gen/aksnodeconfig/v1/bootstrapping_config.pb.go
View file
Open in desktop

Large diffs are not rendered by default.

20 changes: 20 additions & 0 deletions aks-node-controller/proto/aksnodeconfig/v1/bootstrapping_config.proto
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -83,4 +83,24 @@ message BootstrappingConfig {
// Only used when secure TLS bootstrapping is enabled. Optional override passed to the secure TLS bootstrap client during provisioning.
// This is the amount of time given to the bootstrap client to retrieve a credential from the bootstrap server.
optional string secure_tls_bootstrapping_get_credential_timeout = 15;

// Only used when secure TLS bootstrapping is enabled. Optional override consumed by the on-VM
// wrapper that bounds per-VM retry attempts. Caps the number of times the secure TLS bootstrap
// client may be re-invoked by systemd within a single provisioning session before the wrapper
// gives up and emits a terminal RetryCapReached event. See AB#38327355.
optional string secure_tls_bootstrapping_max_attempts = 16;

// Only used when secure TLS bootstrapping is enabled. Optional override consumed by the on-VM
// wrapper. Caps the total wall-clock time (in seconds) the wrapper will keep re-attempting
// within a single provisioning session before giving up.
optional string secure_tls_bootstrapping_max_total_seconds = 17;

// Only used when secure TLS bootstrapping is enabled. Optional override consumed by the on-VM
// wrapper. Initial backoff (in seconds) between consecutive attempts; doubled on each attempt
// up to secure_tls_bootstrapping_max_backoff_seconds.
optional string secure_tls_bootstrapping_initial_backoff_seconds = 18;

// Only used when secure TLS bootstrapping is enabled. Optional override consumed by the on-VM
// wrapper. Caps the exponential backoff delay (in seconds) between consecutive attempts.
optional string secure_tls_bootstrapping_max_backoff_seconds = 19;
}
4 changes: 4 additions & 0 deletions parts/linux/cloud-init/artifacts/cse_cmd.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -140,6 +140,10 @@ SECURE_TLS_BOOTSTRAPPING_GET_NONCE_TIMEOUT="{{GetSecureTLSBootstrappingGetNonceT
SECURE_TLS_BOOTSTRAPPING_GET_ATTESTED_DATA_TIMEOUT="{{GetSecureTLSBootstrappingGetAttestedDataTimeout}}"
SECURE_TLS_BOOTSTRAPPING_GET_CREDENTIAL_TIMEOUT="{{GetSecureTLSBootstrappingGetCredentialTimeout}}"
SECURE_TLS_BOOTSTRAPPING_DEADLINE="{{GetSecureTLSBootstrappingDeadline}}"
SECURE_TLS_BOOTSTRAPPING_MAX_ATTEMPTS="{{GetSecureTLSBootstrappingMaxAttempts}}"
SECURE_TLS_BOOTSTRAPPING_MAX_TOTAL_SECONDS="{{GetSecureTLSBootstrappingMaxTotalSeconds}}"
SECURE_TLS_BOOTSTRAPPING_INITIAL_BACKOFF_SECONDS="{{GetSecureTLSBootstrappingInitialBackoffSeconds}}"
SECURE_TLS_BOOTSTRAPPING_MAX_BACKOFF_SECONDS="{{GetSecureTLSBootstrappingMaxBackoffSeconds}}"
CUSTOM_SECURE_TLS_BOOTSTRAPPING_CLIENT_DOWNLOAD_URL="{{GetCustomSecureTLSBootstrappingClientDownloadURL}}"
ENABLE_KUBELET_SERVING_CERTIFICATE_ROTATION="{{EnableKubeletServingCertificateRotation}}"
DHCPV6_SERVICE_FILEPATH="{{GetDHCPv6ServiceCSEScriptFilepath}}"
Expand Down
21 changes: 21 additions & 0 deletions parts/linux/cloud-init/artifacts/cse_config.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -521,7 +521,13 @@ ensureKubeCACert() {
# file paths defined outside so configureAndStartSecureTLSBootstrapping can be unit tested
SECURE_TLS_BOOTSTRAPPING_DEFAULT_FILE="/etc/default/secure-tls-bootstrap"
SECURE_TLS_BOOTSTRAPPING_DROP_IN="/etc/systemd/system/secure-tls-bootstrap.service.d/10-securetlsbootstrap.conf"
SECURE_TLS_BOOTSTRAPPING_STATE_DIR="/var/lib/aks-secure-tls-bootstrap"
configureAndStartSecureTLSBootstrapping() {
# Reset retry-cap state at the start of every provisioning session so the
# wrapper (secure-tls-bootstrap-retry-cap.sh) counts from zero on fresh
# bootstraps. See AB#38327355.
rm -rf "${SECURE_TLS_BOOTSTRAPPING_STATE_DIR}"

BOOTSTRAP_CLIENT_FLAGS="--aad-resource=${SECURE_TLS_BOOTSTRAPPING_AAD_RESOURCE:-$AKS_AAD_SERVER_APP_ID} --apiserver-fqdn=${API_SERVER_NAME} --cloud-provider-config=${AZURE_JSON_PATH}"
if [ -n "${SECURE_TLS_BOOTSTRAPPING_USER_ASSIGNED_IDENTITY_ID}" ]; then
BOOTSTRAP_CLIENT_FLAGS="${BOOTSTRAP_CLIENT_FLAGS} --user-assigned-identity-id=$SECURE_TLS_BOOTSTRAPPING_USER_ASSIGNED_IDENTITY_ID"
Expand Down Expand Up @@ -555,6 +561,21 @@ configureAndStartSecureTLSBootstrapping() {
if [ -n "${AZURE_ENVIRONMENT_FILEPATH}" ]; then
echo "AZURE_ENVIRONMENT_FILEPATH=${AZURE_ENVIRONMENT_FILEPATH}" >> "${SECURE_TLS_BOOTSTRAPPING_DEFAULT_FILE}"
fi
# Retry-cap knobs consumed by secure-tls-bootstrap-retry-cap.sh. If unset,
# the wrapper falls back to its built-in defaults
# (50 attempts / 7200s budget / 1s initial backoff / 300s max backoff).
if [ -n "${SECURE_TLS_BOOTSTRAPPING_MAX_ATTEMPTS}" ]; then
echo "SECURE_TLS_BOOTSTRAPPING_MAX_ATTEMPTS=${SECURE_TLS_BOOTSTRAPPING_MAX_ATTEMPTS}" >> "${SECURE_TLS_BOOTSTRAPPING_DEFAULT_FILE}"
fi
if [ -n "${SECURE_TLS_BOOTSTRAPPING_MAX_TOTAL_SECONDS}" ]; then
echo "SECURE_TLS_BOOTSTRAPPING_MAX_TOTAL_SECONDS=${SECURE_TLS_BOOTSTRAPPING_MAX_TOTAL_SECONDS}" >> "${SECURE_TLS_BOOTSTRAPPING_DEFAULT_FILE}"
fi
if [ -n "${SECURE_TLS_BOOTSTRAPPING_INITIAL_BACKOFF_SECONDS}" ]; then
echo "SECURE_TLS_BOOTSTRAPPING_INITIAL_BACKOFF_SECONDS=${SECURE_TLS_BOOTSTRAPPING_INITIAL_BACKOFF_SECONDS}" >> "${SECURE_TLS_BOOTSTRAPPING_DEFAULT_FILE}"
fi
if [ -n "${SECURE_TLS_BOOTSTRAPPING_MAX_BACKOFF_SECONDS}" ]; then
echo "SECURE_TLS_BOOTSTRAPPING_MAX_BACKOFF_SECONDS=${SECURE_TLS_BOOTSTRAPPING_MAX_BACKOFF_SECONDS}" >> "${SECURE_TLS_BOOTSTRAPPING_DEFAULT_FILE}"
fi

mkdir -p "$(dirname "${SECURE_TLS_BOOTSTRAPPING_DROP_IN}")"
touch "${SECURE_TLS_BOOTSTRAPPING_DROP_IN}"
Expand Down
Loading
Loading