feat: provide configurable cse timeout in seconds (#7766)

Author: awesomenix

aks-node-controller/parser/helper.go

@@ -32,6 +32,7 @@ import (
 	"github.com/Azure/agentbaker/aks-node-controller/helpers"
 	aksnodeconfigv1 "github.com/Azure/agentbaker/aks-node-controller/pkg/gen/aksnodeconfig/v1"
 	"github.com/Azure/agentbaker/pkg/agent"
+	"github.com/Azure/agentbaker/pkg/agent/datamodel"
 	"google.golang.org/protobuf/encoding/protojson"
 )
 
@@ -809,3 +810,16 @@ func getLocalDnsMemoryLimitInMb(aksnodeconfig *aksnodeconfigv1.Configuration) st
 }
 
 // ---------------------- End of localdns related helper code ----------------------//
+
+// ---------------------- Start of cse timeout helper code ----------------------//
+
+// getCSETimeout returns the CSE timeout value in minutes.
+func getCSETimeout(aksnodeconfig *aksnodeconfigv1.Configuration) string {
+	cseTimeout := 0
+	if aksnodeconfig != nil {
+		cseTimeout = int(aksnodeconfig.GetCseTimeout())
+	}
+	return datamodel.GetCSETimeout(cseTimeout)
+}
+
+// ---------------------- End of cse timeout helper code ----------------------//

aks-node-controller/parser/parser.go

@@ -179,6 +179,7 @@ func getCSEEnv(config *aksnodeconfigv1.Configuration) map[string]string {
 		"SERVICE_ACCOUNT_IMAGE_PULL_DEFAULT_CLIENT_ID":       config.GetServiceAccountImagePullProfile().GetDefaultClientId(),
 		"SERVICE_ACCOUNT_IMAGE_PULL_DEFAULT_TENANT_ID":       config.GetServiceAccountImagePullProfile().GetDefaultTenantId(),
 		"IDENTITY_BINDINGS_LOCAL_AUTHORITY_SNI":              config.GetServiceAccountImagePullProfile().GetLocalAuthoritySni(),
+		"CSE_TIMEOUT":                                        getCSETimeout(config),
 	}
 
 	for i, cert := range config.CustomCaCerts {

aks-node-controller/pkg/gen/aksnodeconfig/v1/config.pb.go

@@ -167,4 +167,7 @@ message Configuration {
 
   // Service account based image pull profile configuration
   ServiceAccountImagePullProfile service_account_image_pull_profile = 43;
+
+  // CSE timeout override in seconds. If not specified, defaults to 15 minutes with a maximum of 360 minutes (6 hours).
+  optional int32 cse_timeout = 44;
 }

aks-node-controller/proto/aksnodeconfig/v1/config.proto

@@ -183,4 +183,5 @@ MCR_REPOSITORY_BASE="{{GetMCRRepositoryBase}}"
 ENABLE_IMDS_RESTRICTION="{{EnableIMDSRestriction}}"
 INSERT_IMDS_RESTRICTION_RULE_TO_MANGLE_TABLE="{{InsertIMDSRestrictionRuleToMangleTable}}"
 PRE_PROVISION_ONLY="{{GetPreProvisionOnly}}"
+CSE_TIMEOUT="{{GetCSETimeout}}"
 /usr/bin/nohup /bin/bash -c "/bin/bash /opt/azure/containers/provision_start.sh"

parts/linux/cloud-init/artifacts/cse_cmd.sh

@@ -7,7 +7,7 @@ export CSE_STARTTIME_SECONDS=$(date -d "$CSE_STARTTIME_FORMATTED" +%s) # Export
 EVENTS_LOGGING_DIR=/var/log/azure/Microsoft.Azure.Extensions.CustomScript/events/
 mkdir -p $EVENTS_LOGGING_DIR
 # this is the "global" CSE execution timeout - we allow CSE to run for 15 minutes before timeout will attempt to kill the script. We exit early from some of the retry loops using `check_cse_timeout` in `cse_helpers.sh`.`
-timeout -k5s 15m /bin/bash /opt/azure/containers/provision.sh >> /var/log/azure/cluster-provision.log 2>&1
+timeout -k5s $CSE_TIMEOUT /bin/bash /opt/azure/containers/provision.sh >> /var/log/azure/cluster-provision.log 2>&1
 EXIT_CODE=$?
 systemctl --no-pager -l status kubelet >> /var/log/azure/cluster-provision-cse-output.log 2>&1
 OUTPUT=$(tail -c 3000 "/var/log/azure/cluster-provision.log")

parts/linux/cloud-init/artifacts/cse_start.sh

@@ -1222,6 +1222,7 @@ func getContainerServiceFuncMap(config *datamodel.NodeBootstrappingConfiguration
 			return profile.GetLocalDNSMemoryLimitInMB()
 		},
 		"GetPreProvisionOnly": func() bool { return config.PreProvisionOnly },
+		"GetCSETimeout":       func() string { return datamodel.GetCSETimeout(config.CSETimeout) },
 		"BlockIptables": func() bool {
 			return cs.Properties.OrchestratorProfile.KubernetesConfig.BlockIptables
 		},

pkg/agent/baker.go

@@ -136,3 +136,9 @@ const (
 	EnableIPv6Only        = "EnableIPv6Only"
 	EnableWinDSR          = "EnableWinDSR"
 )
+
+// CSE Constants.
+const (
+	DefaultCSETimeout = 900   // 15 minutes
+	MaxCSETimeout     = 21600 // 6 hours
+)

pkg/agent/datamodel/const.go

@@ -129,3 +129,13 @@ func getComponentNameFromURL(downloadURL string) (string, error) {
 func IsMIGNode(gpuInstanceProfile string) bool {
 	return gpuInstanceProfile != ""
 }
+
+// returns the CSE timeout value in seconds.
+// if empty or invalid value is provided, it returns the default timeout value of 15minutes or 900 seconds.
+// Maximum allowed timeout is 360 minutes or 6 hours or 21600 seconds.
+func GetCSETimeout(cseTimeout int) string {
+	if cseTimeout <= 0 || cseTimeout > MaxCSETimeout {
+		cseTimeout = DefaultCSETimeout
+	}
+	return fmt.Sprintf("%d", cseTimeout)
+}

pkg/agent/datamodel/helper.go

@@ -1771,6 +1771,9 @@ type NodeBootstrappingConfiguration struct {
 	// PreProvisionOnly creates a pre-provisioned image for later node spawning.
 	// Skips kubelet and some component configuration for image capture scenarios.
 	PreProvisionOnly bool
+
+	// CSETimeout specifies the timeout execution in seconds.
+	CSETimeout int
 }
 
 func (config *NodeBootstrappingConfiguration) IsFlatcar() bool {