fix: dont run scriptless phase2 if preprovision is turned on (#8440)

Author: awesomenix

e2e/vmss.go

@@ -355,7 +355,7 @@ func createVMSSModel(ctx context.Context, s *Scenario) armcompute.VirtualMachine
 		require.NoError(s.T, err)
 		cse = nodeBootstrapping.CSE
 		customData = nodeBootstrapping.CustomData
-		if s.Runtime.NBC.EnableScriptlessNBCCSECmd && !config.Config.DisableScriptLessCompilation && !s.Tags.NetworkIsolated {
+		if s.Runtime.NBC.EnableScriptlessNBCCSECmd && !config.Config.DisableScriptLessCompilation && !s.Tags.NetworkIsolated && !s.Runtime.NBC.PreProvisionOnly {
 			binaryURL, err := CachedCompileAndUploadAKSNodeController(ctx, s.VHD.Arch)
 			require.NoError(s.T, err, "failed to compile and upload aks-node-controller binary")
 			customData, err = CustomDataWithNBCCmdHack(s, customData, binaryURL)

pkg/agent/baker.go

@@ -92,7 +92,7 @@ func (t *TemplateGenerator) getWindowsNodeBootstrappingPayload(config *datamodel
 }
 
 func (t *TemplateGenerator) getLinuxNodeBootstrappingPayload(config *datamodel.NodeBootstrappingConfiguration) string {
-	if config.EnableScriptlessNBCCSECmd {
+	if config.EnableScriptlessNBCCSECmd && !config.PreProvisionOnly {
 		config.DisableCustomData = true
 		config.EnableScriptlessCSECmd = true
 		nbcCMD := t.getLinuxNodeCSECommand(config)
@@ -352,7 +352,7 @@ func (t *TemplateGenerator) getNodeBootstrappingCmd(config *datamodel.NodeBootst
 	if config.AgentPoolProfile.IsWindows() {
 		return t.getWindowsNodeCSECommand(config)
 	}
-	if config.EnableScriptlessNBCCSECmd {
+	if config.EnableScriptlessNBCCSECmd && !config.PreProvisionOnly {
 		return "/opt/azure/containers/aks-node-controller provision-wait"
 	}
 	return t.getLinuxNodeCSECommand(config)

pkg/agent/baker_test.go

@@ -1286,14 +1286,14 @@ var _ = Describe("getLinuxNodeCSECommand", func() {
 })
 
 var _ = Describe("getLinuxNodeBootstrappingPayload", func() {
-	It("should persist nodecustomdata in the scriptless NBC boothook", func() {
-		templateGenerator := InitializeTemplateGenerator()
+	newConfig := func(preProvisionOnly bool) *datamodel.NodeBootstrappingConfiguration {
 		agentPoolProfile := &datamodel.AgentPoolProfile{
 			Name:   "nodepool1",
 			OSType: datamodel.Linux,
 			Distro: datamodel.AKSUbuntuContainerd2204Gen2,
 		}
-		config := &datamodel.NodeBootstrappingConfiguration{
+
+		return &datamodel.NodeBootstrappingConfiguration{
 			ContainerService: &datamodel.ContainerService{
 				Location: "eastus",
 				Properties: &datamodel.Properties{
@@ -1315,7 +1315,13 @@ var _ = Describe("getLinuxNodeBootstrappingPayload", func() {
 			K8sComponents:             &datamodel.K8sComponents{},
 			KubeletConfig:             map[string]string{},
 			EnableScriptlessNBCCSECmd: true,
+			PreProvisionOnly:          preProvisionOnly,
 		}
+	}
+
+	It("should persist nodecustomdata in the scriptless NBC boothook", func() {
+		templateGenerator := InitializeTemplateGenerator()
+		config := newConfig(false)
 
 		payload := templateGenerator.getLinuxNodeBootstrappingPayload(config)
 		decodedPayload, err := base64.StdEncoding.DecodeString(payload)
@@ -1330,6 +1336,62 @@ var _ = Describe("getLinuxNodeBootstrappingPayload", func() {
 		Expect(string(decodedPayload)).To(ContainSubstring(encodedNodeCustomData))
 		Expect(string(decodedPayload)).To(ContainSubstring(nbcCmdFilePath))
 	})
+
+	It("should fall back to regular custom data when pre-provisioning is enabled", func() {
+		templateGenerator := InitializeTemplateGenerator()
+		config := newConfig(true)
+
+		payload := templateGenerator.getLinuxNodeBootstrappingPayload(config)
+		decodedPayload, err := base64.StdEncoding.DecodeString(payload)
+		Expect(err).NotTo(HaveOccurred())
+
+		decompressedPayload, err := getGzipDecodedValue(decodedPayload)
+		Expect(err).NotTo(HaveOccurred())
+
+		expectedCustomData := getCustomDataFromJSON(templateGenerator.getLinuxNodeCustomDataJSONObject(config))
+
+		Expect(string(decompressedPayload)).To(Equal(expectedCustomData))
+		Expect(string(decompressedPayload)).NotTo(ContainSubstring(nodeCustomDataPath))
+		Expect(string(decompressedPayload)).NotTo(ContainSubstring(nbcCmdFilePath))
+	})
+})
+
+var _ = Describe("getNodeBootstrappingCmd", func() {
+	It("should use the regular linux CSE command when pre-provisioning is enabled", func() {
+		templateGenerator := InitializeTemplateGenerator()
+		agentPoolProfile := &datamodel.AgentPoolProfile{
+			Name:   "nodepool1",
+			OSType: datamodel.Linux,
+			Distro: datamodel.AKSUbuntuContainerd2204Gen2,
+		}
+		config := &datamodel.NodeBootstrappingConfiguration{
+			ContainerService: &datamodel.ContainerService{
+				Location: "eastus",
+				Properties: &datamodel.Properties{
+					OrchestratorProfile: &datamodel.OrchestratorProfile{
+						OrchestratorVersion: "1.29.0",
+						OrchestratorType:    datamodel.Kubernetes,
+						KubernetesConfig: &datamodel.KubernetesConfig{
+							ContainerRuntimeConfig: map[string]string{},
+						},
+					},
+					HostedMasterProfile: &datamodel.HostedMasterProfile{
+						FQDN: "test-cluster.hcp.eastus.azmk8s.io",
+					},
+					AgentPoolProfiles: []*datamodel.AgentPoolProfile{agentPoolProfile},
+				},
+			},
+			AgentPoolProfile:          agentPoolProfile,
+			CloudSpecConfig:           datamodel.AzurePublicCloudSpecForTest,
+			K8sComponents:             &datamodel.K8sComponents{},
+			KubeletConfig:             map[string]string{},
+			EnableScriptlessNBCCSECmd: true,
+			PreProvisionOnly:          true,
+		}
+
+		Expect(templateGenerator.getNodeBootstrappingCmd(config)).To(Equal(templateGenerator.getLinuxNodeCSECommand(config)))
+		Expect(templateGenerator.getNodeBootstrappingCmd(config)).NotTo(Equal("/opt/azure/containers/aks-node-controller provision-wait"))
+	})
 })
 
 var _ = Describe("cloudInitToButane", func() {