fix: add ACL-specific butane config with first-boot service workaround (#8447)

Author: aadhar-agarwal

parts/linux/cloud-init/acl.yml

@@ -0,0 +1,55 @@
+variant: flatcar
+version: 1.1.0
+systemd:
+  units:
+    - name: ignition-bootcmds.service
+      enabled: true
+      contents: |
+        [Unit]
+        Description=Ignition Early Boot Commands
+        DefaultDependencies=no
+        After=local-fs.target
+        Before=sysinit.target
+        ConditionPathExists=/etc/ignition-bootcmds.sh
+
+        [Service]
+        Type=oneshot
+        ExecStart=-/etc/ignition-bootcmds.sh
+
+        [Install]
+        WantedBy=sysinit.target
+
+    - name: ignition-file-extract.service
+      enabled: true
+      contents: |
+        [Unit]
+        Description=Extract Ignition file payload
+        DefaultDependencies=no
+        After=local-fs.target
+        Before=sysinit.target ignition-bootcmds.service
+        ConditionPathExists=/var/lib/ignition/ignition-files.tar
+
+        [Service]
+        Type=oneshot
+        ExecStart=tar -xvf /var/lib/ignition/ignition-files.tar -C /
+        ExecStart=rm -f /var/lib/ignition/ignition-files.tar
+        ExecStart=systemctl daemon-reload
+        RemainAfterExit=yes
+
+        [Install]
+        WantedBy=sysinit.target
+
+# Workaround for ACL (Azure Container Linux):
+# After VHD build, /etc/machine-id is empty (not absent). systemd only
+# triggers ConditionFirstBoot=yes when machine-id is missing, so enabled
+# services like these don't start on first boot. On Flatcar, waagent's
+# CoreOS deprovisioning happens to remove machine-id, avoiding this issue.
+# Explicit symlinks ensure these services start regardless of first-boot state.
+storage:
+  links:
+    - path: /etc/systemd/system/sysinit.target.wants/ignition-bootcmds.service
+      target: /etc/systemd/system/ignition-bootcmds.service
+      overwrite: true
+    - path: /etc/systemd/system/sysinit.target.wants/ignition-file-extract.service
+      target: /etc/systemd/system/ignition-file-extract.service
+      overwrite: true

parts/linux/cloud-init/nodecustomdata.yml

@@ -34,34 +34,34 @@ write_files:
     {{GetVariableProperty "cloudInitData" "provisionSource"}}
 
 
-{{if IsAzlOSGuard}}
+{{if IsACL }}
 - path: {{GetCSEHelpersScriptDistroFilepath}}
   permissions: "0744"
   encoding: gzip
   owner: root
   content: !!binary |
-    {{GetVariableProperty "cloudInitData" "provisionSourceAzlOSGuard"}}
-{{- else if IsMariner}}
+    {{GetVariableProperty "cloudInitData" "provisionSourceACL"}}
+{{- else if IsAzlOSGuard}}
 - path: {{GetCSEHelpersScriptDistroFilepath}}
   permissions: "0744"
   encoding: gzip
   owner: root
   content: !!binary |
-    {{GetVariableProperty "cloudInitData" "provisionSourceMariner"}}
-{{- else if IsFlatcar }}
+    {{GetVariableProperty "cloudInitData" "provisionSourceAzlOSGuard"}}
+{{- else if IsMariner}}
 - path: {{GetCSEHelpersScriptDistroFilepath}}
   permissions: "0744"
   encoding: gzip
   owner: root
   content: !!binary |
-    {{GetVariableProperty "cloudInitData" "provisionSourceFlatcar"}}
-{{- else if IsACL }}
+    {{GetVariableProperty "cloudInitData" "provisionSourceMariner"}}
+{{- else if IsFlatcar }}
 - path: {{GetCSEHelpersScriptDistroFilepath}}
   permissions: "0744"
   encoding: gzip
   owner: root
   content: !!binary |
-    {{GetVariableProperty "cloudInitData" "provisionSourceACL"}}
+    {{GetVariableProperty "cloudInitData" "provisionSourceFlatcar"}}
 {{- else }}
 - path: {{GetCSEHelpersScriptDistroFilepath}}
   permissions: "0744"
@@ -108,34 +108,34 @@ write_files:
   content: !!binary |
     {{GetVariableProperty "cloudInitData" "provisionSendLogs"}}
 
-{{if IsAzlOSGuard}}
+{{if IsACL }}
 - path: {{GetCSEInstallScriptDistroFilepath}}
   permissions: "0744"
   encoding: gzip
   owner: root
   content: !!binary |
-    {{GetVariableProperty "cloudInitData" "provisionInstallsAzlOSGuard"}}
-{{- else if IsMariner}}
+    {{GetVariableProperty "cloudInitData" "provisionInstallsACL"}}
+{{- else if IsAzlOSGuard}}
 - path: {{GetCSEInstallScriptDistroFilepath}}
   permissions: "0744"
   encoding: gzip
   owner: root
   content: !!binary |
-    {{GetVariableProperty "cloudInitData" "provisionInstallsMariner"}}
-{{- else if IsFlatcar }}
+    {{GetVariableProperty "cloudInitData" "provisionInstallsAzlOSGuard"}}
+{{- else if IsMariner}}
 - path: {{GetCSEInstallScriptDistroFilepath}}
   permissions: "0744"
   encoding: gzip
   owner: root
   content: !!binary |
-    {{GetVariableProperty "cloudInitData" "provisionInstallsFlatcar"}}
-{{- else if IsACL }}
+    {{GetVariableProperty "cloudInitData" "provisionInstallsMariner"}}
+{{- else if IsFlatcar }}
 - path: {{GetCSEInstallScriptDistroFilepath}}
   permissions: "0744"
   encoding: gzip
   owner: root
   content: !!binary |
-    {{GetVariableProperty "cloudInitData" "provisionInstallsACL"}}
+    {{GetVariableProperty "cloudInitData" "provisionInstallsFlatcar"}}
 {{- else }}
 - path: {{GetCSEInstallScriptDistroFilepath}}
   permissions: "0744"
@@ -210,7 +210,8 @@ write_files:
   content: !!binary |
     {{GetVariableProperty "cloudInitData" "migPartitionScript"}}
 
-{{if IsAzlOSGuard}}
+{{if IsACL}}
+{{- else if IsAzlOSGuard}}
 {{- else if IsMariner}}
 - path: /opt/azure/containers/mariner-package-update.sh
   permissions: "0544"

pkg/agent/baker.go

@@ -245,11 +245,11 @@ func buildIgnitionTarball(entries []ignitionTarEntry) ([]byte, error) {
 	return buf.Bytes(), nil
 }
 
-func cloudInitToButane(customData cloudInit) flatcar1_1.Config {
+func cloudInitToButane(customData cloudInit, butaneYamlPath string) flatcar1_1.Config {
 	butaneconfig := flatcar1_1.Config{}
-	b, e := parts.Templates.ReadFile(kubernetesFlatcarNodeCustomDataYaml)
+	b, e := parts.Templates.ReadFile(butaneYamlPath)
 	if e != nil {
-		panic(fmt.Errorf("yaml file %s does not exist", kubernetesFlatcarNodeCustomDataYaml))
+		panic(fmt.Errorf("yaml file %s does not exist", butaneYamlPath))
 	}
 	if e = yaml.Unmarshal(b, &butaneconfig); e != nil {
 		panic(fmt.Errorf("failed to unmarshal butane config: %w", e))
@@ -304,7 +304,11 @@ func (t *TemplateGenerator) getFlatcarLinuxNodeCustomDataJSONObject(config *data
 		panic(fmt.Errorf("no write files found in customData"))
 	}
 
-	var butaneconfig = cloudInitToButane(customData)
+	butaneYamlPath := kubernetesFlatcarNodeCustomDataYaml
+	if config.IsACL() {
+		butaneYamlPath = kubernetesACLNodeCustomDataYaml
+	}
+	var butaneconfig = cloudInitToButane(customData, butaneYamlPath)
 	ignition, report, e := butaneconfig.ToIgn3_4(butanecommon.TranslateOptions{})
 	if e != nil {
 		panic(fmt.Errorf("butane -> ignition: error: %w:\n%s", e, report.String()))

pkg/agent/baker_test.go

@@ -1404,7 +1404,7 @@ var _ = Describe("cloudInitToButane", func() {
 
 	It("should convert bootcmds to a systemd unit and shell script", func() {
 		var config = cloudInit{BootCommands: []string{"echo hello world", "ls 'some dir'"}}
-		var butane = cloudInitToButane(config)
+		var butane = cloudInitToButane(config, kubernetesFlatcarNodeCustomDataYaml)
 		checkForUnit(butane)
 		Expect(butane.Storage.Files).To(HaveLen(1))
 		var file = butane.Storage.Files[0]
@@ -1447,7 +1447,7 @@ var _ = Describe("cloudInitToButane", func() {
 				Content:     string(gzipped),
 			},
 		}}
-		var butane = cloudInitToButane(config)
+		var butane = cloudInitToButane(config, kubernetesFlatcarNodeCustomDataYaml)
 		Expect(butane.Storage.Files).To(HaveLen(1))
 		var file = butane.Storage.Files[0]
 		tarball, err := decodeButaneResource(file.Contents)
@@ -1476,7 +1476,7 @@ var _ = Describe("cloudInitToButane", func() {
 				Content:     encoded,
 			},
 		}}
-		var butane = cloudInitToButane(config)
+		var butane = cloudInitToButane(config, kubernetesFlatcarNodeCustomDataYaml)
 		Expect(butane.Storage.Files).To(HaveLen(1))
 		var file = butane.Storage.Files[0]
 		tarball, err := decodeButaneResource(file.Contents)
@@ -1496,7 +1496,7 @@ var _ = Describe("cloudInitToButane", func() {
 
 	It("should create a system unit but not a shell script with no bootcmds", func() {
 		var config = cloudInit{BootCommands: []string{}}
-		var butane = cloudInitToButane(config)
+		var butane = cloudInitToButane(config, kubernetesFlatcarNodeCustomDataYaml)
 		checkForUnit(butane)
 		Expect(butane.Storage.Files).To(BeEmpty())
 		Expect(butane.Systemd.Units).NotTo(BeEmpty())
@@ -1511,6 +1511,24 @@ var _ = Describe("cloudInitToButane", func() {
 		}
 		Expect(found).To(BeTrue())
 	})
+
+	It("should include storage links for ACL butane config", func() {
+		var config = cloudInit{BootCommands: []string{"echo hello"}}
+		var butane = cloudInitToButane(config, kubernetesACLNodeCustomDataYaml)
+		checkForUnit(butane)
+		Expect(butane.Storage.Links).To(HaveLen(2))
+		Expect(butane.Storage.Links[0].Path).To(Equal("/etc/systemd/system/sysinit.target.wants/ignition-bootcmds.service"))
+		Expect(*butane.Storage.Links[0].Target).To(Equal("/etc/systemd/system/ignition-bootcmds.service"))
+		Expect(butane.Storage.Links[1].Path).To(Equal("/etc/systemd/system/sysinit.target.wants/ignition-file-extract.service"))
+		Expect(*butane.Storage.Links[1].Target).To(Equal("/etc/systemd/system/ignition-file-extract.service"))
+	})
+
+	It("should not include storage links for Flatcar butane config", func() {
+		var config = cloudInit{BootCommands: []string{"echo hello"}}
+		var butane = cloudInitToButane(config, kubernetesFlatcarNodeCustomDataYaml)
+		checkForUnit(butane)
+		Expect(butane.Storage.Links).To(BeEmpty())
+	})
 })
 
 func decodeButaneResource(resource base0_5.Resource) ([]byte, error) {

pkg/agent/const.go

@@ -42,6 +42,7 @@ const (
 // cloud-init (i.e. ARM customData) source file references.
 const (
 	kubernetesFlatcarNodeCustomDataYaml   = "linux/cloud-init/flatcar.yml"
+	kubernetesACLNodeCustomDataYaml       = "linux/cloud-init/acl.yml"
 	kubernetesNodeCustomDataYaml          = "linux/cloud-init/nodecustomdata.yml"
 	kubernetesCSECommandString            = "linux/cloud-init/artifacts/cse_cmd.sh"
 	kubernetesCSEStartScript              = "linux/cloud-init/artifacts/cse_start.sh"