Add Azure Linux 4.0 ISO LiveOS bootstrap support (#816)

This enables Azure Linux 4.0 ISO (LiveOS) bootstrap and full-OS output.
It was previously gated off because Azure Linux 4.0 uses Fedora-style
BLS bootloader packaging that the LiveOS ISO builder did not handle, and
this change adds distro-aware LiveOS hooks on `DistroHandler` to support
it.

**What changed**
- Distro-aware LiveOS hooks on `DistroHandler` (required packages, grub
EFI prefix directory, initrd dracut modules) for acl, azurelinux,
azurelinux4, fedora, and ubuntu, plus the BLS grub.cfg and PXE
boot-config helpers.
- BLS-layout builder fixes: preserve each loader entry's
`boot/loader/entries` subpath when staging, skip dangling module
symlinks such as `symvers` when building the bootstrap initrd, write a
grub prefix redirector so Azure Linux 4.0's `EFI/azurelinux`-prefixed
grub finds its config, preserve each grub `search --set=<target>` when
relabeling for the ISO volume id, and fall back to the input ISO's
distro handler when repackaging an ISO without OS customization.
- Omit the `selinux` dracut module for Fedora and Azure Linux 4.0, whose
policy loads after `switch_root`.
- Add the `dracut-live` package to the Azure Linux 4.0 bootstrap config
and unskip the Azure Linux 4.0 ISO vmtests.

PXE output stays explicitly unsupported for Azure Linux 4.0. That
follows in a separate change.

Author: vinceaperri

test/vmtests/vmtests/imagecustomizer/test_min_change.py

@@ -417,7 +417,6 @@ def test_min_change_efi_azl3_iso_bootstrap_output(
     )
 
 
-@pytest.mark.skip(reason="Azure Linux 4.0 ISO bootstrap output is not yet supported")
 def test_min_change_efi_azl4_iso_bootstrap_output(
     docker_client: DockerClient,
     image_customizer_container_url: str,
@@ -431,6 +430,7 @@ def test_min_change_efi_azl4_iso_bootstrap_output(
 ) -> None:
     azl_release = 4
     config_path = TEST_CONFIGS_DIR.joinpath("iso-bootstrap-vm-azl4.yaml")
+    config_path = add_preview_features_to_config(config_path, "preview-distro-version", close_list)
     output_format = "iso"
 
     run_min_change_test(
@@ -480,7 +480,6 @@ def test_min_change_efi_azl3_iso_full_os_output(
     )
 
 
-@pytest.mark.skip(reason="Azure Linux 4.0 ISO full OS output is not yet supported")
 def test_min_change_efi_azl4_iso_full_os_output(
     docker_client: DockerClient,
     image_customizer_container_url: str,
@@ -494,6 +493,7 @@ def test_min_change_efi_azl4_iso_full_os_output(
 ) -> None:
     azl_release = 4
     config_path = TEST_CONFIGS_DIR.joinpath("iso-full-os-vm-azl4.yaml")
+    config_path = add_preview_features_to_config(config_path, "preview-distro-version", close_list)
     output_format = "iso"
 
     run_min_change_test(
@@ -577,7 +577,6 @@ def test_min_change_legacy_azl3_iso_output(
 
 
 @pytest.mark.skipif(platform.machine() != "x86_64", reason="no arm64 legacy boot input images are available")
-@pytest.mark.skip(reason="Azure Linux 4.0 ISO bootstrap output is not yet supported")
 def test_min_change_legacy_azl4_iso_output(
     docker_client: DockerClient,
     image_customizer_container_url: str,
@@ -591,6 +590,7 @@ def test_min_change_legacy_azl4_iso_output(
 ) -> None:
     azl_release = 4
     config_path = TEST_CONFIGS_DIR.joinpath("iso-bootstrap-vm-azl4.yaml")
+    config_path = add_preview_features_to_config(config_path, "preview-distro-version", close_list)
     output_format = "iso"
 
     run_min_change_test(

toolkit/tools/pkg/imagecustomizerlib/blsutils.go

@@ -263,3 +263,120 @@ func updateBLSEntryOptions(content string, argsToRemove []string, newArgs []stri
 
 	return result
 }
+
+// forEachBLSEntry rewrites every non-rescue BLS .conf file under {bootDir}/loader/entries by passing its content
+// through edit.
+func forEachBLSEntry(bootDir string, edit func(content string) (string, error)) error {
+	entriesDir := filepath.Join(bootDir, "loader", "entries")
+	entries, err := os.ReadDir(entriesDir)
+	if err != nil {
+		return fmt.Errorf("failed to read BLS entries directory (%s):\n%w", entriesDir, err)
+	}
+
+	for _, entry := range entries {
+		if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".conf") {
+			continue
+		}
+
+		absPath := filepath.Join(entriesDir, entry.Name())
+		contentBytes, err := os.ReadFile(absPath)
+		if err != nil {
+			return fmt.Errorf("failed to read BLS entry file (%s):\n%w", absPath, err)
+		}
+		content := string(contentBytes)
+
+		title := ""
+		for _, field := range bls.ParseFields(content) {
+			if field.Key == "title" {
+				title = field.Value
+				break
+			}
+		}
+		if bls.IsRescueEntryTitle(title) {
+			continue
+		}
+
+		updatedContent, err := edit(content)
+		if err != nil {
+			return fmt.Errorf("failed to update BLS entry file (%s):\n%w", absPath, err)
+		}
+
+		if updatedContent == content {
+			continue
+		}
+
+		err = os.WriteFile(absPath, []byte(updatedContent), 0o644)
+		if err != nil {
+			return fmt.Errorf("failed to write BLS entry file (%s):\n%w", absPath, err)
+		}
+	}
+
+	return nil
+}
+
+// setBLSEntryField replaces the value of the first line with the given key (e.g. "linux" or "initrd") in a BLS entry,
+// preserving every other byte. It returns an error if the key is absent unlike updateBLSEntryOptions above, which adds
+// an options line if none exists.
+func setBLSEntryField(content string, key string, newValue string) (string, error) {
+	for _, line := range bls.ParseLines(content) {
+		if line.Key != key {
+			continue
+		}
+		return content[:line.ContentStart] + key + " " + newValue + content[line.ContentEnd:], nil
+	}
+	return "", fmt.Errorf("failed to find the '%s' key in BLS entry", key)
+}
+
+// updateLiveOSBLSEntries applies the LiveOS-compatibility kernel-entry edits to the BLS entries under bootDir. It is
+// the BLS counterpart of the per-entry portion of updateGrubCfgForLiveOS.
+func updateLiveOSBLSEntries(bootDir string, initramfsImageType imagecustomizerapi.InitramfsImageType,
+	disableSELinux bool, savedConfigs *SavedConfigs,
+) error {
+	var argsToRemove []string
+	var argsToAppend []string
+
+	switch initramfsImageType {
+	case imagecustomizerapi.InitramfsImageTypeFullOS:
+		// Remove 'root' so that no pivoting takes place.
+		argsToRemove = append(argsToRemove, "root")
+	case imagecustomizerapi.InitramfsImageTypeBootstrap:
+		liveosKernelArgs := fmt.Sprintf(kernelArgsLiveOSTemplate, liveOSDir, liveOSImage)
+		argsToAppend = append(argsToAppend, strings.Fields(liveosKernelArgs)...)
+	default:
+		return fmt.Errorf("unsupported initramfs image type (%s)", initramfsImageType)
+	}
+
+	if disableSELinux {
+		argsToRemove = append(argsToRemove, selinuxArgNames...)
+		selinuxArgs, err := selinuxModeToArgs(imagecustomizerapi.SELinuxModeDisabled)
+		if err != nil {
+			return err
+		}
+		argsToAppend = append(argsToAppend, selinuxArgs...)
+	}
+
+	argsToAppend = append(argsToAppend, savedConfigs.LiveOS.KernelCommandLine.ExtraCommandLine...)
+
+	return forEachBLSEntry(bootDir, func(content string) (string, error) {
+		if initramfsImageType == imagecustomizerapi.InitramfsImageTypeFullOS {
+			var err error
+			content, err = setBLSEntryField(content, "initrd", isoInitrdPath)
+			if err != nil {
+				return "", fmt.Errorf("failed to update initrd path in BLS entry:\n%w", err)
+			}
+		}
+		content = updateBLSEntryOptions(content, argsToRemove, argsToAppend)
+		return content, nil
+	})
+}
+
+// setLiveOSBLSEntriesRoot replaces the root= kernel arg in every BLS entry under bootDir and optionally appends
+// additional args.
+func setLiveOSBLSEntriesRoot(bootDir string, rootValue string, extraArgs []string) error {
+	argsToRemove := []string{"root"}
+	newArgs := append([]string{"root=" + rootValue}, extraArgs...)
+
+	return forEachBLSEntry(bootDir, func(content string) (string, error) {
+		return updateBLSEntryOptions(content, argsToRemove, newArgs), nil
+	})
+}

toolkit/tools/pkg/imagecustomizerlib/blsutils_test.go

@@ -8,6 +8,7 @@ import (
 	"path/filepath"
 	"testing"
 
+	"github.com/microsoft/azure-linux-image-tools/toolkit/tools/imagecustomizerapi"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -570,3 +571,110 @@ func TestParseBLSOptionsValue(t *testing.T) {
 		})
 	}
 }
+
+// writeTestBLSEntry creates a single AZL4-style BLS entry under {bootDir}/loader/entries and returns the entry's
+// absolute path.
+func writeTestBLSEntry(t *testing.T, bootDir string) string {
+	entriesDir := filepath.Join(bootDir, "loader", "entries")
+	err := os.MkdirAll(entriesDir, 0o755)
+	assert.NoError(t, err)
+
+	content := "title Azure Linux (6.18.31-1.5.azl4.x86_64) 4.0\n" +
+		"version 6.18.31-1.5.azl4.x86_64\n" +
+		"linux /boot/vmlinuz-6.18.31-1.5.azl4.x86_64\n" +
+		"initrd /boot/initramfs-6.18.31-1.5.azl4.x86_64.img\n" +
+		"options console=ttyS0 root=UUID=1396c02f-6cf5-438c-9c9c-fb2001079bb9 rd.shell=0\n" +
+		"grub_users $grub_users\n" +
+		"grub_arg --unrestricted\n" +
+		"grub_class azurelinux\n"
+
+	entryPath := filepath.Join(entriesDir, "azl4.conf")
+	err = os.WriteFile(entryPath, []byte(content), 0o644)
+	assert.NoError(t, err)
+	return entryPath
+}
+
+func TestUpdateLiveOSBLSEntriesFullOS(t *testing.T) {
+	bootDir := t.TempDir()
+	entryPath := writeTestBLSEntry(t, bootDir)
+
+	savedConfigs := &SavedConfigs{}
+	savedConfigs.LiveOS.KernelCommandLine.ExtraCommandLine = []string{"rd.info"}
+
+	err := updateLiveOSBLSEntries(bootDir, imagecustomizerapi.InitramfsImageTypeFullOS, false /*disableSELinux*/, savedConfigs)
+	assert.NoError(t, err)
+
+	got, err := os.ReadFile(entryPath)
+	assert.NoError(t, err)
+	gotStr := string(got)
+
+	// Full-OS repoints initrd at the single regenerated /boot/initrd.img.
+	assert.Regexp(t, `(?m)^initrd /boot/initrd\.img$`, gotStr)
+	// root= is dropped so no pivot takes place, and the unrelated arg is preserved.
+	assert.NotContains(t, gotStr, "root=UUID=")
+	assert.Regexp(t, `(?m)^options .*console=ttyS0`, gotStr)
+	// The saved extra command line is appended.
+	assert.Regexp(t, `(?m)^options .* rd\.info$`, gotStr)
+	// blscfg-only keys are untouched.
+	assert.Contains(t, gotStr, "grub_class azurelinux")
+}
+
+func TestUpdateLiveOSBLSEntriesBootstrap(t *testing.T) {
+	bootDir := t.TempDir()
+	entryPath := writeTestBLSEntry(t, bootDir)
+
+	savedConfigs := &SavedConfigs{}
+	savedConfigs.LiveOS.KernelCommandLine.ExtraCommandLine = []string{"rd.shell"}
+
+	err := updateLiveOSBLSEntries(bootDir, imagecustomizerapi.InitramfsImageTypeBootstrap, true /*disableSELinux*/, savedConfigs)
+	assert.NoError(t, err)
+
+	got, err := os.ReadFile(entryPath)
+	assert.NoError(t, err)
+	gotStr := string(got)
+
+	// Bootstrap keeps the per-kernel initrd and the original root (the iso/pxe root is set later).
+	assert.Regexp(t, `(?m)^initrd /boot/initramfs-6\.18\.31-1\.5\.azl4\.x86_64\.img$`, gotStr)
+	assert.Contains(t, gotStr, "root=UUID=1396c02f-6cf5-438c-9c9c-fb2001079bb9")
+	// The dracut live-OS args are appended.
+	assert.Contains(t, gotStr, "rd.live.image")
+	assert.Contains(t, gotStr, "rd.live.dir="+liveOSDir)
+	assert.Contains(t, gotStr, "rd.live.squashimg="+liveOSImage)
+	// SELinux is disabled.
+	assert.Contains(t, gotStr, "selinux=0")
+	// The saved extra command line is appended.
+	assert.Regexp(t, `(?m)^options .* rd\.shell$`, gotStr)
+}
+
+func TestSetLiveOSBLSEntriesRootForIso(t *testing.T) {
+	bootDir := t.TempDir()
+	entryPath := writeTestBLSEntry(t, bootDir)
+
+	err := setLiveOSBLSEntriesRoot(bootDir, "live:LABEL=foo", nil)
+	assert.NoError(t, err)
+
+	got, err := os.ReadFile(entryPath)
+	assert.NoError(t, err)
+	gotStr := string(got)
+
+	assert.Contains(t, gotStr, "root=live:LABEL=foo")
+	assert.NotContains(t, gotStr, "root=UUID=")
+}
+
+func TestSetBLSEntryField(t *testing.T) {
+	content := "title Foo\n" +
+		"linux /boot/vmlinuz-6.6\n" +
+		"initrd /boot/initramfs-6.6.img\n" +
+		"options root=/dev/sda1\n"
+
+	got, err := setBLSEntryField(content, "initrd", "/boot/initrd.img")
+	assert.NoError(t, err)
+	assert.Regexp(t, `(?m)^initrd /boot/initrd\.img$`, got)
+	// Other lines are preserved verbatim.
+	assert.Contains(t, got, "linux /boot/vmlinuz-6.6\n")
+	assert.Contains(t, got, "options root=/dev/sda1\n")
+
+	// Absent key is an error rather than a silent no-op, mirroring the inline-grub path.
+	_, err = setBLSEntryField(content, "efi", "/x")
+	assert.Error(t, err)
+}

toolkit/tools/pkg/imagecustomizerlib/distrohandler.go

@@ -131,12 +131,33 @@ type DistroHandler interface {
 	UpdateBootConfigForVerity(verityMetadata []verityDeviceMetadata, bootPartitionTmpDir string,
 		bootRelativePath string, partitions []diskutils.PartitionInfo, buildDir string, bootUuid string) error
 
+	// UpdateLiveOSGrubCfgForLiveOS applies the common LiveOS-compatibility edits to the grub.cfg generation. It is the
+	// base that the iso and pxe steps build on.
+	UpdateLiveOSGrubCfgForLiveOS(grubCfgContent string, bootDir string,
+		initramfsType imagecustomizerapi.InitramfsImageType, disableSELinux bool, savedConfigs *SavedConfigs,
+		kernelVersions []string) (string, error)
+
+	// UpdateLiveOSGrubCfgForIso applies the iso-specific edits on top of the LiveOS edits.
+	UpdateLiveOSGrubCfgForIso(grubCfgContent string, bootDir string,
+		initramfsType imagecustomizerapi.InitramfsImageType) (string, error)
+
 	// ShimPackage returns the package that provides the shim EFI binary for this distro on the current architecture.
 	ShimPackage() string
 
 	// GrubEfiPackage returns the package that provides the grub EFI binary for this distro on the current architecture.
 	GrubEfiPackage() string
 
+	// LiveOSRequiredPackages returns the packages that must already be installed in the target image for Image
+	// Customizer to build a LiveOS bootstrap initrd (the squashfs and dracut live tooling).
+	LiveOSRequiredPackages() []string
+
+	// LiveOSGrubEfiPrefixDir returns the ISO-relative directory that this distro's grub EFI binary uses as its baked-in
+	// 'prefix', or "" if it doesn't use a baked-in prefix.
+	LiveOSGrubEfiPrefixDir() string
+
+	// LiveOSInitrdDracutModules returns the dracut modules to add when building the LiveOS bootstrap initrd.
+	LiveOSInitrdDracutModules() []string
+
 	// Distro has a root partition that is missing placeholder directories for special mounts like /dev.
 	RootMissingMountDirectories() bool
 

toolkit/tools/pkg/imagecustomizerlib/distrohandler_acl.go

@@ -295,6 +295,19 @@ func (d *aclDistroHandler) UpdateBootConfigForVerity(verityMetadata []verityDevi
 	return fs.ErrNotExist
 }
 
+func (d *aclDistroHandler) UpdateLiveOSGrubCfgForLiveOS(grubCfgContent string, bootDir string,
+	initramfsType imagecustomizerapi.InitramfsImageType, disableSELinux bool, savedConfigs *SavedConfigs,
+	kernelVersions []string,
+) (string, error) {
+	return updateGrubCfgForLiveOS(grubCfgContent, initramfsType, disableSELinux, savedConfigs, kernelVersions)
+}
+
+func (d *aclDistroHandler) UpdateLiveOSGrubCfgForIso(grubCfgContent string, bootDir string,
+	initramfsType imagecustomizerapi.InitramfsImageType,
+) (string, error) {
+	return updateGrubCfgForIso(grubCfgContent, initramfsType)
+}
+
 func (d *aclDistroHandler) ShimPackage() string {
 	// ACL uses systemd-boot + UKI (no shim/grub from a package).
 	return ""
@@ -305,6 +318,18 @@ func (d *aclDistroHandler) GrubEfiPackage() string {
 	return ""
 }
 
+func (d *aclDistroHandler) LiveOSRequiredPackages() []string {
+	return liveOSRequiredPackagesAzl3
+}
+
+func (d *aclDistroHandler) LiveOSGrubEfiPrefixDir() string {
+	return ""
+}
+
+func (d *aclDistroHandler) LiveOSInitrdDracutModules() []string {
+	return liveOSInitrdDracutModulesAzl3
+}
+
 func (d *aclDistroHandler) RootMissingMountDirectories() bool {
 	return true
 }