Allow creation of COSI files with --tools-dir.

toolkit/tools/imagecustomizer/main.go

@@ -55,6 +55,7 @@ type InjectFilesCmd struct {
 	OutputImageFile      string `name:"output-image-file" aliases:"output-path" help:"Path to write the injected image to."`
 	OutputImageFormat    string `name:"output-image-format" placeholder:"(vhd|vhd-fixed|vhdx|qcow2|raw|iso|pxe-dir|pxe-tar|cosi|baremetal-image)" help:"Format of output image." enum:"${imageformat}" default:""`
 	CosiCompressionLevel *int   `name:"cosi-compression-level" help:"Zstd compression level for COSI output (1-22, default: 9)."`
+	ToolsDir             string `name:"tools-dir" help:"Path to a directory containing tdnf/dnf and its dependencies. Required for package operations on images that do not include a package manager (e.g. ACL)."`
 }
 
 type ConvertCmd struct {
@@ -63,6 +64,7 @@ type ConvertCmd struct {
 	OutputImageFile      string `name:"output-image-file" aliases:"output-path" help:"Path to write the converted image to." required:""`
 	OutputImageFormat    string `name:"output-image-format" placeholder:"(vhd|vhd-fixed|vhdx|qcow2|raw|cosi|baremetal-image)" help:"Format of output image." required:"" enum:"${imageformatconvert}"`
 	CosiCompressionLevel *int   `name:"cosi-compression-level" help:"Zstd compression level for COSI output (1-22, default: 9)."`
+	ToolsDir             string `name:"tools-dir" help:"Path to a directory containing tdnf/dnf and its dependencies. Required for package operations on images that do not include a package manager (e.g. ACL)."`
 }
 
 type ValidateConfigCmd struct {
@@ -200,6 +202,7 @@ func injectFiles(ctx context.Context, cmd InjectFilesCmd) error {
 			OutputImageFile:      cmd.OutputImageFile,
 			OutputImageFormat:    cmd.OutputImageFormat,
 			CosiCompressionLevel: cmd.CosiCompressionLevel,
+			ToolsDir:             cmd.ToolsDir,
 		})
 	if err != nil {
 		return err
@@ -216,6 +219,7 @@ func convertImage(ctx context.Context, cmd ConvertCmd) error {
 			OutputImageFile:      cmd.OutputImageFile,
 			OutputImageFormat:    imagecustomizerapi.ImageFormatType(cmd.OutputImageFormat),
 			CosiCompressionLevel: cmd.CosiCompressionLevel,
+			ToolsDir:             cmd.ToolsDir,
 		})
 	if err != nil {
 		return err

toolkit/tools/pkg/imagecustomizerlib/artifactsinputoutput.go

@@ -23,6 +23,7 @@ import (
 	"github.com/microsoft/azure-linux-image-tools/toolkit/tools/internal/imageconnection"
 	"github.com/microsoft/azure-linux-image-tools/toolkit/tools/internal/logger"
 	"github.com/microsoft/azure-linux-image-tools/toolkit/tools/internal/randomization"
+	"github.com/microsoft/azure-linux-image-tools/toolkit/tools/internal/safechroot"
 	"github.com/microsoft/azure-linux-image-tools/toolkit/tools/internal/safeloopback"
 	"github.com/microsoft/azure-linux-image-tools/toolkit/tools/internal/safemount"
 	"github.com/microsoft/azure-linux-image-tools/toolkit/tools/internal/sliceutils"
@@ -57,7 +58,7 @@ var ukiRegex = regexp.MustCompile(`^vmlinuz-.*\.efi$`)
 
 func outputArtifacts(ctx context.Context, items []imagecustomizerapi.OutputArtifactsItemType,
 	outputDir string, buildDir string, buildImage string, verityMetadata []verityDeviceMetadata,
-	previewFeatures []imagecustomizerapi.PreviewFeature, distroHandler DistroHandler,
+	distroHandler DistroHandler,
 ) error {
 	logger.Log.Infof("Outputting artifacts")
 
@@ -280,7 +281,7 @@ func outputArtifacts(ctx context.Context, items []imagecustomizerapi.OutputArtif
 		}
 	}
 
-	err = writeInjectFilesYaml(outputArtifactsMetadata, outputDir, previewFeatures)
+	err = writeInjectFilesYaml(outputArtifactsMetadata, outputDir)
 	if err != nil {
 		return fmt.Errorf("%w (outputDir='%s'):\n%w", ErrArtifactInjectFilesYamlWrite, outputDir, err)
 	}
@@ -298,9 +299,7 @@ func outputArtifacts(ctx context.Context, items []imagecustomizerapi.OutputArtif
 	return nil
 }
 
-func writeInjectFilesYaml(metadata []imagecustomizerapi.InjectArtifactMetadata, outputDir string,
-	previewFeatures []imagecustomizerapi.PreviewFeature,
-) error {
+func writeInjectFilesYaml(metadata []imagecustomizerapi.InjectArtifactMetadata, outputDir string) error {
 	injectPreviewFeatures := []imagecustomizerapi.PreviewFeature{imagecustomizerapi.PreviewFeatureInjectFiles}
 
 	yamlStruct := imagecustomizerapi.InjectFilesConfig{
@@ -370,6 +369,22 @@ func injectFilesWithOptions(ctx context.Context, baseConfigPath string,
 		return err
 	}
 
+	var toolsChrootOuter *ToolsChroot
+	var toolsChroot *safechroot.Chroot
+	if options.ToolsDir != "" {
+		if err := validateToolsDir(options.ToolsDir); err != nil {
+			return err
+		}
+
+		toolsChrootOuter, err = initToolsChroot(ctx, options.ToolsDir)
+		if err != nil {
+			return err
+		}
+		defer toolsChrootOuter.Close()
+
+		toolsChroot = toolsChrootOuter.Chroot()
+	}
+
 	rawImageFile := filepath.Join(buildDirAbs, BaseImageName)
 
 	detectedImageFormat, err := convertImageToRaw(options.InputImageFile, rawImageFile)
@@ -392,11 +407,18 @@ func injectFilesWithOptions(ctx context.Context, baseConfigPath string,
 	}
 
 	err = convertRawImageToOutputFormat(ctx, buildDirAbs, rawImageFile, detectedImageFormat, outputImageFile,
-		options.CosiCompressionLevel)
+		options.CosiCompressionLevel, toolsChroot)
 	if err != nil {
 		return err
 	}
 
+	if toolsChrootOuter != nil {
+		err = toolsChrootOuter.CleanClose()
+		if err != nil {
+			return err
+		}
+	}
+
 	logger.Log.Infof("Success!")
 
 	return nil
@@ -461,10 +483,14 @@ func injectFilesIntoImage(buildDir string, baseConfigPath string, rawImageFile s
 }
 
 func prepareImageConversionData(ctx context.Context, rawImageFile string, buildDir string,
+	toolsChroot *safechroot.Chroot,
 ) ([]fstabEntryPartNum, []verityDeviceMetadata, string,
 	[]OsPackage, [randomization.UuidSize]byte, string, *CosiBootloader, []string, error,
 ) {
 	imageMountPoint := filepath.Join(buildDir, "imageroot")
+	if toolsChroot != nil {
+		imageMountPoint = filepath.Join(toolsChroot.RootDir(), toolsRootImageDir)
+	}
 
 	imageConnection, partitionsLayout, baseImageVerityMetadata, readonlyPartUuids, distroHandler, err :=
 		connectToExistingImage(ctx, rawImageFile, buildDir, imageMountPoint, true, true, true, true, nil)
@@ -479,7 +505,7 @@ func prepareImageConversionData(ctx context.Context, rawImageFile string, buildD
 		return nil, nil, "", nil, [randomization.UuidSize]byte{}, "", nil, nil, err
 	}
 
-	osPackages, cosiBootMetadata, err := collectOSInfoHelper(ctx, buildDir, imageConnection, distroHandler, nil)
+	osPackages, cosiBootMetadata, err := collectOSInfoHelper(ctx, buildDir, imageConnection, distroHandler, toolsChroot)
 	if err != nil {
 		return nil, nil, "", nil, [randomization.UuidSize]byte{}, "", nil, nil, err
 	}

toolkit/tools/pkg/imagecustomizerlib/artifactsinputoutput_test.go

@@ -118,20 +118,39 @@ func artifactsOutputConfigFile(t *testing.T, baseImageInfo testBaseImageInfo) st
 
 // artifactsOutputVerityConfigFile returns the artifacts-output-verity test config file appropriate for the
 // given base image version (azl3 vs azl4) and host architecture.
-func artifactsOutputVerityConfigFile(t *testing.T, baseImageInfo testBaseImageInfo) string {
+func artifactsOutputVerityConfigFile(t *testing.T, baseImageInfo testBaseImageInfo, useToolsDir bool) string {
+	toolsDirSuffix := ""
+	if useToolsDir {
+		toolsDirSuffix = "-toolsdir"
+	}
+
 	switch baseImageInfo.Version {
 	case baseImageVersionAzl2, baseImageVersionAzl3:
-		return "artifacts-output-verity-azl3.yaml"
+		return fmt.Sprintf("artifacts-output-verity-azl3%s.yaml", toolsDirSuffix)
+
 	case baseImageVersionAzl4:
-		return fmt.Sprintf("artifacts-output-verity-%s-azl4.yaml", runtime.GOARCH)
+		return fmt.Sprintf("artifacts-output-verity-%s-azl4%s.yaml", runtime.GOARCH, toolsDirSuffix)
+
 	default:
 		t.Fatalf("unsupported base image version for artifacts-output-verity test: %s", baseImageInfo.Version)
 		return ""
 	}
 }
 
 func TestOutputAndInjectArtifactsCosi(t *testing.T) {
-	baseImage, baseImageInfo := checkSkipForCustomizeDefaultAzureLinuxImage(t)
+	for _, baseImageInfo := range baseImageAzureLinuxAll {
+		t.Run(baseImageInfo.Name, func(t *testing.T) {
+			testOutputAndInjectArtifactsCosiHelper(t, baseImageInfo, false)
+		})
+
+		t.Run(baseImageInfo.Name+"ToolsDir", func(t *testing.T) {
+			testOutputAndInjectArtifactsCosiHelper(t, baseImageInfo, true)
+		})
+	}
+}
+
+func testOutputAndInjectArtifactsCosiHelper(t *testing.T, baseImageInfo testBaseImageInfo, useToolsDir bool) {
+	baseImage := checkSkipForCustomizeImage(t, baseImageInfo)
 	if baseImageInfo.Version == baseImageVersionAzl2 {
 		t.Skip("'systemd-boot' is not available on Azure Linux 2.0")
 	}
@@ -142,13 +161,21 @@ func TestOutputAndInjectArtifactsCosi(t *testing.T) {
 		t.Skip("The 'ukify' command is not available")
 	}
 
-	testTempDir := filepath.Join(tmpDir, "TestOutputAndInjectArtifacts")
+	toolsDir := ""
+	if useToolsDir {
+		toolsDir = testutils.GetDownloadedToolsDir(t, testutilsDir, baseImageInfo.Distro, baseImageInfo.Version)
+	}
+
+	testTempDir := filepath.Join(tmpDir, "TestOutputAndInjectArtifacts"+baseImageInfo.Name)
+	if useToolsDir {
+		testTempDir += "ToolsDir"
+	}
 	defer os.RemoveAll(testTempDir)
 
 	buildDir := filepath.Join(testTempDir, "build")
 	outImageFilePath := filepath.Join(testTempDir, "image.raw")
 	cosiFilePath := filepath.Join(testTempDir, "image.cosi")
-	configFile := filepath.Join(testDir, artifactsOutputVerityConfigFile(t, baseImageInfo))
+	configFile := filepath.Join(testDir, artifactsOutputVerityConfigFile(t, baseImageInfo, useToolsDir))
 	outputArtifactsDir := filepath.Join(testDir, "./out/artifacts-output-verity/artifacts")
 	injectConfigPath := filepath.Join(outputArtifactsDir, "inject-files.yaml")
 
@@ -159,8 +186,16 @@ func TestOutputAndInjectArtifactsCosi(t *testing.T) {
 	}
 
 	// Customize image.
-	err = basicCustomizeImageWithConfigFile(t.Context(), buildDir, configFile, baseImage, outImageFilePath, "raw",
-		baseImageInfo.PreviewFeatures)
+	err = CustomizeImageWithConfigFile(t.Context(), configFile, ImageCustomizerOptions{
+		BuildDir:             buildDir,
+		InputImageFile:       baseImage,
+		OutputImageFile:      outImageFilePath,
+		OutputImageFormat:    "raw",
+		UseBaseImageRpmRepos: true,
+		PreviewFeatures:      baseImageInfo.PreviewFeatures,
+		SetFilesContext:      *setfilesContext,
+		ToolsDir:             toolsDir,
+	})
 	if !assert.NoError(t, err) {
 		return
 	}
@@ -173,6 +208,7 @@ func TestOutputAndInjectArtifactsCosi(t *testing.T) {
 		InputImageFile:    outImageFilePath,
 		OutputImageFile:   cosiFilePath,
 		OutputImageFormat: "cosi",
+		ToolsDir:          toolsDir,
 	}
 	err = InjectFilesWithConfigFile(t.Context(), injectConfigPath, options)
 	if !assert.NoError(t, err) {

toolkit/tools/pkg/imagecustomizerlib/convertimage.go

@@ -12,6 +12,7 @@ import (
 	"github.com/microsoft/azure-linux-image-tools/toolkit/tools/imagecustomizerapi"
 	"github.com/microsoft/azure-linux-image-tools/toolkit/tools/internal/file"
 	"github.com/microsoft/azure-linux-image-tools/toolkit/tools/internal/logger"
+	"github.com/microsoft/azure-linux-image-tools/toolkit/tools/internal/safechroot"
 	"go.opentelemetry.io/otel"
 	"go.opentelemetry.io/otel/attribute"
 )
@@ -77,7 +78,7 @@ func ConvertImage(ctx context.Context, options ConvertImageOptions) (err error)
 		outputFormat == imagecustomizerapi.ImageFormatTypeBareMetalImage
 	if isCosiOutput {
 		err = convertImageToCosi(ctx, options.BuildDir, options.InputImageFile, options.OutputImageFile,
-			outputFormat, options.CosiCompressionLevel)
+			outputFormat, options.CosiCompressionLevel, options.ToolsDir)
 		if err != nil {
 			return err
 		}
@@ -95,7 +96,7 @@ func ConvertImage(ctx context.Context, options ConvertImageOptions) (err error)
 }
 
 func convertImageToCosi(ctx context.Context, buildDir string, inputImageFile string, outputImageFile string,
-	outputFormat imagecustomizerapi.ImageFormatType, cosiCompressionLevel *int,
+	outputFormat imagecustomizerapi.ImageFormatType, cosiCompressionLevel *int, toolsDir string,
 ) error {
 	buildDirAbs, err := filepath.Abs(buildDir)
 	if err != nil {
@@ -107,28 +108,52 @@ func convertImageToCosi(ctx context.Context, buildDir string, inputImageFile str
 		return err
 	}
 
+	var toolsChrootOuter *ToolsChroot
+	var toolsChroot *safechroot.Chroot
+	if toolsDir != "" {
+		if err := validateToolsDir(toolsDir); err != nil {
+			return err
+		}
+
+		toolsChrootOuter, err = initToolsChroot(ctx, toolsDir)
+		if err != nil {
+			return err
+		}
+		defer toolsChrootOuter.Close()
+
+		toolsChroot = toolsChrootOuter.Chroot()
+	}
+
 	rawImageFile := filepath.Join(buildDirAbs, BaseImageName)
 	_, err = convertImageToRaw(inputImageFile, rawImageFile)
 	if err != nil {
 		return err
 	}
 
 	err = convertRawImageToOutputFormat(ctx, buildDirAbs, rawImageFile, outputFormat,
-		outputImageFile, cosiCompressionLevel)
+		outputImageFile, cosiCompressionLevel, toolsChroot)
 	if err != nil {
 		return err
 	}
 
+	if toolsChrootOuter != nil {
+		err = toolsChrootOuter.CleanClose()
+		if err != nil {
+			return err
+		}
+	}
+
 	return nil
 }
 
 func convertRawImageToOutputFormat(ctx context.Context, buildDirAbs string, rawImageFile string,
 	outputFormat imagecustomizerapi.ImageFormatType, outputImageFile string, cosiCompressionLevel *int,
+	toolsChroot *safechroot.Chroot,
 ) error {
 	if outputFormat == imagecustomizerapi.ImageFormatTypeCosi || outputFormat == imagecustomizerapi.ImageFormatTypeBareMetalImage {
 		// Convert subcommand doesn't support preview features - pass empty slice
 		partitionsLayout, baseImageVerityMetadata, osRelease, osPackages, imageUuid, imageUuidStr, cosiBootMetadata,
-			readonlyPartUuids, err := prepareImageConversionData(ctx, rawImageFile, buildDirAbs)
+			readonlyPartUuids, err := prepareImageConversionData(ctx, rawImageFile, buildDirAbs, toolsChroot)
 		if err != nil {
 			return fmt.Errorf("%w:\n%w", ErrArtifactCosiImageInfoCollect, err)
 		}

toolkit/tools/pkg/imagecustomizerlib/convertimage_test.go

@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"slices"
 	"testing"
 
 	"github.com/microsoft/azure-linux-image-tools/toolkit/tools/imagecustomizerapi"
@@ -83,25 +84,31 @@ func testConvertImageRawToQcow2(t *testing.T, baseImageInfo testBaseImageInfo) {
 }
 
 func TestConvertImageRawToCosi(t *testing.T) {
-	for _, baseImageInfo := range checkSkipForCustomizeDefaultImages(t) {
+	for _, baseImageInfo := range slices.Concat(baseImageAzureLinuxAll, baseImageUbuntuAll) {
 		t.Run(baseImageInfo.Name, func(t *testing.T) {
-			testConvertImageRawToCosi(t, baseImageInfo)
+			testConvertImageRawToCosi(t, baseImageInfo, false)
+		})
+
+		t.Run(baseImageInfo.Name+"ToolsDir", func(t *testing.T) {
+			testConvertImageRawToCosi(t, baseImageInfo, true)
 		})
 	}
 }
 
-func testConvertImageRawToCosi(t *testing.T, baseImageInfo testBaseImageInfo) {
+func testConvertImageRawToCosi(t *testing.T, baseImageInfo testBaseImageInfo, useToolsDir bool) {
 	baseImage := checkSkipForCustomizeImage(t, baseImageInfo)
 
-	if baseImageInfo.Distro == baseImageDistroAzureLinux && baseImageInfo.Version == baseImageVersionAzl2 {
-		t.Skip("'systemd-boot' is not available on Azure Linux 2.0")
-	}
 	ukifyExists, err := file.CommandExists("ukify")
 	assert.NoError(t, err)
 	if !ukifyExists {
 		t.Skip("The 'ukify' command is not available")
 	}
 
+	toolsDir := ""
+	if useToolsDir {
+		toolsDir = testutils.GetDownloadedToolsDir(t, testutilsDir, baseImageInfo.Distro, baseImageInfo.Version)
+	}
+
 	testTempDir := filepath.Join(tmpDir, fmt.Sprintf("TestConvertImageRawToCosi_%s", baseImageInfo.Name))
 	defer os.RemoveAll(testTempDir)
 
@@ -110,7 +117,7 @@ func testConvertImageRawToCosi(t *testing.T, baseImageInfo testBaseImageInfo) {
 
 	// First, we need a customized image with verity enabled
 	customizedImage := filepath.Join(testTempDir, "customized.raw")
-	configFile := filepath.Join(testDir, "verity-config.yaml")
+	configFile := getRootVerityConfigFile(t, baseImageInfo, useToolsDir)
 
 	err = CustomizeImageWithConfigFile(t.Context(), configFile, ImageCustomizerOptions{
 		BuildDir:             buildDir,
@@ -119,6 +126,7 @@ func testConvertImageRawToCosi(t *testing.T, baseImageInfo testBaseImageInfo) {
 		OutputImageFormat:    "raw",
 		UseBaseImageRpmRepos: true,
 		PreviewFeatures:      baseImageInfo.PreviewFeatures,
+		ToolsDir:             toolsDir,
 	})
 	if baseImageInfo.Distro == baseImageDistroUbuntu {
 		// This check should be removed once bootloader hard-reset support is added for Ubuntu.
@@ -137,6 +145,7 @@ func testConvertImageRawToCosi(t *testing.T, baseImageInfo testBaseImageInfo) {
 		InputImageFile:    customizedImage,
 		OutputImageFile:   outputImageFile,
 		OutputImageFormat: imagecustomizerapi.ImageFormatTypeCosi,
+		ToolsDir:          toolsDir,
 	}
 
 	err = ConvertImage(t.Context(), options)
@@ -184,7 +193,7 @@ func testConvertImageRawToCosiWithCompression(t *testing.T, baseImageInfo testBa
 
 	// First, we need a customized image with verity enabled
 	customizedImage := filepath.Join(testTempDir, "customized.raw")
-	configFile := filepath.Join(testDir, "verity-config.yaml")
+	configFile := getRootVerityConfigFile(t, baseImageInfo, false /*useToolsDir*/)
 
 	err = CustomizeImageWithConfigFile(t.Context(), configFile, ImageCustomizerOptions{
 		BuildDir:             buildDir,

toolkit/tools/pkg/imagecustomizerlib/convertimageoptions.go

@@ -15,6 +15,7 @@ type ConvertImageOptions struct {
 	OutputImageFile      string
 	OutputImageFormat    imagecustomizerapi.ImageFormatType
 	CosiCompressionLevel *int
+	ToolsDir             string
 }
 
 func (o *ConvertImageOptions) IsValid() error {