Skip to content

Commit 24fff9c

Browse files
cwize1cwize1
authored
Simple storage merging for base configs API. (#809)
1. Allow storage to be specified one time in the config tree. This covers a lot of uses cases, without the need for more complex merging logic. 2. Merge preview feature flags from base configs. 2. For the test config files that have distro variations, move common settings into a base config. This helps significantly reduce config duplication. 3. For the image creation test config files, always use a 3 GiB disk size, to avoid unneccessary differences between distros. 4. In the `TestOutputAndInjectArtifacts` and `TestOutputAndInjectArtifactsCosi` test, don't create a temporary copy of the config file. This makes it easier to use the base configs API in the config files. 5. For the `create-minimal-os` config files, move the common list of packages into a package list file.
1 parent d0b7fff commit 24fff9c

50 files changed

Lines changed: 749 additions & 1657 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎toolkit/tools/pkg/imagecustomizerlib/artifactsinputoutput_test.go‎

Lines changed: 14 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -40,13 +40,14 @@ func TestOutputAndInjectArtifacts(t *testing.T) {
4040
buildDir := filepath.Join(testTempDir, "build")
4141
buildDirCustomize := filepath.Join(buildDir, "customize")
4242
outImageFilePath := filepath.Join(testTempDir, "image.raw")
43-
originalConfigFile := filepath.Join(testDir, artifactsOutputConfigFile(t, baseImageInfo))
44-
configFile := filepath.Join(testTempDir, "artifacts-output.yaml")
45-
outputArtifactsDir := filepath.Join(testTempDir, "output")
43+
configFile := filepath.Join(testDir, artifactsOutputConfigFile(t, baseImageInfo))
44+
outputArtifactsDir := filepath.Join(testDir, "out/artifacts-output/artifacts")
4645

47-
// Copy test config to the temp dir so it's isolated
48-
err = file.Copy(originalConfigFile, configFile)
49-
assert.NoError(t, err)
46+
// Clean artifacts dir
47+
err = os.RemoveAll(outputArtifactsDir)
48+
if !assert.NoError(t, err) {
49+
return
50+
}
5051

5152
// Customize image
5253
err = basicCustomizeImageWithConfigFile(t.Context(), buildDirCustomize, configFile, baseImage, outImageFilePath, "raw",
@@ -147,14 +148,15 @@ func TestOutputAndInjectArtifactsCosi(t *testing.T) {
147148
buildDir := filepath.Join(testTempDir, "build")
148149
outImageFilePath := filepath.Join(testTempDir, "image.raw")
149150
cosiFilePath := filepath.Join(testTempDir, "image.cosi")
150-
originalConfigFile := filepath.Join(testDir, artifactsOutputVerityConfigFile(t, baseImageInfo))
151-
configFile := filepath.Join(testTempDir, "artifacts-output-verity.yaml")
152-
outputArtifactsDir := filepath.Join(testTempDir, "output")
151+
configFile := filepath.Join(testDir, artifactsOutputVerityConfigFile(t, baseImageInfo))
152+
outputArtifactsDir := filepath.Join(testDir, "./out/artifacts-output-verity/artifacts")
153153
injectConfigPath := filepath.Join(outputArtifactsDir, "inject-files.yaml")
154154

155-
// Copy test config to the temp dir so it's isolated
156-
err = file.Copy(originalConfigFile, configFile)
157-
assert.NoError(t, err)
155+
// Clean artifacts dir
156+
err = os.RemoveAll(outputArtifactsDir)
157+
if !assert.NoError(t, err) {
158+
return
159+
}
158160

159161
// Customize image.
160162
err = basicCustomizeImageWithConfigFile(t.Context(), buildDir, configFile, baseImage, outImageFilePath, "raw",

‎toolkit/tools/pkg/imagecustomizerlib/baseconfigs_test.go‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -300,18 +300,18 @@ func hierarchicalConfigFile(t *testing.T, baseImageInfo testBaseImageInfo) strin
300300
}
301301
}
302302

303-
func TestBaseConfigsStorageInBaseConfig(t *testing.T) {
303+
func TestBaseConfigsStorageMoreThanOne(t *testing.T) {
304304
baseImage, baseImageInfo := checkSkipForCustomizeDefaultAzureLinuxImage(t)
305305

306-
testTmpDir := filepath.Join(tmpDir, "TestBaseConfigsStorageInBaseConfig")
306+
testTmpDir := filepath.Join(tmpDir, "TestBaseConfigsStorageMoreThanOne")
307307
defer os.RemoveAll(testTmpDir)
308308

309309
buildDir := filepath.Join(testTmpDir, "build")
310310
outImageFilePath := filepath.Join(testTmpDir, "image.raw")
311311

312-
currentConfigFile := filepath.Join(testDir, "storage-in-base-config.yaml")
312+
currentConfigFile := filepath.Join(testDir, "storage-more-than-one.yaml")
313313

314314
err := basicCustomizeImageWithConfigFile(t.Context(), buildDir, currentConfigFile, baseImage,
315315
outImageFilePath, "raw", baseImageInfo.PreviewFeatures)
316-
assert.ErrorIs(t, err, ErrStorageInBaseConfig)
316+
assert.ErrorIs(t, err, ErrStorageMoreThanOne)
317317
}

‎toolkit/tools/pkg/imagecustomizerlib/configvalidation.go‎

Lines changed: 21 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,7 @@ var (
5555
ErrInvalidInputImageAzureLinux = NewImageCustomizerError("Validation:InvalidInputImageAzureLinux", "invalid input.image.azureLinux config")
5656
ErrInputImageAzureLinuxNotFound = NewImageCustomizerError("Validation:InputImageAzureLinuxNotFound", "input.image.azureLinux not found")
5757
ErrInputImageOciNotFound = NewImageCustomizerError("Validation:InputImageOciNotFound", "input.image.oci not found")
58-
ErrStorageInBaseConfig = NewImageCustomizerError("Validation:StorageBaseConfig", "storage is not yet supported as a value in a base config")
58+
ErrStorageMoreThanOne = NewImageCustomizerError("Validation:StorageMoreThanOne", "storage may only be specified in one config file:\nstorage merging logic not yet implemented")
5959
)
6060

6161
// ValidateConfigWithConfigFileOptions validates a configuration file without performing customization.
@@ -136,15 +136,18 @@ func ValidateConfig(ctx context.Context, baseConfigPath string, config *imagecus
136136
return nil, err
137137
}
138138

139-
rc.PreviewFeatures = append(rc.PreviewFeatures, config.PreviewFeatures...)
140-
rc.PreviewFeatures = append(rc.PreviewFeatures, options.PreviewFeatures...)
141-
142-
err = options.verifyPreviewFeatures(rc.PreviewFeatures)
139+
rc.ConfigChain, err = buildConfigChain(ctx, config, baseConfigPath)
143140
if err != nil {
144141
return nil, err
145142
}
146143

147-
rc.ConfigChain, err = buildConfigChain(ctx, config, baseConfigPath)
144+
rc.PreviewFeatures = append(rc.PreviewFeatures, options.PreviewFeatures...)
145+
146+
for _, configWithBase := range rc.ConfigChain {
147+
rc.PreviewFeatures = append(rc.PreviewFeatures, configWithBase.Config.PreviewFeatures...)
148+
}
149+
150+
err = options.verifyPreviewFeatures(rc.PreviewFeatures)
148151
if err != nil {
149152
return nil, err
150153
}
@@ -260,8 +263,11 @@ func ValidateConfig(ctx context.Context, baseConfigPath string, config *imagecus
260263
}
261264

262265
func resolveStorage(rc *ResolvedConfig) (imagecustomizerapi.Storage, error) {
263-
// Disallow storage fields in base configs until merging logic is implemented.
264-
for i := 0; i < len(rc.ConfigChain)-1; i++ {
266+
foundStorage := imagecustomizerapi.Storage{}
267+
foundStorageCount := 0
268+
269+
// Only allow storage to be specified in 1 config file in the tree.
270+
for i := 0; i < len(rc.ConfigChain); i++ {
265271
storage := rc.ConfigChain[i].Config.Storage
266272

267273
if storage.ResetPartitionsUuidsType != imagecustomizerapi.ResetPartitionsUuidsTypeDefault ||
@@ -271,12 +277,16 @@ func resolveStorage(rc *ResolvedConfig) (imagecustomizerapi.Storage, error) {
271277
len(storage.Verity) > 0 ||
272278
storage.ReinitializeVerity != imagecustomizerapi.ReinitializeVerityTypeDefault {
273279

274-
return imagecustomizerapi.Storage{}, ErrStorageInBaseConfig
280+
foundStorage = storage
281+
foundStorageCount += 1
275282
}
276283
}
277284

278-
topLevelConfig := rc.ConfigChain[len(rc.ConfigChain)-1]
279-
return topLevelConfig.Config.Storage, nil
285+
if foundStorageCount > 1 {
286+
return imagecustomizerapi.Storage{}, ErrStorageMoreThanOne
287+
}
288+
289+
return foundStorage, nil
280290
}
281291

282292
func ValidateConfigPostImageDownload(rc *ResolvedConfig) error {

‎toolkit/tools/pkg/imagecustomizerlib/createimage_test.go‎

Lines changed: 1 addition & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -252,23 +252,7 @@ func verifyCreateMinimalOs(t *testing.T, buildDir string, outputImageFilePath st
252252
distroHandler, err := NewDistroHandler(imageInfo.TargetOs())
253253
assert.NoError(t, err)
254254

255-
expectedVirtualSize := int64(0)
256-
switch imageInfo.Distro {
257-
case "azurelinux":
258-
switch imageInfo.Version {
259-
case "3.0":
260-
expectedVirtualSize = int64(1 * diskutils.GiB)
261-
262-
case "4.0":
263-
expectedVirtualSize = int64(3 * diskutils.GiB)
264-
265-
default:
266-
t.Fatalf("Unsupported AZL version for test (%s)", imageInfo.Version)
267-
}
268-
269-
default:
270-
t.Fatalf("Unsupported distro for test (%s)", imageInfo.Distro)
271-
}
255+
expectedVirtualSize := int64(3 * diskutils.GiB)
272256

273257
fileType, err := testutils.GetImageFileType(outputImageFilePath)
274258
assert.NoError(t, err)

‎toolkit/tools/pkg/imagecustomizerlib/createimagevalidation.go‎

Lines changed: 34 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -12,31 +12,36 @@ import (
1212
"github.com/microsoft/azure-linux-image-tools/toolkit/tools/imagecustomizerapi"
1313
)
1414

15-
func validateCreateImageSupportedFields(c *imagecustomizerapi.Config) error {
15+
func validateCreateImageSupportedFields(rc *ResolvedConfig) error {
1616
// Verify that the config file does not contain any fields that are not supported by the create subcommand.
17-
if c.Input != (imagecustomizerapi.Input{}) {
17+
if rc.InputImage != (imagecustomizerapi.InputImage{}) {
1818
return fmt.Errorf("input field is not supported by the create subcommand")
1919
}
20-
if c.Iso != nil {
21-
return fmt.Errorf("iso field is not supported by the create subcommand")
22-
}
23-
if c.Pxe != nil {
24-
return fmt.Errorf("pxe field is not supported by the create subcommand")
25-
}
2620

27-
if c.Storage.ResetPartitionsUuidsType != imagecustomizerapi.ResetPartitionsUuidsTypeDefault {
21+
if rc.Storage.ResetPartitionsUuidsType != imagecustomizerapi.ResetPartitionsUuidsTypeDefault {
2822
return fmt.Errorf("reset partitions uuids field is not supported by the create subcommand")
2923
}
3024

31-
if c.Storage.Verity != nil {
25+
if rc.Storage.Verity != nil {
3226
return fmt.Errorf("storage verity field is not supported by the create subcommand")
3327
}
3428

35-
if c.OS != nil {
36-
if err := validateCreateImageSupportedOsFields(c.OS); err != nil {
37-
return err
29+
if rc.Uki != nil {
30+
return fmt.Errorf("uki field is not supported by the create subcommand")
31+
}
32+
33+
if rc.SELinux != (imagecustomizerapi.SELinux{}) {
34+
return fmt.Errorf("selinux field is not supported by the create subcommand")
35+
}
36+
37+
for _, config := range rc.ConfigChain {
38+
if config.Config.OS != nil {
39+
if err := validateCreateImageSupportedOsFields(config.Config.OS); err != nil {
40+
return err
41+
}
3842
}
3943
}
44+
4045
return nil
4146
}
4247

@@ -49,14 +54,6 @@ func validateCreateImageSupportedOsFields(osConfig *imagecustomizerapi.OS) error
4954
return fmt.Errorf("os.additionalDirectories field is not supported by the create subcommand")
5055
}
5156

52-
if osConfig.Uki != nil {
53-
return fmt.Errorf("uki field is not supported by the create subcommand")
54-
}
55-
56-
if osConfig.SELinux != (imagecustomizerapi.SELinux{}) {
57-
return fmt.Errorf("selinux field is not supported by the create subcommand")
58-
}
59-
6057
if len(osConfig.Modules) > 0 {
6158
return fmt.Errorf("os.modules field is not supported by the create subcommand")
6259
}
@@ -70,18 +67,6 @@ func validateCreateImageSupportedOsFields(osConfig *imagecustomizerapi.OS) error
7067
func validateCreateImageConfig(ctx context.Context, baseConfigPath string, config *imagecustomizerapi.Config,
7168
options ImageCreateOptions,
7269
) (*ResolvedConfig, error) {
73-
err := validateCreateImageSupportedFields(config)
74-
if err != nil {
75-
return nil, fmt.Errorf("invalid config file (%s):\n%w", baseConfigPath, err)
76-
}
77-
78-
// Validate mandatory fields for creating a seed image
79-
err = validateCreateImageMandatoryFields(baseConfigPath, config, options.RpmsSources, options.ToolsDir)
80-
if err != nil {
81-
return nil, err
82-
}
83-
84-
// TODO: Validate for distro and release
8570
rc, err := ValidateConfig(ctx, baseConfigPath, config, true, false,
8671
imagecustomizerapi.ValidateResourceTypes{
8772
imagecustomizerapi.ValidateResourceTypeAll,
@@ -104,6 +89,17 @@ func validateCreateImageConfig(ctx context.Context, baseConfigPath string, confi
10489
"the 'create' feature is currently in preview; please add 'create' to 'previewFeatures' to enable it")
10590
}
10691

92+
err = validateCreateImageSupportedFields(rc)
93+
if err != nil {
94+
return nil, fmt.Errorf("invalid config file (%s):\n%w", baseConfigPath, err)
95+
}
96+
97+
// Validate mandatory fields for creating a seed image
98+
err = validateCreateImageMandatoryFields(baseConfigPath, rc)
99+
if err != nil {
100+
return nil, err
101+
}
102+
107103
if len(config.OS.Packages.Install) == 0 {
108104
return nil, fmt.Errorf(
109105
"no packages to install specified, please specify at least one package to install for a new image")
@@ -112,23 +108,21 @@ func validateCreateImageConfig(ctx context.Context, baseConfigPath string, confi
112108
return rc, nil
113109
}
114110

115-
func validateCreateImageMandatoryFields(baseConfigPath string, config *imagecustomizerapi.Config,
116-
rpmsSources []string, toolsDir string,
117-
) error {
111+
func validateCreateImageMandatoryFields(baseConfigPath string, rc *ResolvedConfig) error {
118112
// check if storage disks is not empty for creating a seed image
119-
if len(config.Storage.Disks) == 0 {
113+
if len(rc.Storage.Disks) == 0 {
120114
return fmt.Errorf("storage.disks field is required in the config file (%s)", baseConfigPath)
121115
}
122116

123117
// rpmSources must not be empty for creating a seed image
124-
if len(rpmsSources) == 0 {
118+
if len(rc.Options.RpmsSources) == 0 {
125119
return fmt.Errorf("rpm sources must be specified for creating a seed image")
126120
}
127121

128-
if toolsDir == "" {
122+
if rc.Options.ToolsDir == "" {
129123
return fmt.Errorf("tools directory is required for image creation")
130124
}
131-
err := validateToolsDir(toolsDir)
125+
err := validateToolsDir(rc.Options.ToolsDir)
132126
if err != nil {
133127
return err
134128
}

‎toolkit/tools/pkg/imagecustomizerlib/createimagevalidation_test.go‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -179,6 +179,7 @@ func TestValidateCreateImageConfig_EmptyConfig(t *testing.T) {
179179
}
180180

181181
options := ImageCreateOptions{
182+
OutputImageFile: "./out/image.vhdx",
182183
OutputImageFormat: "vhdx",
183184
BuildDir: "./",
184185
}
@@ -238,6 +239,7 @@ func TestValidateCreateImageConfig_InvalidFieldsVerityConfig(t *testing.T) {
238239

239240
options := ImageCreateOptions{
240241
RpmsSources: []string{testDir}, // Use the test directory as a dummy RPM source
242+
OutputImageFile: "./out/image.vhdx",
241243
OutputImageFormat: "vhdx",
242244
BuildDir: "./",
243245
PreviewFeatures: []imagecustomizerapi.PreviewFeature{
@@ -259,6 +261,7 @@ func TestValidateCreateImageConfig_InvalidFieldsOverlaysConfig(t *testing.T) {
259261

260262
options := ImageCreateOptions{
261263
RpmsSources: []string{testDir}, // Use the test directory as a dummy RPM source
264+
OutputImageFile: "./out/image.vhdx",
262265
OutputImageFormat: "vhdx",
263266
BuildDir: "./",
264267
PreviewFeatures: []imagecustomizerapi.PreviewFeature{

‎toolkit/tools/pkg/imagecustomizerlib/testdata/artifacts-output-amd64-azl4.yaml‎

Lines changed: 3 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -1,65 +1,16 @@
11
# Differs from artifacts-output-azl3 in its use of systemd-boot-unsigned (not systemd-boot) and grub2-efi-x64 (not grub2-efi-binary)
2-
storage:
3-
disks:
4-
- partitionTableType: gpt
5-
partitions:
6-
- id: esp
7-
type: esp
8-
size: 100M
9-
10-
- id: boot
11-
size: 1G
12-
13-
- id: rootfs
14-
size: 2G
15-
16-
bootType: efi
17-
18-
filesystems:
19-
- deviceId: esp
20-
type: fat32
21-
mountPoint:
22-
path: /boot/efi
23-
options: umask=0077
24-
25-
- deviceId: boot
26-
type: ext4
27-
mountPoint:
28-
path: /boot
29-
30-
- deviceId: rootfs
31-
type: ext4
32-
mountPoint:
33-
path: /
34-
352
previewFeatures:
36-
- uki
37-
- output-artifacts
3+
- base-configs
384

39-
output:
40-
artifacts:
41-
items:
42-
- ukis
43-
- shim
44-
- bootloader
45-
path: ./output
5+
baseConfigs:
6+
- path: artifacts-output-base.yaml
467

478
os:
48-
bootloader:
49-
resetType: hard-reset
50-
51-
uki:
52-
mode: create
53-
549
packages:
5510
install:
5611
- systemd-boot-unsigned
5712
- openssh-server
5813

59-
services:
60-
enable:
61-
- sshd
62-
6314
scripts:
6415
postCustomization:
6516
- content: |

0 commit comments

Comments
 (0)