From 30731cb0c8bb50aea6b548c3c05066b280c53cdf Mon Sep 17 00:00:00 2001 From: Nikita Dubrovskii Date: Tue, 28 Oct 2025 12:53:10 +0100 Subject: [PATCH 01/31] mantle/platform/qemu: add MachineBuilder for customizable machine creation Introduce MachineBuilder struct to allow callers to customize QEMU machine creation by providing hooks for builder initialization, disk setup, network configuration. This will be used to fold 'testiso' tests into the standard 'kola run', where custom machine creation logic is needed for ISO-based installations. --- mantle/platform/machine/qemu/cluster.go | 314 ++++++++++++++---------- 1 file changed, 191 insertions(+), 123 deletions(-) diff --git a/mantle/platform/machine/qemu/cluster.go b/mantle/platform/machine/qemu/cluster.go index 296cf40353..162a96e891 100644 --- a/mantle/platform/machine/qemu/cluster.go +++ b/mantle/platform/machine/qemu/cluster.go @@ -42,6 +42,17 @@ type Cluster struct { tearingDown atomic.Bool } +// MachineBuilder provides hooks to customize machine creation. +// All fields are optional; if nil, default implementations will be used. +type MachineBuilder struct { + // InitBuilder configures the QemuBuilder with architecture, firmware, memory, etc. + InitBuilder func(options platform.MachineOptions, builder *platform.QemuBuilder) error + // SetupDisks configures the primary disk and any additional disks. + SetupDisks func(options platform.MachineOptions, builder *platform.QemuBuilder) error + // SetupNetwork configures networking including port forwarding and additional NICs. + SetupNetwork func(options platform.MachineOptions, builder *platform.QemuBuilder) error +} + func (qc *Cluster) NewMachine(userdata *conf.UserData) (platform.Machine, error) { return qc.NewMachineWithOptions(userdata, platform.MachineOptions{}) } @@ -50,41 +61,27 @@ func (qc *Cluster) NewMachineWithOptions(userdata *conf.UserData, options platfo if options.InstanceType != "" { return nil, errors.New("platform qemu does not support changing instance types") } - id := uuid.New() + return qc.NewMachineWithBuilder(userdata, options, nil) +} - dir := filepath.Join(qc.RuntimeConf().OutputDir, id) - if err := os.Mkdir(dir, 0777); err != nil { - return nil, err - } +// NewMachineWithBuilder creates a new machine with custom builder hooks. +// If builder is nil or any of its fields are nil, default implementations are used. +func (qc *Cluster) NewMachineWithBuilder(userdata any, options platform.MachineOptions, builder *MachineBuilder) (platform.Machine, error) { + // Use default builder if none provided + builder = qc.ensureBuilderDefaults(builder) - config, err := qc.RenderUserDataIfNeeded(userdata) + qm, config, err := qc.createMachine(userdata) if err != nil { return nil, err } - journal, err := platform.NewJournal(dir) - if err != nil { + qemuBuilder := platform.NewQemuBuilder() + qemuBuilder.SetConfig(config) + defer qemuBuilder.Close() + if err := builder.InitBuilder(options, qemuBuilder); err != nil { return nil, err } - qm := &machine{ - qc: qc, - id: id, - journal: journal, - consolePath: filepath.Join(dir, "console.txt"), - } - - builder := platform.NewQemuBuilder() - if options.DisablePDeathSig { - builder.Pdeathsig = false - } - - if qc.flight.opts.SecureExecution { - if err := builder.SetSecureExecution(qc.flight.opts.SecureExecutionIgnitionPubKey, qc.flight.opts.SecureExecutionHostKey, config); err != nil { - return nil, err - } - } - // If requested, bind mount Host (COSA) directories into the machine for use. // These could either come in as MachineOptions OR via the flight options // (CLI --qemu-bind-ro option). @@ -99,121 +96,40 @@ func (qc *Cluster) NewMachineWithOptions(userdata *conf.UserData, options platfo return nil, err } readonly := true - builder.MountHost(src, dest, readonly) + qemuBuilder.MountHost(src, dest, readonly) config.MountHost(dest, readonly) } - builder.SetConfig(config) - defer builder.Close() - builder.UUID = qm.id - if qc.flight.opts.Arch != "" { - if err := builder.SetArchitecture(qc.flight.opts.Arch); err != nil { - return nil, err - } - } - if qc.flight.opts.Firmware != "" { - builder.Firmware = qc.flight.opts.Firmware - } - builder.Swtpm = qc.flight.opts.Swtpm - builder.Hostname = fmt.Sprintf("qemu%d", qc.BaseCluster.AllocateMachineSerial()) - builder.ConsoleFile = qm.consolePath - - if qc.flight.opts.Memory != "" { - memory, err := strconv.ParseInt(qc.flight.opts.Memory, 10, 32) - if err != nil { - return nil, errors.Wrapf(err, "parsing memory option") - } - builder.MemoryMiB = int(memory) - } else if options.MinMemory != 0 { - builder.MemoryMiB = options.MinMemory - } else if qc.flight.opts.SecureExecution { - builder.MemoryMiB = 4096 // SE needs at least 4GB - } - - builder.NumaNodes = options.NumaNodes - var primaryDisk platform.Disk - if options.PrimaryDisk != "" { - var diskp *platform.Disk - if diskp, err = platform.ParseDisk(options.PrimaryDisk, true); err != nil { - return nil, errors.Wrapf(err, "parsing primary disk spec '%s'", options.PrimaryDisk) - } - primaryDisk = *diskp - } - - if qc.flight.opts.Cex || options.Cex { - if err := builder.AddCexDevice(); err != nil { - return nil, err - } - } - - if qc.flight.opts.Nvme || options.Nvme { - primaryDisk.Channel = "nvme" - } - if qc.flight.opts.Native4k { - primaryDisk.SectorSize = 4096 - } else if qc.flight.opts.Disk512e { - primaryDisk.SectorSize = 4096 - primaryDisk.LogicalSectorSize = 512 - } - if options.MultiPathDisk || qc.flight.opts.MultiPathDisk { - primaryDisk.MultiPathDisk = true - } - if options.MinDiskSize > 0 { - primaryDisk.Size = fmt.Sprintf("%dG", options.MinDiskSize) - } else if qc.flight.opts.DiskSize != "" { - primaryDisk.Size = qc.flight.opts.DiskSize - } - primaryDisk.BackingFile = qc.flight.opts.DiskImage - if options.OverrideBackingFile != "" { - primaryDisk.BackingFile = options.OverrideBackingFile - } + qemuBuilder.UUID = qm.id + qemuBuilder.ConsoleFile = qm.consolePath + qemuBuilder.NumaNodes = options.NumaNodes - if err = builder.AddBootDisk(&primaryDisk); err != nil { + if err := builder.SetupDisks(options, qemuBuilder); err != nil { return nil, err } - if err = builder.AddDisksFromSpecs(options.AdditionalDisks); err != nil { + if err := builder.SetupNetwork(options, qemuBuilder); err != nil { return nil, err } - if len(options.HostForwardPorts) > 0 { - builder.EnableUsermodeNetworking(options.HostForwardPorts, "") - } else { - h := []platform.HostForwardPort{ - {Service: "ssh", HostPort: 0, GuestPort: 22}, + // S390x specific stuff + if qc.flight.opts.SecureExecution { + if err := qemuBuilder.SetSecureExecution(qc.flight.opts.SecureExecutionIgnitionPubKey, qc.flight.opts.SecureExecutionHostKey, config); err != nil { + return nil, err } - builder.EnableUsermodeNetworking(h, "") - } - if options.AdditionalNics > 0 { - builder.AddAdditionalNics(options.AdditionalNics) - } - if options.AppendKernelArgs != "" { - builder.AppendKernelArgs = options.AppendKernelArgs } - if options.AppendFirstbootKernelArgs != "" { - builder.AppendFirstbootKernelArgs = options.AppendFirstbootKernelArgs - } - if !qc.RuntimeConf().InternetAccess { - builder.RestrictNetworking = true - } - if options.Firmware != "" { - builder.Firmware = options.Firmware + if qc.flight.opts.Cex || options.Cex { + if err := qemuBuilder.AddCexDevice(); err != nil { + return nil, err + } } - inst, err := builder.Exec() + inst, err := qemuBuilder.Exec() if err != nil { return nil, err } qm.inst = inst - err = util.Retry(6, 5*time.Second, func() error { - var err error - qm.ip, err = inst.SSHAddress() - if err != nil { - return err - } - return nil - }) - if err != nil { + if err := qc.waitForSSHAddress(qm, inst); err != nil { return nil, err } @@ -265,3 +181,155 @@ func (qc *Cluster) RenderUserDataIfNeeded(userdata any) (*conf.Conf, error) { } return config, nil } + +// ensures all builder callbacks have default implementations. +func (qc *Cluster) ensureBuilderDefaults(builder *MachineBuilder) *MachineBuilder { + if builder == nil { + builder = &MachineBuilder{} + } + + if builder.InitBuilder == nil { + builder.InitBuilder = qc.InitDefaultBuilder + } + if builder.SetupDisks == nil { + builder.SetupDisks = qc.SetupDefaultDisks + } + if builder.SetupNetwork == nil { + builder.SetupNetwork = qc.SetupDefaultNetwork + } + + return builder +} + +// createMachine creates a new machine instance with its directory, config, and journal. +func (qc *Cluster) createMachine(userdata any) (*machine, *conf.Conf, error) { + id := uuid.New() + + dir := filepath.Join(qc.RuntimeConf().OutputDir, id) + if err := os.Mkdir(dir, 0777); err != nil { + return nil, nil, err + } + + config, err := qc.RenderUserDataIfNeeded(userdata) + if err != nil { + return nil, nil, err + } + + journal, err := platform.NewJournal(dir) + if err != nil { + return nil, nil, err + } + + qm := &machine{ + qc: qc, + id: id, + journal: journal, + consolePath: filepath.Join(dir, "console.txt"), + } + + return qm, config, nil +} + +// waitForSSHAddress waits for the machine to provide an SSH address. +func (qc *Cluster) waitForSSHAddress(qm *machine, inst *platform.QemuInstance) error { + return util.Retry(6, 5*time.Second, func() error { + var err error + qm.ip, err = inst.SSHAddress() + return err + }) +} + +func (qc *Cluster) InitDefaultBuilder(options platform.MachineOptions, builder *platform.QemuBuilder) error { + if options.DisablePDeathSig { + builder.Pdeathsig = false + } + if qc.flight.opts.Arch != "" { + if err := builder.SetArchitecture(qc.flight.opts.Arch); err != nil { + return err + } + } + if qc.flight.opts.Firmware != "" { + builder.Firmware = qc.flight.opts.Firmware + } + if qc.flight.opts.Memory != "" { + memory, err := strconv.ParseInt(qc.flight.opts.Memory, 10, 32) + if err != nil { + return errors.Wrapf(err, "parsing memory option") + } + builder.MemoryMiB = int(memory) + } else if options.MinMemory != 0 { + builder.MemoryMiB = options.MinMemory + } else if qc.flight.opts.SecureExecution { + builder.MemoryMiB = 4096 // SE needs at least 4GB + } + builder.Swtpm = qc.flight.opts.Swtpm + builder.Hostname = fmt.Sprintf("qemu%d", qc.BaseCluster.AllocateMachineSerial()) + if options.Firmware != "" { + builder.Firmware = options.Firmware + } + if options.AppendKernelArgs != "" { + builder.AppendKernelArgs = options.AppendKernelArgs + } + if options.AppendFirstbootKernelArgs != "" { + builder.AppendFirstbootKernelArgs = options.AppendFirstbootKernelArgs + } + + return nil +} + +func (qc *Cluster) SetupDefaultDisks(options platform.MachineOptions, builder *platform.QemuBuilder) error { + var primaryDisk platform.Disk + if options.PrimaryDisk != "" { + diskp, err := platform.ParseDisk(options.PrimaryDisk, true) + if err != nil { + return errors.Wrapf(err, "parsing primary disk spec '%s'", options.PrimaryDisk) + } + primaryDisk = *diskp + } + if qc.flight.opts.Nvme || options.Nvme { + primaryDisk.Channel = "nvme" + } + if qc.flight.opts.Native4k { + primaryDisk.SectorSize = 4096 + } else if qc.flight.opts.Disk512e { + primaryDisk.SectorSize = 4096 + primaryDisk.LogicalSectorSize = 512 + } + if options.MultiPathDisk || qc.flight.opts.MultiPathDisk { + primaryDisk.MultiPathDisk = true + } + if options.MinDiskSize > 0 { + primaryDisk.Size = fmt.Sprintf("%dG", options.MinDiskSize) + } else if qc.flight.opts.DiskSize != "" { + primaryDisk.Size = qc.flight.opts.DiskSize + } + primaryDisk.BackingFile = qc.flight.opts.DiskImage + if options.OverrideBackingFile != "" { + primaryDisk.BackingFile = options.OverrideBackingFile + } + if err := builder.AddBootDisk(&primaryDisk); err != nil { + return err + } + if err := builder.AddDisksFromSpecs(options.AdditionalDisks); err != nil { + return err + } + return nil +} + +func (qc *Cluster) SetupDefaultNetwork(options platform.MachineOptions, builder *platform.QemuBuilder) error { + if len(options.HostForwardPorts) > 0 { + builder.EnableUsermodeNetworking(options.HostForwardPorts, "") + } else { + h := []platform.HostForwardPort{ + {Service: "ssh", HostPort: 0, GuestPort: 22}, + } + builder.EnableUsermodeNetworking(h, "") + } + if options.AdditionalNics > 0 { + builder.AddAdditionalNics(options.AdditionalNics) + } + if !qc.RuntimeConf().InternetAccess { + builder.RestrictNetworking = true + } + return nil +} From 0158cb87271df01f1c5f79368aac0b78e79e2e96 Mon Sep 17 00:00:00 2001 From: Nikita Dubrovskii Date: Tue, 28 Oct 2025 12:57:32 +0100 Subject: [PATCH 02/31] kola/tests/iso: fold testiso live-login.* tests into 'kola run' This migrates the iso-live-login tests from the standalone 'kola testiso' command into the standard 'kola run' framework, making them available alongside other kola tests. Changes: - Moved iso-live-login* tests implementation from cmd/kola/testiso.go to mantle/kola/tests/iso/live-login.go - Removed 4k test variants (iso-live-login.4k.uefi) as they were non-functional for this test type Why 4k tests were dropped: The 4k flag in testiso only affects disk sector size during disk creation. However, the live-login test boots directly from ISO without creating any disks, so the 4k flag had no effect. The 4k test variants were redundant and ran identical code to their non-4k counterparts. --- mantle/cmd/kola/testiso.go | 40 -------- mantle/kola/registry/registry.go | 1 + mantle/kola/tests/iso/live-login.go | 143 ++++++++++++++++++++++++++++ 3 files changed, 144 insertions(+), 40 deletions(-) create mode 100644 mantle/kola/tests/iso/live-login.go diff --git a/mantle/cmd/kola/testiso.go b/mantle/cmd/kola/testiso.go index 199091e26b..9eff5feb41 100644 --- a/mantle/cmd/kola/testiso.go +++ b/mantle/cmd/kola/testiso.go @@ -82,10 +82,6 @@ var ( "iso-as-disk.uefi-secure", "iso-as-disk.4k.uefi", "iso-install.bios", - "iso-live-login.bios", - "iso-live-login.uefi", - "iso-live-login.uefi-secure", - "iso-live-login.4k.uefi", "iso-offline-install.bios", "iso-offline-install.mpath.bios", "iso-offline-install-fromram.4k.uefi", @@ -102,7 +98,6 @@ var ( "pxe-online-install.4k.uefi", } tests_s390x = []string{ - "iso-live-login.s390fw", "iso-offline-install.s390fw", "iso-offline-install.mpath.s390fw", "iso-offline-install.4k.s390fw", @@ -117,7 +112,6 @@ var ( //"iso-offline-install-iscsi.manual.s390fw", } tests_ppc64le = []string{ - "iso-live-login.ppcfw", "iso-offline-install.ppcfw", "iso-offline-install.mpath.ppcfw", "iso-offline-install-fromram.4k.ppcfw", @@ -133,8 +127,6 @@ var ( //"iso-offline-install-iscsi.manual.ppcfw", } tests_aarch64 = []string{ - "iso-live-login.uefi", - "iso-live-login.4k.uefi", "iso-offline-install.uefi", "iso-offline-install.mpath.uefi", "iso-offline-install-fromram.4k.uefi", @@ -631,8 +623,6 @@ func runTestIso(cmd *cobra.Command, args []string) (err error) { duration, err = testPXE(ctx, inst, filepath.Join(outputDir, test)) case "iso-as-disk": duration, err = testAsDisk(ctx, filepath.Join(outputDir, test)) - case "iso-live-login": - duration, err = testLiveLogin(ctx, filepath.Join(outputDir, test)) case "iso-fips": duration, err = testLiveFIPS(ctx, filepath.Join(outputDir, test)) case "iso-install", "iso-offline-install", "iso-offline-install-fromram": @@ -1002,36 +992,6 @@ RequiredBy=fips-signal-ok.service return awaitCompletion(ctx, mach, outdir, completionChannel, nil, []string{liveOKSignal}) } -func testLiveLogin(ctx context.Context, outdir string) (time.Duration, error) { - builddir := kola.CosaBuild.Dir - isopath := filepath.Join(builddir, kola.CosaBuild.Meta.BuildArtifacts.LiveIso.Path) - builder, err := newBaseQemuBuilder(outdir) - if err != nil { - return 0, err - } - defer builder.Close() - // Drop the bootindex bit (applicable to all arches except s390x and ppc64le); we want it to be the default - if err := builder.AddIso(isopath, "", false); err != nil { - return 0, err - } - - completionChannel, err := builder.VirtioChannelRead("coreos.liveiso-success") - if err != nil { - return 0, err - } - - // No network device to test https://github.com/coreos/fedora-coreos-config/pull/326 - builder.Append("-net", "none") - - mach, err := builder.Exec() - if err != nil { - return 0, errors.Wrapf(err, "running iso") - } - defer mach.Destroy() - - return awaitCompletion(ctx, mach, outdir, completionChannel, nil, []string{"coreos-liveiso-success"}) -} - func testAsDisk(ctx context.Context, outdir string) (time.Duration, error) { builddir := kola.CosaBuild.Dir isopath := filepath.Join(builddir, kola.CosaBuild.Meta.BuildArtifacts.LiveIso.Path) diff --git a/mantle/kola/registry/registry.go b/mantle/kola/registry/registry.go index c22c474e86..94a1ab66c3 100644 --- a/mantle/kola/registry/registry.go +++ b/mantle/kola/registry/registry.go @@ -7,6 +7,7 @@ import ( _ "github.com/coreos/coreos-assembler/mantle/kola/tests/etcd" _ "github.com/coreos/coreos-assembler/mantle/kola/tests/fips" _ "github.com/coreos/coreos-assembler/mantle/kola/tests/ignition" + _ "github.com/coreos/coreos-assembler/mantle/kola/tests/iso" _ "github.com/coreos/coreos-assembler/mantle/kola/tests/metadata" _ "github.com/coreos/coreos-assembler/mantle/kola/tests/misc" _ "github.com/coreos/coreos-assembler/mantle/kola/tests/ostree" diff --git a/mantle/kola/tests/iso/live-login.go b/mantle/kola/tests/iso/live-login.go new file mode 100644 index 0000000000..1d41baef1a --- /dev/null +++ b/mantle/kola/tests/iso/live-login.go @@ -0,0 +1,143 @@ +package testiso + +import ( + "bufio" + "fmt" + "io" + "path/filepath" + "strings" + "time" + + "github.com/coreos/coreos-assembler/mantle/kola" + "github.com/coreos/coreos-assembler/mantle/platform" + coreosarch "github.com/coreos/stream-metadata-go/arch" + "github.com/pkg/errors" + + "github.com/coreos/coreos-assembler/mantle/kola/cluster" + "github.com/coreos/coreos-assembler/mantle/kola/register" + "github.com/coreos/coreos-assembler/mantle/platform/conf" + "github.com/coreos/coreos-assembler/mantle/platform/machine/qemu" +) + +var ( + tests_live_login_x86_64 = []string{ + "live-login.bios", + "live-login.uefi", + "live-login.uefi-secure", + } + tests_live_login_aarch64 = []string{ + "live-login.uefi", + } + tests_live_login_ppc64le = []string{ + "live-login", + } + tests_live_login_s390x = []string{ + "live-login", + } +) + +func getAllLiveLoginTests() []string { + arch := coreosarch.CurrentRpmArch() + switch arch { + case "x86_64": + return tests_live_login_x86_64 + case "aarch64": + return tests_live_login_aarch64 + case "ppc64le": + return tests_live_login_ppc64le + case "s390x": + return tests_live_login_s390x + default: + return []string{} + } +} + +func init() { + for _, testName := range getAllLiveLoginTests() { + var firmware string + if strings.Contains(testName, "uefi-secure") { + firmware = "uefi-secure" + } else if strings.Contains(testName, "uefi") { + firmware = "uefi" + } + + register.RegisterTest(®ister.Test{ + Run: func(c cluster.TestCluster) { + testLiveLogin(c, firmware) + }, + ClusterSize: 0, + Name: "iso." + testName, + Description: "Verify ISO live login works.", + Flags: []register.Flag{}, + Platforms: []string{"qemu"}, + }) + } +} + +func testLiveLogin(c cluster.TestCluster, firmware string) { + if kola.CosaBuild.Meta.BuildArtifacts.LiveIso == nil || kola.CosaBuild.Meta.BuildArtifacts.LiveKernel == nil { + c.Fatalf("Build %s is missing live artifacts\n", kola.CosaBuild.Meta.Name) + } + + butane := conf.Butane(` +variant: fcos +version: 1.1.0`) + + errchan := make(chan error) + + setupDisks := func(_ platform.MachineOptions, builder *platform.QemuBuilder) error { + // https://github.com/coreos/fedora-coreos-config/blob/testing-devel/overlay.d/05core/usr/lib/systemd/system/coreos-liveiso-success.service + output, err := builder.VirtioChannelRead("coreos.liveiso-success") + if err != nil { + return errors.Wrap(err, "setting up virtio-serial channel") + } + + // Read line in a goroutine and send errors to channel + go func() { + exp := "coreos-liveiso-success" + line, err := bufio.NewReader(output).ReadString('\n') + if err != nil { + if err == io.EOF { + // this may be from QEMU getting killed or exiting; wait a bit + // to give a chance for .Wait() above to feed the channel with a + // better error + time.Sleep(1 * time.Second) + errchan <- fmt.Errorf("Got EOF from completion channel, %s expected", exp) + } else { + errchan <- errors.Wrapf(err, "reading from completion channel") + } + return + } + line = strings.TrimSpace(line) + if line != exp { + errchan <- fmt.Errorf("Unexpected string from completion channel: %q, expected: %q", line, exp) + return + } + // OK! + errchan <- nil + }() + + isopath := filepath.Join(kola.CosaBuild.Dir, kola.CosaBuild.Meta.BuildArtifacts.LiveIso.Path) + // Drop the bootindex bit (applicable to all arches except s390x and ppc64le); we want it to be the default + return builder.AddIso(isopath, "", false) + } + + switch pc := c.Cluster.(type) { + case *qemu.Cluster: + options := platform.MachineOptions{Firmware: firmware} + builder := &qemu.MachineBuilder{ + SetupDisks: setupDisks, + } + _, err := pc.NewMachineWithBuilder(butane, options, builder) + if err != nil { + c.Fatalf("Unable to create test machine: %v", err) + } + default: + c.Fatalf("Unsupported cluster type") + } + + err := <-errchan + if err != nil { + c.Fatal(err) + } +} From 417e0acc7695b2f825133ee7fc1667e1ca9077b0 Mon Sep 17 00:00:00 2001 From: Nikita Dubrovskii Date: Thu, 13 Nov 2025 15:07:56 +0100 Subject: [PATCH 03/31] mantle/platform: move metal.go to machine/qemu/ Relocate metal.go to machine/qemu/ directory where it belongs, as it contains QEMU-specific code for ISO installations and is only used by ISO tests, not as a general platform implementation. --- mantle/cmd/kola/testiso.go | 11 ++++---- mantle/platform/{ => machine/qemu}/metal.go | 31 ++++++++------------- mantle/platform/qemu.go | 9 +++++- 3 files changed, 26 insertions(+), 25 deletions(-) rename mantle/platform/{ => machine/qemu}/metal.go (96%) diff --git a/mantle/cmd/kola/testiso.go b/mantle/cmd/kola/testiso.go index 9eff5feb41..2a94e71c0c 100644 --- a/mantle/cmd/kola/testiso.go +++ b/mantle/cmd/kola/testiso.go @@ -35,6 +35,7 @@ import ( "github.com/coreos/coreos-assembler/mantle/harness/reporters" "github.com/coreos/coreos-assembler/mantle/harness/testresult" "github.com/coreos/coreos-assembler/mantle/platform/conf" + "github.com/coreos/coreos-assembler/mantle/platform/machine/qemu" "github.com/coreos/coreos-assembler/mantle/util" coreosarch "github.com/coreos/stream-metadata-go/arch" "github.com/pkg/errors" @@ -381,7 +382,7 @@ func getAllTests(build *util.LocalBuild) []string { } func newBaseQemuBuilder(outdir string) (*platform.QemuBuilder, error) { - builder := platform.NewMetalQemuBuilderDefault() + builder := qemu.NewMetalQemuBuilderDefault() if enableUefiSecure { builder.Firmware = "uefi-secure" } else if enableUefi { @@ -548,7 +549,7 @@ func runTestIso(cmd *cobra.Command, args []string) (err error) { } }() - baseInst := platform.Install{ + baseInst := qemu.Install{ CosaBuild: kola.CosaBuild, NmKeyfiles: make(map[string]string), } @@ -798,7 +799,7 @@ func printResult(test string, duration time.Duration, err error) bool { return false } -func testPXE(ctx context.Context, inst platform.Install, outdir string) (time.Duration, error) { +func testPXE(ctx context.Context, inst qemu.Install, outdir string) (time.Duration, error) { if addNmKeyfile { return 0, errors.New("--add-nm-keyfile not yet supported for PXE") } @@ -862,7 +863,7 @@ func testPXE(ctx context.Context, inst platform.Install, outdir string) (time.Du return awaitCompletion(ctx, mach.QemuInst, outdir, completionChannel, mach.BootStartedErrorChannel, []string{liveOKSignal, signalCompleteString}) } -func testLiveIso(ctx context.Context, inst platform.Install, outdir string, minimal bool) (time.Duration, error) { +func testLiveIso(ctx context.Context, inst qemu.Install, outdir string, minimal bool) (time.Duration, error) { tmpd, err := os.MkdirTemp("", "kola-testiso") if err != nil { return 0, err @@ -1047,7 +1048,7 @@ func testAsDisk(ctx context.Context, outdir string) (time.Duration, error) { // 6 - /var/nested-ign.json contains an ignition config: // - when the system is booted, write a success string to /dev/virtio-ports/testisocompletion // - as this serial device is mapped to the host serial device, the test concludes -func testLiveInstalliscsi(ctx context.Context, inst platform.Install, outdir string, butane string) (time.Duration, error) { +func testLiveInstalliscsi(ctx context.Context, inst qemu.Install, outdir string, butane string) (time.Duration, error) { builddir := kola.CosaBuild.Dir isopath := filepath.Join(builddir, kola.CosaBuild.Meta.BuildArtifacts.LiveIso.Path) diff --git a/mantle/platform/metal.go b/mantle/platform/machine/qemu/metal.go similarity index 96% rename from mantle/platform/metal.go rename to mantle/platform/machine/qemu/metal.go index 02a9da88c6..4c1dc4fec7 100644 --- a/mantle/platform/metal.go +++ b/mantle/platform/machine/qemu/metal.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package platform +package qemu import ( "bufio" @@ -25,6 +25,7 @@ import ( "strings" "time" + "github.com/coreos/coreos-assembler/mantle/platform" coreosarch "github.com/coreos/stream-metadata-go/arch" "github.com/pkg/errors" "gopkg.in/yaml.v2" @@ -45,14 +46,6 @@ const ( var baseKargs = []string{"rd.neednet=1", "ip=dhcp", "ignition.firstboot", "ignition.platform.id=metal"} var ( - // TODO expose this as an API that can be used by cosa too - consoleKernelArgument = map[string]string{ - "x86_64": "ttyS0,115200n8", - "ppc64le": "hvc0", - "aarch64": "ttyAMA0", - "s390x": "ttysclp0", - } - bootStartedUnit = fmt.Sprintf(`[Unit] Description=TestISO Boot Started Requires=dev-virtio\\x2dports-bootstarted.device @@ -69,8 +62,8 @@ var ( // NewMetalQemuBuilderDefault returns a QEMU builder instance with some // defaults set up for bare metal. -func NewMetalQemuBuilderDefault() *QemuBuilder { - builder := NewQemuBuilder() +func NewMetalQemuBuilderDefault() *platform.QemuBuilder { + builder := platform.NewQemuBuilder() // https://github.com/coreos/fedora-coreos-tracker/issues/388 // https://github.com/coreos/fedora-coreos-docs/pull/46 builder.MemoryMiB = 4096 @@ -79,7 +72,7 @@ func NewMetalQemuBuilderDefault() *QemuBuilder { type Install struct { CosaBuild *util.LocalBuild - Builder *QemuBuilder + Builder *platform.QemuBuilder Insecure bool Native4k bool MultiPathDisk bool @@ -94,7 +87,7 @@ type Install struct { type InstalledMachine struct { Tempdir string - QemuInst *QemuInstance + QemuInst *platform.QemuInstance BootStartedErrorChannel chan error } @@ -123,7 +116,7 @@ func (inst *Install) PXE(kargs []string, liveIgnition, ignition conf.Conf, offli } installerConfig := installerConfig{ - Console: []string{consoleKernelArgument[coreosarch.CurrentRpmArch()]}, + Console: []string{platform.ConsoleKernelArgument[coreosarch.CurrentRpmArch()]}, AppendKargs: renderCosaTestIsoDebugKargs(), } installerConfigData, err := yaml.Marshal(installerConfig) @@ -181,7 +174,7 @@ type pxeSetup struct { type installerRun struct { inst *Install - builder *QemuBuilder + builder *platform.QemuBuilder builddir string tempdir string @@ -348,7 +341,7 @@ func (inst *Install) setup(kern *kernelSetup) (*installerRun, error) { } func renderBaseKargs() []string { - return append(baseKargs, fmt.Sprintf("console=%s", consoleKernelArgument[coreosarch.CurrentRpmArch()])) + return append(baseKargs, fmt.Sprintf("console=%s", platform.ConsoleKernelArgument[coreosarch.CurrentRpmArch()])) } func renderInstallKargs(t *installerRun, offline bool) []string { @@ -469,7 +462,7 @@ func (t *installerRun) completePxeSetup(kargs []string) error { return nil } -func switchBootOrderSignal(qinst *QemuInstance, bootstartedchan *os.File, booterrchan *chan error) { +func switchBootOrderSignal(qinst *platform.QemuInstance, bootstartedchan *os.File, booterrchan *chan error) { *booterrchan = make(chan error) go func() { err := qinst.Wait() @@ -529,7 +522,7 @@ func cat(outfile string, infiles ...string) error { return nil } -func (t *installerRun) run() (*QemuInstance, error) { +func (t *installerRun) run() (*platform.QemuInstance, error) { builder := t.builder netdev := fmt.Sprintf("%s,netdev=mynet0,mac=52:54:00:12:34:56", t.pxe.networkdevice) if t.pxe.bootindex == "" { @@ -626,7 +619,7 @@ func (inst *Install) InstallViaISOEmbed(kargs []string, liveIgnition, targetIgni // XXX: https://github.com/coreos/coreos-installer/issues/1171 if coreosarch.CurrentRpmArch() != "s390x" { - installerConfig.Console = []string{consoleKernelArgument[coreosarch.CurrentRpmArch()]} + installerConfig.Console = []string{platform.ConsoleKernelArgument[coreosarch.CurrentRpmArch()]} } if inst.MultiPathDisk { diff --git a/mantle/platform/qemu.go b/mantle/platform/qemu.go index d9bf41f591..91f519db1a 100644 --- a/mantle/platform/qemu.go +++ b/mantle/platform/qemu.go @@ -57,6 +57,13 @@ import ( var ( // ErrInitramfsEmergency is the marker error returned upon node blocking in emergency mode in initramfs. ErrInitramfsEmergency = errors.New("entered emergency.target in initramfs") + + ConsoleKernelArgument = map[string]string{ + "x86_64": "ttyS0,115200n8", + "ppc64le": "hvc0", + "aarch64": "ttyAMA0", + "s390x": "ttysclp0", + } ) // HostForwardPort contains details about port-forwarding for the VM. @@ -1612,7 +1619,7 @@ func (builder *QemuBuilder) setupIso() error { if kargsSupported, err := coreosInstallerSupportsISOKargs(); err != nil { return err } else if kargsSupported { - allargs := fmt.Sprintf("console=%s %s", consoleKernelArgument[coreosarch.CurrentRpmArch()], builder.AppendKernelArgs) + allargs := fmt.Sprintf("console=%s %s", ConsoleKernelArgument[coreosarch.CurrentRpmArch()], builder.AppendKernelArgs) instCmdKargs := exec.Command("coreos-installer", "iso", "kargs", "modify", "--append", allargs, isoEmbeddedPath) var stderrb bytes.Buffer instCmdKargs.Stderr = &stderrb From d87650e667c85f886b6b72b2b2b0d947652e9b3e Mon Sep 17 00:00:00 2001 From: Nikita Dubrovskii Date: Thu, 13 Nov 2025 16:20:57 +0100 Subject: [PATCH 04/31] mantle/platform/machine/qemu: merge InstalledMachine into machine struct Temporarily consolidate InstalledMachine into machine struct to simplify the codebase during the testiso migration. This prepares for migrating ISO installation tests from the testiso command into the standard 'kola run' test framework. This merge is temporary and will be refactored once the migration is complete. --- mantle/cmd/kola/testiso.go | 14 ++--- mantle/platform/machine/qemu/machine.go | 75 ++++++++++++++++++++----- mantle/platform/machine/qemu/metal.go | 39 ++++--------- 3 files changed, 78 insertions(+), 50 deletions(-) diff --git a/mantle/cmd/kola/testiso.go b/mantle/cmd/kola/testiso.go index 2a94e71c0c..1ee9f9a5f1 100644 --- a/mantle/cmd/kola/testiso.go +++ b/mantle/cmd/kola/testiso.go @@ -854,13 +854,9 @@ func testPXE(ctx context.Context, inst qemu.Install, outdir string) (time.Durati if err != nil { return 0, errors.Wrapf(err, "running PXE") } - defer func() { - if err := mach.Destroy(); err != nil { - plog.Errorf("Failed to destroy PXE: %v", err) - } - }() + defer mach.Destroy() - return awaitCompletion(ctx, mach.QemuInst, outdir, completionChannel, mach.BootStartedErrorChannel, []string{liveOKSignal, signalCompleteString}) + return awaitCompletion(ctx, mach.Instance(), outdir, completionChannel, mach.BootStartedErrorChannel(), []string{liveOKSignal, signalCompleteString}) } func testLiveIso(ctx context.Context, inst qemu.Install, outdir string, minimal bool) (time.Duration, error) { @@ -925,12 +921,14 @@ func testLiveIso(ctx context.Context, inst qemu.Install, outdir string, minimal return 0, errors.Wrapf(err, "running iso install") } defer func() { - if err := mach.Destroy(); err != nil { + err := mach.DeleteTempdir() + mach.Destroy() + if err != nil { plog.Errorf("Failed to destroy iso: %v", err) } }() - return awaitCompletion(ctx, mach.QemuInst, outdir, completionChannel, mach.BootStartedErrorChannel, []string{liveOKSignal, signalCompleteString}) + return awaitCompletion(ctx, mach.Instance(), outdir, completionChannel, mach.BootStartedErrorChannel(), []string{liveOKSignal, signalCompleteString}) } // testLiveFIPS verifies that adding fips=1 to the ISO results in a FIPS mode system diff --git a/mantle/platform/machine/qemu/machine.go b/mantle/platform/machine/qemu/machine.go index e567690018..9416bca277 100644 --- a/mantle/platform/machine/qemu/machine.go +++ b/mantle/platform/machine/qemu/machine.go @@ -26,19 +26,35 @@ import ( ) type machine struct { - qc *Cluster - id string - inst *platform.QemuInstance - journal *platform.Journal - consolePath string - console string - ip string + qc *Cluster + id string + inst *platform.QemuInstance + journal *platform.Journal + consolePath string + console string + ip string + tempdir string + bootStartedErrorChannel chan error } func (m *machine) ID() string { return m.id } +// Instance returns the underlying QemuInstance for this machine. +// This is primarily used for ISO installation tests that need direct access +// to the QEMU instance. +func (m *machine) Instance() *platform.QemuInstance { + return m.inst +} + +// BootStartedErrorChannel returns the channel used to signal boot completion +// or errors during the boot process. This is used by ISO installation tests +// to coordinate boot order changes. +func (m *machine) BootStartedErrorChannel() chan error { + return m.bootStartedErrorChannel +} + func (m *machine) IP() string { return m.ip } @@ -91,18 +107,49 @@ func (m *machine) WaitForSoftReboot(timeout time.Duration, oldSoftRebootsCount s return platform.WaitForMachineSoftReboot(m, m.journal, timeout, oldSoftRebootsCount) } +// DeleteTempdir removes the temporary directory associated with this machine. +// It's safe to call multiple times. This is automatically called by Destroy(), +// but can be called earlier if needed to free disk space. +func (m *machine) DeleteTempdir() error { + if m.tempdir == "" { + return nil + } + err := os.RemoveAll(m.tempdir) + m.tempdir = "" + return err +} + func (m *machine) Destroy() { - m.inst.Destroy() + if m == nil { + return + } + + if m.inst != nil { + m.inst.Destroy() + m.inst = nil + } - m.journal.Destroy() + if m.journal != nil { + m.journal.Destroy() + m.journal = nil + } - if buf, err := os.ReadFile(m.consolePath); err == nil { - m.console = string(buf) - } else { - plog.Errorf("Error reading console for instance %v: %v", m.ID(), err) + if m.consolePath != "" { + if buf, err := os.ReadFile(m.consolePath); err == nil { + m.console = string(buf) + } else { + plog.Errorf("Error reading console for instance %v: %v", m.ID(), err) + } } - m.qc.DelMach(m) + if m.qc != nil { + m.qc.DelMach(m) + m.qc = nil + } + + if err := m.DeleteTempdir(); err != nil { + plog.Errorf("Error removing tempdir for instance %v: %v", m.ID(), err) + } } func (m *machine) ConsoleOutput() string { diff --git a/mantle/platform/machine/qemu/metal.go b/mantle/platform/machine/qemu/metal.go index 4c1dc4fec7..d49e489bae 100644 --- a/mantle/platform/machine/qemu/metal.go +++ b/mantle/platform/machine/qemu/metal.go @@ -85,12 +85,6 @@ type Install struct { liveIgnition conf.Conf } -type InstalledMachine struct { - Tempdir string - QemuInst *platform.QemuInstance - BootStartedErrorChannel chan error -} - // Check that artifact has been built and locally exists func (inst *Install) checkArtifactsExist(artifacts []string) error { version := inst.CosaBuild.Meta.OstreeVersion @@ -109,7 +103,7 @@ func (inst *Install) checkArtifactsExist(artifacts []string) error { return nil } -func (inst *Install) PXE(kargs []string, liveIgnition, ignition conf.Conf, offline bool) (*InstalledMachine, error) { +func (inst *Install) PXE(kargs []string, liveIgnition, ignition conf.Conf, offline bool) (*machine, error) { artifacts := []string{"live-kernel", "live-rootfs"} if err := inst.checkArtifactsExist(artifacts); err != nil { return nil, err @@ -146,17 +140,6 @@ func (inst *Install) PXE(kargs []string, liveIgnition, ignition conf.Conf, offli return mach, nil } -func (inst *InstalledMachine) Destroy() error { - if inst.QemuInst != nil { - inst.QemuInst.Destroy() - inst.QemuInst = nil - } - if inst.Tempdir != "" { - return os.RemoveAll(inst.Tempdir) - } - return nil -} - type kernelSetup struct { kernel, initramfs, rootfs string } @@ -544,7 +527,7 @@ func (t *installerRun) run() (*platform.QemuInstance, error) { return inst, nil } -func (inst *Install) runPXE(kern *kernelSetup, offline bool) (*InstalledMachine, error) { +func (inst *Install) runPXE(kern *kernelSetup, offline bool) (*machine, error) { t, err := inst.setup(kern) if err != nil { return nil, errors.Wrapf(err, "setting up install") @@ -572,11 +555,11 @@ func (inst *Install) runPXE(kern *kernelSetup, offline bool) (*InstalledMachine, } tempdir := t.tempdir t.tempdir = "" // Transfer ownership - instmachine := InstalledMachine{ - QemuInst: qinst, - Tempdir: tempdir, + instmachine := machine{ + inst: qinst, + tempdir: tempdir, } - switchBootOrderSignal(qinst, bootStartedChan, &instmachine.BootStartedErrorChannel) + switchBootOrderSignal(qinst, bootStartedChan, &instmachine.bootStartedErrorChannel) return &instmachine, nil } @@ -592,7 +575,7 @@ type installerConfig struct { Console []string `yaml:"console,omitempty"` } -func (inst *Install) InstallViaISOEmbed(kargs []string, liveIgnition, targetIgnition conf.Conf, outdir string, offline, minimal bool) (*InstalledMachine, error) { +func (inst *Install) InstallViaISOEmbed(kargs []string, liveIgnition, targetIgnition conf.Conf, outdir string, offline, minimal bool) (*machine, error) { artifacts := []string{"live-iso"} if !offline { if inst.Native4k { @@ -845,10 +828,10 @@ After=dev-mapper-mpatha.device`) return nil, err } cleanupTempdir = false // Transfer ownership - instmachine := InstalledMachine{ - QemuInst: qinst, - Tempdir: tempdir, + instmachine := machine{ + inst: qinst, + tempdir: tempdir, } - switchBootOrderSignal(qinst, bootStartedChan, &instmachine.BootStartedErrorChannel) + switchBootOrderSignal(qinst, bootStartedChan, &instmachine.bootStartedErrorChannel) return &instmachine, nil } From bfa8b0d53fa90173ed1d551e56c7de55a2bd3949 Mon Sep 17 00:00:00 2001 From: Nikita Dubrovskii Date: Thu, 20 Nov 2025 13:47:40 +0100 Subject: [PATCH 05/31] kola/tests/iso: fold testiso (iso|miniso)-(offline-)?install* tests --- mantle/cmd/kola/testiso.go | 211 ---------- mantle/kola/tests/iso/live-iso.go | 642 ++++++++++++++++++++++++++++++ 2 files changed, 642 insertions(+), 211 deletions(-) create mode 100644 mantle/kola/tests/iso/live-iso.go diff --git a/mantle/cmd/kola/testiso.go b/mantle/cmd/kola/testiso.go index 1ee9f9a5f1..35fc57b9ed 100644 --- a/mantle/cmd/kola/testiso.go +++ b/mantle/cmd/kola/testiso.go @@ -68,7 +68,6 @@ var ( enableUefi bool enableUefiSecure bool isOffline bool - isISOFromRAM bool // These tests only run on RHCOS tests_RHCOS_uefi = []string{ @@ -82,44 +81,23 @@ var ( "iso-as-disk.uefi", "iso-as-disk.uefi-secure", "iso-as-disk.4k.uefi", - "iso-install.bios", - "iso-offline-install.bios", - "iso-offline-install.mpath.bios", - "iso-offline-install-fromram.4k.uefi", "iso-offline-install-iscsi.ibft.uefi", "iso-offline-install-iscsi.ibft-with-mpath.bios", "iso-offline-install-iscsi.manual.bios", - "miniso-install.bios", - "miniso-install.nm.bios", - "miniso-install.4k.uefi", - "miniso-install.4k.nm.uefi", "pxe-offline-install.rootfs-appended.bios", "pxe-offline-install.4k.uefi", "pxe-online-install.bios", "pxe-online-install.4k.uefi", } tests_s390x = []string{ - "iso-offline-install.s390fw", - "iso-offline-install.mpath.s390fw", - "iso-offline-install.4k.s390fw", "pxe-online-install.rootfs-appended.s390fw", "pxe-offline-install.s390fw", - "miniso-install.s390fw", - "miniso-install.nm.s390fw", - "miniso-install.4k.nm.s390fw", // FIXME https://github.com/coreos/fedora-coreos-tracker/issues/1657 //"iso-offline-install-iscsi.ibft.s390fw, //"iso-offline-install-iscsi.ibft-with-mpath.s390fw", //"iso-offline-install-iscsi.manual.s390fw", } tests_ppc64le = []string{ - "iso-offline-install.ppcfw", - "iso-offline-install.mpath.ppcfw", - "iso-offline-install-fromram.4k.ppcfw", - "miniso-install.ppcfw", - "miniso-install.nm.ppcfw", - "miniso-install.4k.ppcfw", - "miniso-install.4k.nm.ppcfw", "pxe-online-install.rootfs-appended.ppcfw", "pxe-offline-install.4k.ppcfw", // FIXME https://github.com/coreos/fedora-coreos-tracker/issues/1657 @@ -128,13 +106,6 @@ var ( //"iso-offline-install-iscsi.manual.ppcfw", } tests_aarch64 = []string{ - "iso-offline-install.uefi", - "iso-offline-install.mpath.uefi", - "iso-offline-install-fromram.4k.uefi", - "miniso-install.uefi", - "miniso-install.nm.uefi", - "miniso-install.4k.uefi", - "miniso-install.4k.nm.uefi", "pxe-offline-install.uefi", "pxe-offline-install.rootfs-appended.4k.uefi", "pxe-online-install.uefi", @@ -148,8 +119,6 @@ var ( const ( installTimeoutMins = 12 - // https://github.com/coreos/fedora-coreos-config/pull/2544 - liveISOFromRAMKarg = "coreos.liveiso.fromram" ) var liveOKSignal = "live-test-OK" @@ -227,18 +196,6 @@ ExecStart=/bin/sh -c '[ ! -e /boot/ignition ]' [Install] RequiredBy=multi-user.target` -var multipathedRoot = `[Unit] -Description=TestISO Verify Multipathed Root -OnFailure=emergency.target -OnFailureJobMode=isolate -Before=coreos-test-installer.service -[Service] -Type=oneshot -RemainAfterExit=yes -ExecStart=/bin/bash -c 'lsblk -pno NAME "/dev/mapper/$(multipath -l -v 1)" | grep -qw "$(findmnt -nvr /sysroot -o SOURCE)"' -[Install] -RequiredBy=multi-user.target` - // This test is broken. Please fix! // https://github.com/coreos/coreos-assembler/issues/3554 var verifyNoEFIBootEntry = `[Unit] @@ -257,92 +214,6 @@ RequiredBy=coreos-installer.target # for iso-as-disk RequiredBy=multi-user.target` -// Verify that the volume ID is the OS name. See also -// https://github.com/openshift/assisted-image-service/pull/477. -// This is the same as the LABEL of the block device for ISO9660. See -// https://github.com/util-linux/util-linux/blob/643bdae8e38055e36acf2963c3416de206081507/libblkid/src/superblocks/iso9660.c#L366-L377 -var verifyIsoVolumeId = `[Unit] -Description=Verify ISO Volume ID -OnFailure=emergency.target -OnFailureJobMode=isolate -# only if we're actually mounting the ISO -ConditionPathIsMountPoint=/run/media/iso -[Service] -Type=oneshot -RemainAfterExit=yes -# the backing device name is arch-dependent, but we know it's mounted on /run/media/iso -ExecStart=bash -c "[[ $(findmnt -no LABEL /run/media/iso) == %s-* ]]" -[Install] -RequiredBy=coreos-installer.target` - -// Unit to check that /run/media/iso is not mounted when -// coreos.liveiso.fromram kernel argument is passed -var isoNotMountedUnit = `[Unit] -Description=Verify ISO is not mounted when coreos.liveiso.fromram -OnFailure=emergency.target -OnFailureJobMode=isolate -ConditionKernelCommandLine=coreos.liveiso.fromram -[Service] -Type=oneshot -StandardOutput=kmsg+console -StandardError=kmsg+console -RemainAfterExit=yes -# Would like to use SuccessExitStatus but it doesn't support what -# we want: https://github.com/systemd/systemd/issues/10297#issuecomment-1672002635 -ExecStart=bash -c "if mountpoint /run/media/iso 2>/dev/null; then exit 1; fi" -[Install] -RequiredBy=coreos-installer.target` - -var nmConnectionId = "CoreOS DHCP" -var nmConnectionFile = "coreos-dhcp.nmconnection" -var nmConnection = fmt.Sprintf(`[connection] -id=%s -type=ethernet -# add wait-device-timeout here so we make sure NetworkManager-wait-online.service will -# wait for a device to be present before exiting. See -# https://github.com/coreos/fedora-coreos-tracker/issues/1275#issuecomment-1231605438 -wait-device-timeout=20000 - -[ipv4] -method=auto -`, nmConnectionId) - -var nmstateConfigFile = "/etc/nmstate/br-ex.yml" -var nmstateConfig = `interfaces: - - name: br-ex - type: linux-bridge - state: up - ipv4: - enabled: false - ipv6: - enabled: false - bridge: - port: [] -` - -// This is used to verify *both* the live and the target system in the `--add-nm-keyfile` path. -var verifyNmKeyfile = fmt.Sprintf(`[Unit] -Description=TestISO Verify NM Keyfile Propagation -OnFailure=emergency.target -OnFailureJobMode=isolate -Wants=network-online.target -After=network-online.target -Before=live-signal-ok.service -Before=coreos-test-installer.service -[Service] -Type=oneshot -RemainAfterExit=yes -ExecStart=/usr/bin/journalctl -u nm-initrd --no-pager --grep "policy: set '%[1]s' (.*) as default .* routing and DNS" -ExecStart=/usr/bin/journalctl -u NetworkManager --no-pager --grep "policy: set '%[1]s' (.*) as default .* routing and DNS" -ExecStart=/usr/bin/grep "%[1]s" /etc/NetworkManager/system-connections/%[2]s -# Also verify nmstate config -ExecStart=/usr/bin/nmcli c show br-ex -[Install] -# for live system -RequiredBy=coreos-installer.target -# for target system -RequiredBy=multi-user.target`, nmConnectionId, nmConnectionFile) - //go:embed resources/iscsi_butane_setup.yaml var iscsi_butane_config string @@ -612,12 +483,6 @@ func runTestIso(cmd *cobra.Command, args []string) (err error) { if kola.HasString("offline", strings.Split(components[0], "-")) { isOffline = true } - // For fromram it is a part of the first component. i.e. for - // iso-offline-install-fromram.uefi we need to search for 'fromram' in - // iso-offline-install-fromram, which is currently in components[0]. - if kola.HasString("fromram", strings.Split(components[0], "-")) { - isISOFromRAM = true - } switch components[0] { case "pxe-offline-install", "pxe-online-install": @@ -626,10 +491,6 @@ func runTestIso(cmd *cobra.Command, args []string) (err error) { duration, err = testAsDisk(ctx, filepath.Join(outputDir, test)) case "iso-fips": duration, err = testLiveFIPS(ctx, filepath.Join(outputDir, test)) - case "iso-install", "iso-offline-install", "iso-offline-install-fromram": - duration, err = testLiveIso(ctx, inst, filepath.Join(outputDir, test), false) - case "miniso-install": - duration, err = testLiveIso(ctx, inst, filepath.Join(outputDir, test), true) case "iso-offline-install-iscsi": var butane_config string switch components[1] { @@ -859,78 +720,6 @@ func testPXE(ctx context.Context, inst qemu.Install, outdir string) (time.Durati return awaitCompletion(ctx, mach.Instance(), outdir, completionChannel, mach.BootStartedErrorChannel(), []string{liveOKSignal, signalCompleteString}) } -func testLiveIso(ctx context.Context, inst qemu.Install, outdir string, minimal bool) (time.Duration, error) { - tmpd, err := os.MkdirTemp("", "kola-testiso") - if err != nil { - return 0, err - } - defer os.RemoveAll(tmpd) - - sshPubKeyBuf, _, err := util.CreateSSHAuthorizedKey(tmpd) - if err != nil { - return 0, err - } - - builder, virtioJournalConfig, err := newQemuBuilderWithDisk(outdir) - if err != nil { - return 0, err - } - inst.Builder = builder - completionChannel, err := inst.Builder.VirtioChannelRead("testisocompletion") - if err != nil { - return 0, err - } - - var isoKernelArgs []string - var keys []string - keys = append(keys, strings.TrimSpace(string(sshPubKeyBuf))) - virtioJournalConfig.AddAuthorizedKeys("core", keys) - - liveConfig := *virtioJournalConfig - liveConfig.AddSystemdUnit("live-signal-ok.service", liveSignalOKUnit, conf.Enable) - liveConfig.AddSystemdUnit("verify-no-efi-boot-entry.service", verifyNoEFIBootEntry, conf.Enable) - liveConfig.AddSystemdUnit("iso-not-mounted-when-fromram.service", isoNotMountedUnit, conf.Enable) - liveConfig.AddSystemdUnit("coreos-test-entered-emergency-target.service", signalFailureUnit, conf.Enable) - volumeIdUnitContents := fmt.Sprintf(verifyIsoVolumeId, kola.CosaBuild.Meta.Name) - liveConfig.AddSystemdUnit("verify-iso-volume-id.service", volumeIdUnitContents, conf.Enable) - - targetConfig := *virtioJournalConfig - targetConfig.AddSystemdUnit("coreos-test-installer.service", signalCompletionUnit, conf.Enable) - targetConfig.AddSystemdUnit("coreos-test-entered-emergency-target.service", signalFailureUnit, conf.Enable) - targetConfig.AddSystemdUnit("coreos-test-installer-no-ignition.service", checkNoIgnition, conf.Enable) - if inst.MultiPathDisk { - targetConfig.AddSystemdUnit("coreos-test-installer-multipathed.service", multipathedRoot, conf.Enable) - } - - if addNmKeyfile { - liveConfig.AddSystemdUnit("coreos-test-nm-keyfile.service", verifyNmKeyfile, conf.Enable) - targetConfig.AddSystemdUnit("coreos-test-nm-keyfile.service", verifyNmKeyfile, conf.Enable) - // NM keyfile via `iso network embed` - inst.NmKeyfiles[nmConnectionFile] = nmConnection - // nmstate config via live Ignition config, propagated via - // --copy-network, which is enabled by inst.NmKeyfiles - liveConfig.AddFile(nmstateConfigFile, nmstateConfig, 0644) - } - - if isISOFromRAM { - isoKernelArgs = append(isoKernelArgs, liveISOFromRAMKarg) - } - - mach, err := inst.InstallViaISOEmbed(isoKernelArgs, liveConfig, targetConfig, outdir, isOffline, minimal) - if err != nil { - return 0, errors.Wrapf(err, "running iso install") - } - defer func() { - err := mach.DeleteTempdir() - mach.Destroy() - if err != nil { - plog.Errorf("Failed to destroy iso: %v", err) - } - }() - - return awaitCompletion(ctx, mach.Instance(), outdir, completionChannel, mach.BootStartedErrorChannel(), []string{liveOKSignal, signalCompleteString}) -} - // testLiveFIPS verifies that adding fips=1 to the ISO results in a FIPS mode system func testLiveFIPS(ctx context.Context, outdir string) (time.Duration, error) { tmpd, err := os.MkdirTemp("", "kola-testiso") diff --git a/mantle/kola/tests/iso/live-iso.go b/mantle/kola/tests/iso/live-iso.go new file mode 100644 index 0000000000..5f7d9bb71a --- /dev/null +++ b/mantle/kola/tests/iso/live-iso.go @@ -0,0 +1,642 @@ +package testiso + +import ( + "bufio" + "fmt" + "io" + "os" + "path/filepath" + "strconv" + "strings" + "time" + + coreosarch "github.com/coreos/stream-metadata-go/arch" + "github.com/pkg/errors" + + "github.com/coreos/coreos-assembler/mantle/kola" + "github.com/coreos/coreos-assembler/mantle/platform" + "github.com/coreos/coreos-assembler/mantle/platform/machine/qemu" + "github.com/coreos/coreos-assembler/mantle/util" + + "github.com/coreos/coreos-assembler/mantle/kola/cluster" + "github.com/coreos/coreos-assembler/mantle/kola/register" + "github.com/coreos/coreos-assembler/mantle/platform/conf" +) + +var ( + tests_live_iso_x86_64 = []string{ + "iso-install.bios", + "iso-offline-install.bios", + "iso-offline-install.mpath.bios", + "iso-offline-install-fromram.bios", + "iso-offline-install-fromram.4k.uefi", + "miniso-install.bios", + "miniso-install.4k.uefi", + "miniso-install.nm.bios", + "miniso-install.4k.nm.uefi", + } + tests_live_iso_aarch64 = []string{ + "iso-offline-install.uefi", + "iso-offline-install.mpath.uefi", + "iso-offline-install-fromram.4k.uefi", + "miniso-install.uefi", + "miniso-install.4k.uefi", + "miniso-install.nm.uefi", + "miniso-install.4k.nm.uefi", + } + tests_live_iso_ppc64le = []string{ + "iso-offline-install.ppcfw", + "iso-offline-install.mpath.ppcfw", + "iso-offline-install-fromram.4k.ppcfw", + "miniso-install.ppcfw", + "miniso-install.4k.ppcfw", + "miniso-install.nm.ppcfw", + "miniso-install.4k.nm.ppcfw", + } + tests_live_iso_s390x = []string{ + "iso-offline-install.s390fw", + "iso-offline-install.4k.s390fw", + "iso-offline-install.mpath.s390fw", + "miniso-install.s390fw", + "miniso-install.nm.s390fw", + "miniso-install.4k.nm.s390fw", + } +) + +func getAllLiveIsoTests() []string { + arch := coreosarch.CurrentRpmArch() + switch arch { + case "x86_64": + return tests_live_iso_x86_64 + case "aarch64": + return tests_live_iso_aarch64 + case "ppc64le": + return tests_live_iso_ppc64le + case "s390x": + return tests_live_iso_s390x + default: + return []string{} + } +} + +func getIsoTestOpts(testName string) IsoTestOpts { + opts := IsoTestOpts{} + + // Parse test name to determine options + if strings.Contains(testName, "4k") { + opts.enable4k = true + } + if strings.Contains(testName, "uefi") { + opts.enableUefi = true + } + if strings.Contains(testName, "mpath") { + opts.enableMultipath = true + } + if strings.Contains(testName, "offline") { + opts.isOffline = true + } + if strings.Contains(testName, "fromram") { + opts.isISOFromRAM = true + } + if strings.Contains(testName, "miniso") { + opts.isMiniso = true + } + if strings.Contains(testName, ".nm") { + opts.addNmKeyfile = true + } + + opts.SetInsecureOnDevBuild() + return opts +} + +func init() { + for _, testName := range getAllLiveIsoTests() { + register.RegisterTest(®ister.Test{ + Run: func(c cluster.TestCluster) { + opts := getIsoTestOpts(testName) + isoLiveIso(c, opts) + }, + ClusterSize: 0, + Name: "iso." + testName, + Description: "Verify ISO live install works.", + Timeout: 12 * time.Minute, + Flags: []register.Flag{}, + Platforms: []string{"qemu"}, + }) + } +} + +var liveOKSignal = "live-test-OK" +var liveSignalOKUnit = fmt.Sprintf(` +[Unit] +Description=TestISO Signal Live ISO Completion +Requires=dev-virtio\\x2dports-testisocompletion.device +OnFailure=emergency.target +OnFailureJobMode=isolate +Before=coreos-installer.service +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=/bin/sh -c '/usr/bin/echo %s >/dev/virtio-ports/testisocompletion' +[Install] +# for install tests +RequiredBy=coreos-installer.target +# for iso-as-disk +RequiredBy=multi-user.target`, liveOKSignal) + +var signalCompleteString = "coreos-installer-test-OK" +var signalCompletionUnit = fmt.Sprintf(` +[Unit] +Description=TestISO Signal Completion +Requires=dev-virtio\\x2dports-testisocompletion.device +OnFailure=emergency.target +OnFailureJobMode=isolate +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=/bin/sh -c '/usr/bin/echo %s >/dev/virtio-ports/testisocompletion && systemctl poweroff' +[Install] +RequiredBy=multi-user.target`, signalCompleteString) + +var signalEmergencyString = "coreos-installer-test-entered-emergency-target" +var signalFailureUnit = fmt.Sprintf(` +[Unit] +Description=TestISO Signal Failure +Requires=dev-virtio\\x2dports-testisocompletion.device +DefaultDependencies=false +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=/bin/sh -c '/usr/bin/echo %s >/dev/virtio-ports/testisocompletion && systemctl poweroff' +[Install] +RequiredBy=emergency.target`, signalEmergencyString) + +var multipathedRoot = `[Unit] +Description=TestISO Verify Multipathed Root +OnFailure=emergency.target +OnFailureJobMode=isolate +Before=coreos-test-installer.service +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=/bin/bash -c 'lsblk -pno NAME "/dev/mapper/$(multipath -l -v 1)" | grep -qw "$(findmnt -nvr /sysroot -o SOURCE)"' +[Install] +RequiredBy=multi-user.target` + +var checkNoIgnition = ` +[Unit] +Description=TestISO Verify No Ignition Config +OnFailure=emergency.target +OnFailureJobMode=isolate +Before=coreos-test-installer.service +After=coreos-ignition-firstboot-complete.service +RequiresMountsFor=/boot +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=/bin/sh -c '[ ! -e /boot/ignition ]' +[Install] +RequiredBy=multi-user.target` + +// This test is broken. Please fix! +// https://github.com/coreos/coreos-assembler/issues/3554 +var verifyNoEFIBootEntry = ` +[Unit] +Description=TestISO Verify No EFI Boot Entry +OnFailure=emergency.target +OnFailureJobMode=isolate +ConditionPathExists=/sys/firmware/efi +Before=live-signal-ok.service +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=/bin/sh -c '! efibootmgr -v | grep -E "(HD|CDROM)\("' +[Install] +# for install tests +RequiredBy=coreos-installer.target +# for iso-as-disk +RequiredBy=multi-user.target` + +// Verify that the volume ID is the OS name. See also +// https://github.com/openshift/assisted-image-service/pull/477. +// This is the same as the LABEL of the block device for ISO9660. See +// https://github.com/util-linux/util-linux/blob/643bdae8e38055e36acf2963c3416de206081507/libblkid/src/superblocks/iso9660.c#L366-L377 +var verifyIsoVolumeId = ` +[Unit] +Description=Verify ISO Volume ID +OnFailure=emergency.target +OnFailureJobMode=isolate +# only if we're actually mounting the ISO +ConditionPathIsMountPoint=/run/media/iso +[Service] +Type=oneshot +RemainAfterExit=yes +# the backing device name is arch-dependent, but we know it's mounted on /run/media/iso +ExecStart=bash -c "[[ $(findmnt -no LABEL /run/media/iso) == %s-* ]]" +[Install] +RequiredBy=coreos-installer.target` + +// Unit to check that /run/media/iso is not mounted when +// coreos.liveiso.fromram kernel argument is passed +var isoNotMountedUnit = ` +[Unit] +Description=Verify ISO is not mounted when coreos.liveiso.fromram +OnFailure=emergency.target +OnFailureJobMode=isolate +ConditionKernelCommandLine=coreos.liveiso.fromram +[Service] +Type=oneshot +StandardOutput=kmsg+console +StandardError=kmsg+console +RemainAfterExit=yes +# Would like to use SuccessExitStatus but it doesn't support what +# we want: https://github.com/systemd/systemd/issues/10297#issuecomment-1672002635 +ExecStart=bash -c "if mountpoint /run/media/iso 2>/dev/null; then exit 1; fi" +[Install] +RequiredBy=coreos-installer.target` + +var nmConnectionId = "CoreOS DHCP" +var nmConnectionFile = "coreos-dhcp.nmconnection" +var nmConnection = fmt.Sprintf(`[connection] +id=%s +type=ethernet +# add wait-device-timeout here so we make sure NetworkManager-wait-online.service will +# wait for a device to be present before exiting. See +# https://github.com/coreos/fedora-coreos-tracker/issues/1275#issuecomment-1231605438 +wait-device-timeout=20000 + +[ipv4] +method=auto +`, nmConnectionId) + +var nmstateConfigFile = "/etc/nmstate/br-ex.yml" +var nmstateConfig = `interfaces: + - name: br-ex + type: linux-bridge + state: up + ipv4: + enabled: false + ipv6: + enabled: false + bridge: + port: [] +` + +// This is used to verify *both* the live and the target system in the `--add-nm-keyfile` path. +var verifyNmKeyfile = fmt.Sprintf(`[Unit] +Description=TestISO Verify NM Keyfile Propagation +OnFailure=emergency.target +OnFailureJobMode=isolate +Wants=network-online.target +After=network-online.target +Before=live-signal-ok.service +Before=coreos-test-installer.service +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=/usr/bin/journalctl -u nm-initrd --no-pager --grep "policy: set '%[1]s' (.*) as default .* routing and DNS" +ExecStart=/usr/bin/journalctl -u NetworkManager --no-pager --grep "policy: set '%[1]s' (.*) as default .* routing and DNS" +ExecStart=/usr/bin/grep "%[1]s" /etc/NetworkManager/system-connections/%[2]s +# Also verify nmstate config +ExecStart=/usr/bin/nmcli c show br-ex +[Install] +# for live system +RequiredBy=coreos-installer.target +# for target system +RequiredBy=multi-user.target`, nmConnectionId, nmConnectionFile) + +type IsoTestOpts struct { + // Flags().BoolVarP(&instInsecure, "inst-insecure", "S", false, "Do not verify signature on metal image") + instInsecure bool + // Flags().BoolVar(&console, "console", false, "Connect qemu console to terminal, turn off automatic initramfs failure checking") + console bool + addNmKeyfile bool + enable4k bool + enableMultipath bool + isOffline bool + isISOFromRAM bool + isMiniso bool + enableUefi bool + enableUefiSecure bool +} + +func (o *IsoTestOpts) SetInsecureOnDevBuild() { + // Ignore signing verification by default when running with development build + // https://github.com/coreos/fedora-coreos-tracker/issues/908 + if strings.Contains(kola.CosaBuild.Meta.BuildID, ".dev.") { + o.instInsecure = true + fmt.Printf("Detected development build; disabling signature verification\n") + } +} + +const ( + installTimeoutMins = 12 + // https://github.com/coreos/fedora-coreos-config/pull/2544 + liveISOFromRAMKarg = "coreos.liveiso.fromram" +) + +func newBaseQemuBuilder(opts IsoTestOpts, outdir string) (*platform.QemuBuilder, error) { + builder := qemu.NewMetalQemuBuilderDefault() + if opts.enableUefiSecure { + builder.Firmware = "uefi-secure" + } else if opts.enableUefi { + builder.Firmware = "uefi" + } + + if err := os.MkdirAll(outdir, 0755); err != nil { + return nil, err + } + + builder.InheritConsole = opts.console + if !opts.console { + builder.ConsoleFile = filepath.Join(outdir, "console.txt") + } + + if kola.QEMUOptions.Memory != "" { + parsedMem, err := strconv.ParseInt(kola.QEMUOptions.Memory, 10, 32) + if err != nil { + return nil, err + } + builder.MemoryMiB = int(parsedMem) + } + + return builder, nil +} + +func newQemuBuilder(opts IsoTestOpts, outdir string) (*platform.QemuBuilder, *conf.Conf, error) { + builder, err := newBaseQemuBuilder(opts, outdir) + if err != nil { + return nil, nil, err + } + + config, err := conf.EmptyIgnition().Render(conf.FailWarnings) + if err != nil { + return nil, nil, err + } + + err = forwardJournal(outdir, builder, config) + if err != nil { + return nil, nil, err + } + + return builder, config, nil +} + +func forwardJournal(outdir string, builder *platform.QemuBuilder, config *conf.Conf) error { + journalPipe, err := builder.VirtioJournal(config, "") + if err != nil { + return err + } + journalOut, err := os.OpenFile(filepath.Join(outdir, "journal.txt"), os.O_WRONLY|os.O_CREATE, 0644) + if err != nil { + return err + } + + go func() { + _, err := io.Copy(journalOut, journalPipe) + if err != nil && err != io.EOF { + panic(err) + } + }() + + return nil +} + +func newQemuBuilderWithDisk(opts IsoTestOpts, outdir string) (*platform.QemuBuilder, *conf.Conf, error) { + builder, config, err := newQemuBuilder(opts, outdir) + + if err != nil { + return nil, nil, err + } + + sectorSize := 0 + if opts.enable4k { + sectorSize = 4096 + } + + disk := platform.Disk{ + Size: "12G", // Arbitrary + SectorSize: sectorSize, + MultiPathDisk: opts.enableMultipath, + } + + //TBD: see if we can remove this and just use AddDisk and inject bootindex during startup + if coreosarch.CurrentRpmArch() == "s390x" || coreosarch.CurrentRpmArch() == "aarch64" { + // s390x and aarch64 need to use bootindex as they don't support boot once + if err := builder.AddDisk(&disk); err != nil { + return nil, nil, err + } + } else { + if err := builder.AddPrimaryDisk(&disk); err != nil { + return nil, nil, err + } + } + + return builder, config, nil +} + +func isoLiveIso(c cluster.TestCluster, opts IsoTestOpts) { + var outdir string + var qc *qemu.Cluster + switch pc := c.Cluster.(type) { + case *qemu.Cluster: + outdir = pc.RuntimeConf().OutputDir + qc = pc + default: + c.Fatalf("Unsupported cluster type") + } + + if kola.CosaBuild.Meta.BuildArtifacts.LiveIso == nil || kola.CosaBuild.Meta.BuildArtifacts.LiveKernel == nil { + c.Fatalf("build %s is missing live artifacts", kola.CosaBuild.Meta.Name) + } + + inst := qemu.Install{ + CosaBuild: kola.CosaBuild, + NmKeyfiles: make(map[string]string), + Insecure: opts.instInsecure, + Native4k: opts.enable4k, + MultiPathDisk: opts.enableMultipath, + } + + tmpd, err := os.MkdirTemp("", "kola-iso.live") + if err != nil { + c.Fatal(err) + } + defer os.RemoveAll(tmpd) + + sshPubKeyBuf, _, err := util.CreateSSHAuthorizedKey(tmpd) + if err != nil { + c.Fatal(err) + } + + builder, virtioJournalConfig, err := newQemuBuilderWithDisk(opts, outdir) + if err != nil { + c.Fatal(err) + } + inst.Builder = builder + completionChannel, err := inst.Builder.VirtioChannelRead("testisocompletion") + if err != nil { + c.Fatal(err) + } + + var isoKernelArgs []string + var keys []string + keys = append(keys, strings.TrimSpace(string(sshPubKeyBuf))) + virtioJournalConfig.AddAuthorizedKeys("core", keys) + + liveConfig := *virtioJournalConfig + liveConfig.AddSystemdUnit("live-signal-ok.service", liveSignalOKUnit, conf.Enable) + liveConfig.AddSystemdUnit("verify-no-efi-boot-entry.service", verifyNoEFIBootEntry, conf.Enable) + liveConfig.AddSystemdUnit("iso-not-mounted-when-fromram.service", isoNotMountedUnit, conf.Enable) + liveConfig.AddSystemdUnit("coreos-test-entered-emergency-target.service", signalFailureUnit, conf.Enable) + volumeIdUnitContents := fmt.Sprintf(verifyIsoVolumeId, kola.CosaBuild.Meta.Name) + liveConfig.AddSystemdUnit("verify-iso-volume-id.service", volumeIdUnitContents, conf.Enable) + + targetConfig := *virtioJournalConfig + targetConfig.AddSystemdUnit("coreos-test-installer.service", signalCompletionUnit, conf.Enable) + targetConfig.AddSystemdUnit("coreos-test-entered-emergency-target.service", signalFailureUnit, conf.Enable) + targetConfig.AddSystemdUnit("coreos-test-installer-no-ignition.service", checkNoIgnition, conf.Enable) + if inst.MultiPathDisk { + targetConfig.AddSystemdUnit("coreos-test-installer-multipathed.service", multipathedRoot, conf.Enable) + } + + if opts.addNmKeyfile { + liveConfig.AddSystemdUnit("coreos-test-nm-keyfile.service", verifyNmKeyfile, conf.Enable) + targetConfig.AddSystemdUnit("coreos-test-nm-keyfile.service", verifyNmKeyfile, conf.Enable) + // NM keyfile via `iso network embed` + inst.NmKeyfiles[nmConnectionFile] = nmConnection + // nmstate config via live Ignition config, propagated via + // --copy-network, which is enabled by inst.NmKeyfiles + liveConfig.AddFile(nmstateConfigFile, nmstateConfig, 0644) + } + + if opts.isISOFromRAM { + isoKernelArgs = append(isoKernelArgs, liveISOFromRAMKarg) + } + + mach, err := inst.InstallViaISOEmbed(isoKernelArgs, liveConfig, targetConfig, outdir, opts.isOffline, opts.isMiniso) + if err != nil { + c.Fatal(err) + } + qc.AddMach(mach) + err = awaitCompletion(c, mach.Instance(), opts.console, outdir, completionChannel, mach.BootStartedErrorChannel(), []string{liveOKSignal, signalCompleteString}) + if err != nil { + c.Fatal(err) + } +} + +func awaitCompletion(c cluster.TestCluster, inst *platform.QemuInstance, console bool, outdir string, qchan *os.File, booterrchan chan error, expected []string) error { + ctx := c.Context() + + errchan := make(chan error) + go func() { + timeout := (time.Duration(installTimeoutMins*(100+kola.Options.ExtendTimeoutPercent)) * time.Minute) / 100 + time.Sleep(timeout) + errchan <- fmt.Errorf("timed out after %v", timeout) + }() + if !console { + go func() { + errBuf, err := inst.WaitIgnitionError(ctx) + if err == nil { + if errBuf != "" { + c.Logf("entered emergency.target in initramfs") + path := filepath.Join(outdir, "ignition-virtio-dump.txt") + if err := os.WriteFile(path, []byte(errBuf), 0644); err != nil { + c.Errorf("Failed to write journal: %v", err) + } + err = platform.ErrInitramfsEmergency + } + } + if err != nil { + errchan <- err + } + }() + } + go func() { + err := inst.Wait() + // only one Wait() gets process data, so also manually check for signal + //plog.Debugf("qemu exited err=%v", err) + if err == nil && inst.Signaled() { + err = errors.New("process killed") + } + if err != nil { + errchan <- errors.Wrapf(err, "QEMU unexpectedly exited while awaiting completion") + } + time.Sleep(1 * time.Minute) + errchan <- fmt.Errorf("QEMU exited; timed out waiting for completion") + }() + go func() { + r := bufio.NewReader(qchan) + for _, exp := range expected { + l, err := r.ReadString('\n') + if err != nil { + if err == io.EOF { + // this may be from QEMU getting killed or exiting; wait a bit + // to give a chance for .Wait() above to feed the channel with a + // better error + time.Sleep(1 * time.Second) + errchan <- fmt.Errorf("Got EOF from completion channel, %s expected", exp) + } else { + errchan <- errors.Wrapf(err, "reading from completion channel") + } + return + } + line := strings.TrimSpace(l) + if line != exp { + errchan <- fmt.Errorf("Unexpected string from completion channel: %s expected: %s", line, exp) + return + } + } + // OK! + errchan <- nil + }() + go func() { + //check for error when switching boot order + if booterrchan != nil { + if err := <-booterrchan; err != nil { + errchan <- err + } + } + }() + err := <-errchan + if err == nil { + // No error so far, check the console and journal files + consoleFile := filepath.Join(outdir, "console.txt") + journalFile := filepath.Join(outdir, "journal.txt") + files := []string{consoleFile, journalFile} + for _, file := range files { + fileName := filepath.Base(file) + // Check if the file exists + _, err := os.Stat(file) + if os.IsNotExist(err) { + fmt.Printf("The file: %v does not exist\n", fileName) + continue + } else if err != nil { + fmt.Println(err) + return err + } + // Read the contents of the file + fileContent, err := os.ReadFile(file) + if err != nil { + fmt.Println(err) + return err + } + // Check for badness with CheckConsole + warnOnly, badlines := kola.CheckConsole([]byte(fileContent), nil) + if len(badlines) > 0 { + for _, badline := range badlines { + if warnOnly { + c.Errorf("bad log line detected: %v", badline) + } else { + c.Logf("bad log line detected: %v", badline) + } + } + if !warnOnly { + err = fmt.Errorf("errors found in log files") + return err + } + } + } + } + return err +} From 1f5c97c77bd345bebec46a61c76ba1444ec9e39f Mon Sep 17 00:00:00 2001 From: Nikita Dubrovskii Date: Thu, 20 Nov 2025 16:32:09 +0100 Subject: [PATCH 06/31] kola/tests/iso: fold testiso *pxe* tests --- mantle/cmd/kola/testiso.go | 156 ---------------------------- mantle/kola/tests/iso/live-iso.go | 167 +++++++++++++++++++++++++++++- 2 files changed, 166 insertions(+), 157 deletions(-) diff --git a/mantle/cmd/kola/testiso.go b/mantle/cmd/kola/testiso.go index 35fc57b9ed..ee8219341f 100644 --- a/mantle/cmd/kola/testiso.go +++ b/mantle/cmd/kola/testiso.go @@ -62,7 +62,6 @@ var ( console bool - addNmKeyfile bool enable4k bool enableMultipath bool enableUefi bool @@ -84,32 +83,20 @@ var ( "iso-offline-install-iscsi.ibft.uefi", "iso-offline-install-iscsi.ibft-with-mpath.bios", "iso-offline-install-iscsi.manual.bios", - "pxe-offline-install.rootfs-appended.bios", - "pxe-offline-install.4k.uefi", - "pxe-online-install.bios", - "pxe-online-install.4k.uefi", } tests_s390x = []string{ - "pxe-online-install.rootfs-appended.s390fw", - "pxe-offline-install.s390fw", // FIXME https://github.com/coreos/fedora-coreos-tracker/issues/1657 //"iso-offline-install-iscsi.ibft.s390fw, //"iso-offline-install-iscsi.ibft-with-mpath.s390fw", //"iso-offline-install-iscsi.manual.s390fw", } tests_ppc64le = []string{ - "pxe-online-install.rootfs-appended.ppcfw", - "pxe-offline-install.4k.ppcfw", // FIXME https://github.com/coreos/fedora-coreos-tracker/issues/1657 //"iso-offline-install-iscsi.ibft.ppcfw", //"iso-offline-install-iscsi.ibft-with-mpath.ppcfw", //"iso-offline-install-iscsi.manual.ppcfw", } tests_aarch64 = []string{ - "pxe-offline-install.uefi", - "pxe-offline-install.rootfs-appended.4k.uefi", - "pxe-online-install.uefi", - "pxe-online-install.4k.uefi", // FIXME https://github.com/coreos/fedora-coreos-tracker/issues/1657 //"iso-offline-install-iscsi.ibft.uefi", //"iso-offline-install-iscsi.ibft-with-mpath.uefi", @@ -139,36 +126,6 @@ RequiredBy=coreos-installer.target RequiredBy=multi-user.target `, liveOKSignal) -var downloadCheck = `[Unit] -Description=TestISO Verify CoreOS Installer Download -After=coreos-installer.service -Before=coreos-installer.target -[Service] -Type=oneshot -StandardOutput=kmsg+console -StandardError=kmsg+console -ExecStart=/bin/sh -c "journalctl -t coreos-installer-service | /usr/bin/awk '/[Dd]ownload/ {exit 1}'" -ExecStart=/bin/sh -c "/usr/bin/udevadm settle" -ExecStart=/bin/sh -c "/usr/bin/mount /dev/disk/by-label/root /mnt" -ExecStart=/bin/sh -c "/usr/bin/jq -er '.[\"build\"]? + .[\"version\"]? == \"%s\"' /mnt/.coreos-aleph-version.json" -[Install] -RequiredBy=coreos-installer.target -` - -var signalCompleteString = "coreos-installer-test-OK" -var signalCompletionUnit = fmt.Sprintf(`[Unit] -Description=TestISO Signal Completion -Requires=dev-virtio\\x2dports-testisocompletion.device -OnFailure=emergency.target -OnFailureJobMode=isolate -[Service] -Type=oneshot -RemainAfterExit=yes -ExecStart=/bin/sh -c '/usr/bin/echo %s >/dev/virtio-ports/testisocompletion && systemctl poweroff' -[Install] -RequiredBy=multi-user.target -`, signalCompleteString) - var signalEmergencyString = "coreos-installer-test-entered-emergency-target" var signalFailureUnit = fmt.Sprintf(`[Unit] Description=TestISO Signal Failure @@ -182,20 +139,6 @@ ExecStart=/bin/sh -c '/usr/bin/echo %s >/dev/virtio-ports/testisocompletion && s RequiredBy=emergency.target `, signalEmergencyString) -var checkNoIgnition = `[Unit] -Description=TestISO Verify No Ignition Config -OnFailure=emergency.target -OnFailureJobMode=isolate -Before=coreos-test-installer.service -After=coreos-ignition-firstboot-complete.service -RequiresMountsFor=/boot -[Service] -Type=oneshot -RemainAfterExit=yes -ExecStart=/bin/sh -c '[ ! -e /boot/ignition ]' -[Install] -RequiredBy=multi-user.target` - // This test is broken. Please fix! // https://github.com/coreos/coreos-assembler/issues/3554 var verifyNoEFIBootEntry = `[Unit] @@ -319,39 +262,6 @@ func forwardJournal(outdir string, builder *platform.QemuBuilder, config *conf.C return nil } -func newQemuBuilderWithDisk(outdir string) (*platform.QemuBuilder, *conf.Conf, error) { - builder, config, err := newQemuBuilder(outdir) - - if err != nil { - return nil, nil, err - } - - sectorSize := 0 - if enable4k { - sectorSize = 4096 - } - - disk := platform.Disk{ - Size: "12G", // Arbitrary - SectorSize: sectorSize, - MultiPathDisk: enableMultipath, - } - - //TBD: see if we can remove this and just use AddDisk and inject bootindex during startup - if coreosarch.CurrentRpmArch() == "s390x" || coreosarch.CurrentRpmArch() == "aarch64" { - // s390x and aarch64 need to use bootindex as they don't support boot once - if err := builder.AddDisk(&disk); err != nil { - return nil, nil, err - } - } else { - if err := builder.AddPrimaryDisk(&disk); err != nil { - return nil, nil, err - } - } - - return builder, config, nil -} - // See similar semantics in the `filterTests` of `kola.go`. func filterTests(tests []string, patterns []string) ([]string, error) { r := []string{} @@ -448,7 +358,6 @@ func runTestIso(cmd *cobra.Command, args []string) (err error) { return err } - addNmKeyfile = false enable4k = false enableMultipath = false enableUefi = false @@ -465,9 +374,6 @@ func runTestIso(cmd *cobra.Command, args []string) (err error) { enable4k = true inst.Native4k = true } - if kola.HasString("nm", components) { - addNmKeyfile = true - } if kola.HasString("mpath", components) { enableMultipath = true inst.MultiPathDisk = true @@ -485,8 +391,6 @@ func runTestIso(cmd *cobra.Command, args []string) (err error) { } switch components[0] { - case "pxe-offline-install", "pxe-online-install": - duration, err = testPXE(ctx, inst, filepath.Join(outputDir, test)) case "iso-as-disk": duration, err = testAsDisk(ctx, filepath.Join(outputDir, test)) case "iso-fips": @@ -660,66 +564,6 @@ func printResult(test string, duration time.Duration, err error) bool { return false } -func testPXE(ctx context.Context, inst qemu.Install, outdir string) (time.Duration, error) { - if addNmKeyfile { - return 0, errors.New("--add-nm-keyfile not yet supported for PXE") - } - tmpd, err := os.MkdirTemp("", "kola-testiso") - if err != nil { - return 0, errors.Wrapf(err, "creating tempdir") - } - defer os.RemoveAll(tmpd) - - sshPubKeyBuf, _, err := util.CreateSSHAuthorizedKey(tmpd) - if err != nil { - return 0, errors.Wrapf(err, "creating SSH AuthorizedKey") - } - - builder, virtioJournalConfig, err := newQemuBuilderWithDisk(outdir) - if err != nil { - return 0, errors.Wrapf(err, "creating QemuBuilder") - } - - // increase the memory for pxe tests with appended rootfs in the initrd - // we were bumping up into the 4GiB limit in RHCOS/c9s - // pxe-offline-install.rootfs-appended.bios tests - if inst.PxeAppendRootfs && builder.MemoryMiB < 5120 { - builder.MemoryMiB = 5120 - } - - inst.Builder = builder - completionChannel, err := inst.Builder.VirtioChannelRead("testisocompletion") - if err != nil { - return 0, errors.Wrapf(err, "setting up virtio-serial channel") - } - - var keys []string - keys = append(keys, strings.TrimSpace(string(sshPubKeyBuf))) - virtioJournalConfig.AddAuthorizedKeys("core", keys) - - liveConfig := *virtioJournalConfig - liveConfig.AddSystemdUnit("live-signal-ok.service", liveSignalOKUnit, conf.Enable) - liveConfig.AddSystemdUnit("coreos-test-entered-emergency-target.service", signalFailureUnit, conf.Enable) - - if isOffline { - contents := fmt.Sprintf(downloadCheck, kola.CosaBuild.Meta.OstreeVersion) - liveConfig.AddSystemdUnit("coreos-installer-offline-check.service", contents, conf.Enable) - } - - targetConfig := *virtioJournalConfig - targetConfig.AddSystemdUnit("coreos-test-installer.service", signalCompletionUnit, conf.Enable) - targetConfig.AddSystemdUnit("coreos-test-entered-emergency-target.service", signalFailureUnit, conf.Enable) - targetConfig.AddSystemdUnit("coreos-test-installer-no-ignition.service", checkNoIgnition, conf.Enable) - - mach, err := inst.PXE(pxeKernelArgs, liveConfig, targetConfig, isOffline) - if err != nil { - return 0, errors.Wrapf(err, "running PXE") - } - defer mach.Destroy() - - return awaitCompletion(ctx, mach.Instance(), outdir, completionChannel, mach.BootStartedErrorChannel(), []string{liveOKSignal, signalCompleteString}) -} - // testLiveFIPS verifies that adding fips=1 to the ISO results in a FIPS mode system func testLiveFIPS(ctx context.Context, outdir string) (time.Duration, error) { tmpd, err := os.MkdirTemp("", "kola-testiso") diff --git a/mantle/kola/tests/iso/live-iso.go b/mantle/kola/tests/iso/live-iso.go index 5f7d9bb71a..c23b81240e 100644 --- a/mantle/kola/tests/iso/live-iso.go +++ b/mantle/kola/tests/iso/live-iso.go @@ -61,6 +61,26 @@ var ( "miniso-install.nm.s390fw", "miniso-install.4k.nm.s390fw", } + tests_pxe_x86_64 = []string{ + "pxe-offline-install.rootfs-appended.bios", + "pxe-offline-install.4k.uefi", + "pxe-online-install.bios", + "pxe-online-install.4k.uefi", + } + tests_pxe_aarch64 = []string{ + "pxe-offline-install.uefi", + "pxe-offline-install.rootfs-appended.4k.uefi", + "pxe-online-install.uefi", + "pxe-online-install.4k.uefi", + } + tests_pxe_ppc64le = []string{ + "pxe-online-install.rootfs-appended.ppcfw", + "pxe-offline-install.4k.ppcfw", + } + tests_pxe_s390x = []string{ + "pxe-online-install.rootfs-appended.s390fw", + "pxe-offline-install.s390fw", + } ) func getAllLiveIsoTests() []string { @@ -79,6 +99,22 @@ func getAllLiveIsoTests() []string { } } +func getAllPxeTests() []string { + arch := coreosarch.CurrentRpmArch() + switch arch { + case "x86_64": + return tests_pxe_x86_64 + case "aarch64": + return tests_pxe_aarch64 + case "ppc64le": + return tests_pxe_ppc64le + case "s390x": + return tests_pxe_s390x + default: + return []string{} + } +} + func getIsoTestOpts(testName string) IsoTestOpts { opts := IsoTestOpts{} @@ -104,6 +140,9 @@ func getIsoTestOpts(testName string) IsoTestOpts { if strings.Contains(testName, ".nm") { opts.addNmKeyfile = true } + if strings.Contains(testName, "rootfs-appended") { + opts.pxeAppendRootfs = true + } opts.SetInsecureOnDevBuild() return opts @@ -124,6 +163,22 @@ func init() { Platforms: []string{"qemu"}, }) } + + // PXE tests + for _, testName := range getAllPxeTests() { + register.RegisterTest(®ister.Test{ + Run: func(c cluster.TestCluster) { + opts := getIsoTestOpts(testName) + testPXE(c, opts) + }, + ClusterSize: 0, + Name: "iso." + testName, + Description: "Verify PXE install works.", + Timeout: 12 * time.Minute, + Flags: []register.Flag{}, + Platforms: []string{"qemu"}, + }) + } } var liveOKSignal = "live-test-OK" @@ -308,6 +363,8 @@ RequiredBy=multi-user.target`, nmConnectionId, nmConnectionFile) type IsoTestOpts struct { // Flags().BoolVarP(&instInsecure, "inst-insecure", "S", false, "Do not verify signature on metal image") instInsecure bool + // Flags().StringSliceVar(&pxeKernelArgs, "pxe-kargs", nil, "Additional kernel arguments for PXE") + pxeKernelArgs []string // Flags().BoolVar(&console, "console", false, "Connect qemu console to terminal, turn off automatic initramfs failure checking") console bool addNmKeyfile bool @@ -318,6 +375,7 @@ type IsoTestOpts struct { isMiniso bool enableUefi bool enableUefiSecure bool + pxeAppendRootfs bool } func (o *IsoTestOpts) SetInsecureOnDevBuild() { @@ -325,7 +383,7 @@ func (o *IsoTestOpts) SetInsecureOnDevBuild() { // https://github.com/coreos/fedora-coreos-tracker/issues/908 if strings.Contains(kola.CosaBuild.Meta.BuildID, ".dev.") { o.instInsecure = true - fmt.Printf("Detected development build; disabling signature verification\n") + //fmt.Printf("Detected development build; disabling signature verification\n") } } @@ -360,6 +418,13 @@ func newBaseQemuBuilder(opts IsoTestOpts, outdir string) (*platform.QemuBuilder, builder.MemoryMiB = int(parsedMem) } + // increase the memory for pxe tests with appended rootfs in the initrd + // we were bumping up into the 4GiB limit in RHCOS/c9s + // pxe-offline-install.rootfs-appended.bios tests + if opts.pxeAppendRootfs && builder.MemoryMiB < 5120 { + builder.MemoryMiB = 5120 + } + return builder, nil } @@ -525,6 +590,106 @@ func isoLiveIso(c cluster.TestCluster, opts IsoTestOpts) { } } +var downloadCheck = `[Unit] +Description=TestISO Verify CoreOS Installer Download +After=coreos-installer.service +Before=coreos-installer.target +[Service] +Type=oneshot +StandardOutput=kmsg+console +StandardError=kmsg+console +ExecStart=/bin/sh -c "journalctl -t coreos-installer-service | /usr/bin/awk '/[Dd]ownload/ {exit 1}'" +ExecStart=/bin/sh -c "/usr/bin/udevadm settle" +ExecStart=/bin/sh -c "/usr/bin/mount /dev/disk/by-label/root /mnt" +ExecStart=/bin/sh -c "/usr/bin/jq -er '.[\"build\"]? + .[\"version\"]? == \"%s\"' /mnt/.coreos-aleph-version.json" +[Install] +RequiredBy=coreos-installer.target +` + +func testPXE(c cluster.TestCluster, opts IsoTestOpts) { + var outdir string + var qc *qemu.Cluster + + switch pc := c.Cluster.(type) { + case *qemu.Cluster: + outdir = pc.RuntimeConf().OutputDir + qc = pc + default: + c.Fatalf("Unsupported cluster type") + } + + if opts.addNmKeyfile { + c.Fatal("--add-nm-keyfile not yet supported for PXE") + } + + inst := qemu.Install{ + CosaBuild: kola.CosaBuild, + NmKeyfiles: make(map[string]string), + Insecure: opts.instInsecure, + Native4k: opts.enable4k, + MultiPathDisk: opts.enableMultipath, + PxeAppendRootfs: opts.pxeAppendRootfs, + } + + tmpd, err := os.MkdirTemp("", "kola-iso.pxe") + if err != nil { + c.Fatal(err) + } + defer os.RemoveAll(tmpd) + + sshPubKeyBuf, _, err := util.CreateSSHAuthorizedKey(tmpd) + if err != nil { + c.Fatal(err) + } + + builder, virtioJournalConfig, err := newQemuBuilderWithDisk(opts, outdir) + if err != nil { + c.Fatal(err) + } + + // increase the memory for pxe tests with appended rootfs in the initrd + // we were bumping up into the 4GiB limit in RHCOS/c9s + // pxe-offline-install.rootfs-appended.bios tests + if inst.PxeAppendRootfs && builder.MemoryMiB < 5120 { + builder.MemoryMiB = 5120 + } + + inst.Builder = builder + completionChannel, err := inst.Builder.VirtioChannelRead("testisocompletion") + if err != nil { + c.Fatal(err) // , "setting up virtio-serial channel") + } + + var keys []string + keys = append(keys, strings.TrimSpace(string(sshPubKeyBuf))) + virtioJournalConfig.AddAuthorizedKeys("core", keys) + + liveConfig := *virtioJournalConfig + liveConfig.AddSystemdUnit("live-signal-ok.service", liveSignalOKUnit, conf.Enable) + liveConfig.AddSystemdUnit("coreos-test-entered-emergency-target.service", signalFailureUnit, conf.Enable) + + if opts.isOffline { + contents := fmt.Sprintf(downloadCheck, kola.CosaBuild.Meta.OstreeVersion) + liveConfig.AddSystemdUnit("coreos-installer-offline-check.service", contents, conf.Enable) + } + + targetConfig := *virtioJournalConfig + targetConfig.AddSystemdUnit("coreos-test-installer.service", signalCompletionUnit, conf.Enable) + targetConfig.AddSystemdUnit("coreos-test-entered-emergency-target.service", signalFailureUnit, conf.Enable) + targetConfig.AddSystemdUnit("coreos-test-installer-no-ignition.service", checkNoIgnition, conf.Enable) + + mach, err := inst.PXE(opts.pxeKernelArgs, liveConfig, targetConfig, opts.isOffline) + if err != nil { + c.Fatal(err) + } + qc.AddMach(mach) + + err = awaitCompletion(c, mach.Instance(), opts.console, outdir, completionChannel, mach.BootStartedErrorChannel(), []string{liveOKSignal, signalCompleteString}) + if err != nil { + c.Fatal(err) + } +} + func awaitCompletion(c cluster.TestCluster, inst *platform.QemuInstance, console bool, outdir string, qchan *os.File, booterrchan chan error, expected []string) error { ctx := c.Context() From 5ebdd5ea77c3a36f3c0cddec193e1c58b42b1585 Mon Sep 17 00:00:00 2001 From: Nikita Dubrovskii Date: Fri, 21 Nov 2025 10:48:48 +0100 Subject: [PATCH 07/31] kola/tests/iso: fold testiso *iscsi* tests --- mantle/cmd/kola/testiso.go | 167 +-------------- .../tests/iso}/iscsi_butane_setup.yaml | 0 mantle/kola/tests/iso/live-iso.go | 199 ++++++++++++++++++ 3 files changed, 201 insertions(+), 165 deletions(-) rename mantle/{cmd/kola/resources => kola/tests/iso}/iscsi_butane_setup.yaml (100%) diff --git a/mantle/cmd/kola/testiso.go b/mantle/cmd/kola/testiso.go index ee8219341f..abd0452c85 100644 --- a/mantle/cmd/kola/testiso.go +++ b/mantle/cmd/kola/testiso.go @@ -63,10 +63,8 @@ var ( console bool enable4k bool - enableMultipath bool enableUefi bool enableUefiSecure bool - isOffline bool // These tests only run on RHCOS tests_RHCOS_uefi = []string{ @@ -80,27 +78,6 @@ var ( "iso-as-disk.uefi", "iso-as-disk.uefi-secure", "iso-as-disk.4k.uefi", - "iso-offline-install-iscsi.ibft.uefi", - "iso-offline-install-iscsi.ibft-with-mpath.bios", - "iso-offline-install-iscsi.manual.bios", - } - tests_s390x = []string{ - // FIXME https://github.com/coreos/fedora-coreos-tracker/issues/1657 - //"iso-offline-install-iscsi.ibft.s390fw, - //"iso-offline-install-iscsi.ibft-with-mpath.s390fw", - //"iso-offline-install-iscsi.manual.s390fw", - } - tests_ppc64le = []string{ - // FIXME https://github.com/coreos/fedora-coreos-tracker/issues/1657 - //"iso-offline-install-iscsi.ibft.ppcfw", - //"iso-offline-install-iscsi.ibft-with-mpath.ppcfw", - //"iso-offline-install-iscsi.manual.ppcfw", - } - tests_aarch64 = []string{ - // FIXME https://github.com/coreos/fedora-coreos-tracker/issues/1657 - //"iso-offline-install-iscsi.ibft.uefi", - //"iso-offline-install-iscsi.ibft-with-mpath.uefi", - //"iso-offline-install-iscsi.manual.uefi", } ) @@ -157,9 +134,6 @@ RequiredBy=coreos-installer.target # for iso-as-disk RequiredBy=multi-user.target` -//go:embed resources/iscsi_butane_setup.yaml -var iscsi_butane_config string - func init() { cmdTestIso.Flags().BoolVarP(&instInsecure, "inst-insecure", "S", false, "Do not verify signature on metal image") cmdTestIso.Flags().BoolVar(&console, "console", false, "Connect qemu console to terminal, turn off automatic initramfs failure checking") @@ -182,12 +156,8 @@ func getAllTests(build *util.LocalBuild) []string { switch arch { case "x86_64": tests = tests_x86_64 - case "ppc64le": - tests = tests_ppc64le - case "s390x": - tests = tests_s390x - case "aarch64": - tests = tests_aarch64 + default: + return []string{} } if kola.CosaBuild.Meta.Name == "rhcos" && arch != "s390x" && arch != "ppc64le" { tests = append(tests, tests_RHCOS_uefi...) @@ -359,10 +329,8 @@ func runTestIso(cmd *cobra.Command, args []string) (err error) { } enable4k = false - enableMultipath = false enableUefi = false enableUefiSecure = false - isOffline = false inst := baseInst // Pretend this is Rust and I wrote .copy() fmt.Printf("Running test: %s\n", test) @@ -374,40 +342,17 @@ func runTestIso(cmd *cobra.Command, args []string) (err error) { enable4k = true inst.Native4k = true } - if kola.HasString("mpath", components) { - enableMultipath = true - inst.MultiPathDisk = true - } if kola.HasString("uefi-secure", components) { enableUefiSecure = true } else if kola.HasString("uefi", components) { enableUefi = true } - // For offline it is a part of the first component. i.e. for - // iso-offline-install.bios we need to search for 'offline' in - // iso-offline-install, which is currently in components[0]. - if kola.HasString("offline", strings.Split(components[0], "-")) { - isOffline = true - } switch components[0] { case "iso-as-disk": duration, err = testAsDisk(ctx, filepath.Join(outputDir, test)) case "iso-fips": duration, err = testLiveFIPS(ctx, filepath.Join(outputDir, test)) - case "iso-offline-install-iscsi": - var butane_config string - switch components[1] { - case "ibft": - butane_config = strings.ReplaceAll(iscsi_butane_config, "COREOS_INSTALLER_KARGS", "--append-karg rd.iscsi.firmware=1") - case "manual": - butane_config = strings.ReplaceAll(iscsi_butane_config, "COREOS_INSTALLER_KARGS", "--append-karg netroot=iscsi:10.0.2.15::::iqn.2024-05.com.coreos:0") - case "ibft-with-mpath": - butane_config = strings.ReplaceAll(iscsi_butane_config, "COREOS_INSTALLER_KARGS", "--append-karg rd.iscsi.firmware=1 --append-karg rd.multipath=default --append-karg root=/dev/disk/by-label/dm-mpath-root --append-karg rw") - default: - plog.Fatalf("Unknown test name:%s", test) - } - duration, err = testLiveInstalliscsi(ctx, inst, filepath.Join(outputDir, test), butane_config) default: plog.Fatalf("Unknown test name:%s", test) } @@ -654,111 +599,3 @@ func testAsDisk(ctx context.Context, outdir string) (time.Duration, error) { return awaitCompletion(ctx, mach, outdir, completionChannel, nil, []string{liveOKSignal}) } - -// iscsi_butane_setup.yaml contains the full butane config but here is an overview of the setup -// 1 - Boot a live ISO with two extra 10G disks with labels "target" and "var" -// - Format and mount `virtio-var` to /var -// -// 2 - target.container -> start an iscsi target, using quay.io/coreos-assembler/targetcli -// 3 - setup-targetcli.service calls /usr/local/bin/targetcli_script: -// - instructs targetcli to serve /dev/disk/by-id/virtio-target as an iscsi target -// - disables authentication -// - verifies the iscsi service is active and reachable -// -// 4 - install-coreos-to-iscsi-target.service calls /usr/local/bin/install-coreos-iscsi: -// - mount iscsi target -// - run coreos-installer on the mounted block device -// - unmount iscsi -// -// 5 - coreos-iscsi-vm.container -> start a coreos-assembler conainer: -// - launch kola qemuexec instructing it to boot from an iPXE script -// wich in turns mount the iscsi target and load kernel -// - note the virtserial port device: we pass through the serial port -// that was created by kola for test completion -// -// 6 - /var/nested-ign.json contains an ignition config: -// - when the system is booted, write a success string to /dev/virtio-ports/testisocompletion -// - as this serial device is mapped to the host serial device, the test concludes -func testLiveInstalliscsi(ctx context.Context, inst qemu.Install, outdir string, butane string) (time.Duration, error) { - - builddir := kola.CosaBuild.Dir - isopath := filepath.Join(builddir, kola.CosaBuild.Meta.BuildArtifacts.LiveIso.Path) - builder, err := newBaseQemuBuilder(outdir) - if err != nil { - return 0, err - } - defer builder.Close() - if err := builder.AddIso(isopath, "", false); err != nil { - return 0, err - } - - completionChannel, err := builder.VirtioChannelRead("testisocompletion") - if err != nil { - return 0, err - } - - // Create a serial channel to read the logs from the nested VM - nestedVmLogsChannel, err := builder.VirtioChannelRead("nestedvmlogs") - if err != nil { - return 0, err - } - - // Create a file to write the contents of the serial channel into - nestedVMConsole, err := os.OpenFile(filepath.Join(outdir, "nested_vm_console.txt"), os.O_WRONLY|os.O_CREATE, 0644) - if err != nil { - return 0, err - } - - go func() { - _, err := io.Copy(nestedVMConsole, nestedVmLogsChannel) - if err != nil && err != io.EOF { - panic(err) - } - }() - - // empty disk to use as an iscsi target to install coreOS on and subseqently boot - // Also add a 10G disk that we will mount on /var, to increase space available when pulling containers - err = builder.AddDisksFromSpecs([]string{"10G:serial=target", "10G:serial=var"}) - if err != nil { - return 0, err - } - - // We need more memory to start another VM within ! - builder.MemoryMiB = 2048 - - var iscsiTargetConfig = conf.Butane(butane) - - config, err := iscsiTargetConfig.Render(conf.FailWarnings) - if err != nil { - return 0, err - } - err = forwardJournal(outdir, builder, config) - if err != nil { - return 0, err - } - - // Add a failure target to stop the test if something go wrong rather than waiting for the 10min timeout - config.AddSystemdUnit("coreos-test-entered-emergency-target.service", signalFailureUnit, conf.Enable) - - // enable network - builder.EnableUsermodeNetworking([]platform.HostForwardPort{}, "") - - // keep auto-login enabled for easier debug when running console - config.AddAutoLogin() - - builder.SetConfig(config) - - // Bind mount in the COSA rootfs into the VM so we can use it as a - // read-only rootfs for quickly starting the container to kola - // qemuexec the nested VM for the test. See resources/iscsi_butane_setup.yaml - builder.MountHost("/", "/var/cosaroot", true) - config.MountHost("/var/cosaroot", true) - - mach, err := builder.Exec() - if err != nil { - return 0, errors.Wrapf(err, "running iso") - } - defer mach.Destroy() - - return awaitCompletion(ctx, mach, outdir, completionChannel, nil, []string{"iscsi-boot-ok"}) -} diff --git a/mantle/cmd/kola/resources/iscsi_butane_setup.yaml b/mantle/kola/tests/iso/iscsi_butane_setup.yaml similarity index 100% rename from mantle/cmd/kola/resources/iscsi_butane_setup.yaml rename to mantle/kola/tests/iso/iscsi_butane_setup.yaml diff --git a/mantle/kola/tests/iso/live-iso.go b/mantle/kola/tests/iso/live-iso.go index c23b81240e..42ee9e672b 100644 --- a/mantle/kola/tests/iso/live-iso.go +++ b/mantle/kola/tests/iso/live-iso.go @@ -2,6 +2,7 @@ package testiso import ( "bufio" + _ "embed" "fmt" "io" "os" @@ -81,6 +82,29 @@ var ( "pxe-online-install.rootfs-appended.s390fw", "pxe-offline-install.s390fw", } + tests_iscsi_x86_64 = []string{ + "iso-offline-install-iscsi.ibft.uefi", + "iso-offline-install-iscsi.ibft-with-mpath.bios", + "iso-offline-install-iscsi.manual.bios", + } + tests_iscsi_s390x = []string{ + // FIXME https://github.com/coreos/fedora-coreos-tracker/issues/1657 + //"iso-offline-install-iscsi.ibft.s390fw", + //"iso-offline-install-iscsi.ibft-with-mpath.s390fw", + //"iso-offline-install-iscsi.manual.s390fw", + } + tests_iscsi_ppc64le = []string{ + // FIXME https://github.com/coreos/fedora-coreos-tracker/issues/1657 + //"iso-offline-install-iscsi.ibft.ppcfw", + //"iso-offline-install-iscsi.ibft-with-mpath.ppcfw", + //"iso-offline-install-iscsi.manual.ppcfw", + } + tests_iscsi_aarch64 = []string{ + // FIXME https://github.com/coreos/fedora-coreos-tracker/issues/1657 + //"iso-offline-install-iscsi.ibft.uefi", + //"iso-offline-install-iscsi.ibft-with-mpath.uefi", + //"iso-offline-install-iscsi.manual.uefi", + } ) func getAllLiveIsoTests() []string { @@ -115,6 +139,22 @@ func getAllPxeTests() []string { } } +func getAllIscsiTests() []string { + arch := coreosarch.CurrentRpmArch() + switch arch { + case "x86_64": + return tests_iscsi_x86_64 + case "aarch64": + return tests_iscsi_aarch64 + case "ppc64le": + return tests_iscsi_ppc64le + case "s390x": + return tests_iscsi_s390x + default: + return []string{} + } +} + func getIsoTestOpts(testName string) IsoTestOpts { opts := IsoTestOpts{} @@ -143,6 +183,12 @@ func getIsoTestOpts(testName string) IsoTestOpts { if strings.Contains(testName, "rootfs-appended") { opts.pxeAppendRootfs = true } + if strings.Contains(testName, "ibft") { + opts.enableIbft = true + } + if strings.Contains(testName, "manual") { + opts.manual = true + } opts.SetInsecureOnDevBuild() return opts @@ -179,6 +225,22 @@ func init() { Platforms: []string{"qemu"}, }) } + + // iSCSI tests + for _, testName := range getAllIscsiTests() { + register.RegisterTest(®ister.Test{ + Run: func(c cluster.TestCluster) { + opts := getIsoTestOpts(testName) + isoInstalliScsi(c, opts) + }, + ClusterSize: 0, + Name: "iso." + testName, + Description: "Verify iSCSI install works.", + Timeout: 12 * time.Minute, + Flags: []register.Flag{}, + Platforms: []string{"qemu"}, + }) + } } var liveOKSignal = "live-test-OK" @@ -375,6 +437,8 @@ type IsoTestOpts struct { isMiniso bool enableUefi bool enableUefiSecure bool + enableIbft bool + manual bool pxeAppendRootfs bool } @@ -690,6 +754,141 @@ func testPXE(c cluster.TestCluster, opts IsoTestOpts) { } } +//go:embed iscsi_butane_setup.yaml +var iscsi_butane_config string + +// iscsi_butane_setup.yaml contains the full butane config but here is an overview of the setup +// 1 - Boot a live ISO with two extra 10G disks with labels "target" and "var" +// - Format and mount `virtio-var` to /var +// +// 2 - target.container -> start an iscsi target, using quay.io/coreos-assembler/targetcli +// 3 - setup-targetcli.service calls /usr/local/bin/targetcli_script: +// - instructs targetcli to serve /dev/disk/by-id/virtio-target as an iscsi target +// - disables authentication +// - verifies the iscsi service is active and reachable +// +// 4 - install-coreos-to-iscsi-target.service calls /usr/local/bin/install-coreos-iscsi: +// - mount iscsi target +// - run coreos-installer on the mounted block device +// - unmount iscsi +// +// 5 - coreos-iscsi-vm.container -> start a coreos-assembler conainer: +// - launch kola qemuexec instructing it to boot from an iPXE script +// wich in turns mount the iscsi target and load kernel +// - note the virtserial port device: we pass through the serial port +// that was created by kola for test completion +// +// 6 - /var/nested-ign.json contains an ignition config: +// - when the system is booted, write a success string to /dev/virtio-ports/testisocompletion +// - as this serial device is mapped to the host serial device, the test concludes +func isoInstalliScsi(c cluster.TestCluster, opts IsoTestOpts) { + var outdir string + //var qc *qemu.Cluster + switch pc := c.Cluster.(type) { + case *qemu.Cluster: + outdir = pc.RuntimeConf().OutputDir + //qc = pc + default: + c.Fatalf("Unsupported cluster type") + } + + var butane string + if opts.enableIbft && opts.enableMultipath { + butane = strings.ReplaceAll(iscsi_butane_config, "COREOS_INSTALLER_KARGS", "--append-karg rd.iscsi.firmware=1 --append-karg rd.multipath=default --append-karg root=/dev/disk/by-label/dm-mpath-root --append-karg rw") + } else if opts.enableIbft { + butane = strings.ReplaceAll(iscsi_butane_config, "COREOS_INSTALLER_KARGS", "--append-karg rd.iscsi.firmware=1") + } else if opts.manual { + butane = strings.ReplaceAll(iscsi_butane_config, "COREOS_INSTALLER_KARGS", "--append-karg netroot=iscsi:10.0.2.15::::iqn.2024-05.com.coreos:0") + } + + if kola.CosaBuild.Meta.BuildArtifacts.LiveIso == nil || kola.CosaBuild.Meta.BuildArtifacts.LiveKernel == nil { + c.Fatalf("build %s is missing live artifacts", kola.CosaBuild.Meta.Name) + } + builddir := kola.CosaBuild.Dir + isopath := filepath.Join(builddir, kola.CosaBuild.Meta.BuildArtifacts.LiveIso.Path) + builder, err := newBaseQemuBuilder(opts, outdir) + if err != nil { + c.Fatal(err) + } + defer builder.Close() + if err := builder.AddIso(isopath, "", false); err != nil { + c.Fatal(err) + } + + completionChannel, err := builder.VirtioChannelRead("testisocompletion") + if err != nil { + c.Fatal(err) + } + + // Create a serial channel to read the logs from the nested VM + nestedVmLogsChannel, err := builder.VirtioChannelRead("nestedvmlogs") + if err != nil { + c.Fatal(err) + } + + // Create a file to write the contents of the serial channel into + nestedVMConsole, err := os.OpenFile(filepath.Join(outdir, "nested_vm_console.txt"), os.O_WRONLY|os.O_CREATE, 0644) + if err != nil { + c.Fatal(err) + } + + go func() { + _, err := io.Copy(nestedVMConsole, nestedVmLogsChannel) + if err != nil && err != io.EOF { + panic(err) + } + }() + + // empty disk to use as an iscsi target to install coreOS on and subseqently boot + // Also add a 10G disk that we will mount on /var, to increase space available when pulling containers + err = builder.AddDisksFromSpecs([]string{"10G:serial=target", "10G:serial=var"}) + if err != nil { + c.Fatal(err) + } + + // We need more memory to start another VM within ! + builder.MemoryMiB = 2048 + + var iscsiTargetConfig = conf.Butane(butane) + + config, err := iscsiTargetConfig.Render(conf.FailWarnings) + if err != nil { + c.Fatal(err) + } + err = forwardJournal(outdir, builder, config) + if err != nil { + c.Fatal(err) + } + + // Add a failure target to stop the test if something go wrong rather than waiting for the 10min timeout + config.AddSystemdUnit("coreos-test-entered-emergency-target.service", signalFailureUnit, conf.Enable) + + // enable network + builder.EnableUsermodeNetworking([]platform.HostForwardPort{}, "") + + // keep auto-login enabled for easier debug when running console + config.AddAutoLogin() + + builder.SetConfig(config) + + // Bind mount in the COSA rootfs into the VM so we can use it as a + // read-only rootfs for quickly starting the container to kola + // qemuexec the nested VM for the test. See resources/iscsi_butane_setup.yaml + builder.MountHost("/", "/var/cosaroot", true) + config.MountHost("/var/cosaroot", true) + + mach, err := builder.Exec() + if err != nil { + c.Fatal(err) + } + defer mach.Destroy() + + err = awaitCompletion(c, mach, opts.console, outdir, completionChannel, nil, []string{"iscsi-boot-ok"}) + if err != nil { + c.Fatal(err) + } +} + func awaitCompletion(c cluster.TestCluster, inst *platform.QemuInstance, console bool, outdir string, qchan *os.File, booterrchan chan error, expected []string) error { ctx := c.Context() From 8059f5923979f0ef10bb2d1268fd75adde094d38 Mon Sep 17 00:00:00 2001 From: Nikita Dubrovskii Date: Fri, 21 Nov 2025 11:05:56 +0100 Subject: [PATCH 08/31] kola/tests/iso: fold testiso iso-as-disk tests --- mantle/cmd/kola/testiso.go | 123 +++--------------------------- mantle/kola/tests/iso/live-iso.go | 69 +++++++++++++++++ 2 files changed, 78 insertions(+), 114 deletions(-) diff --git a/mantle/cmd/kola/testiso.go b/mantle/cmd/kola/testiso.go index abd0452c85..6f78e2da59 100644 --- a/mantle/cmd/kola/testiso.go +++ b/mantle/cmd/kola/testiso.go @@ -22,7 +22,6 @@ package main import ( "bufio" "context" - _ "embed" "fmt" "io" "os" @@ -36,7 +35,6 @@ import ( "github.com/coreos/coreos-assembler/mantle/harness/testresult" "github.com/coreos/coreos-assembler/mantle/platform/conf" "github.com/coreos/coreos-assembler/mantle/platform/machine/qemu" - "github.com/coreos/coreos-assembler/mantle/util" coreosarch "github.com/coreos/stream-metadata-go/arch" "github.com/pkg/errors" @@ -56,29 +54,14 @@ var ( SilenceUsage: true, } - instInsecure bool - + instInsecure bool pxeKernelArgs []string - - console bool - - enable4k bool - enableUefi bool - enableUefiSecure bool - + console bool + enableUefi bool // These tests only run on RHCOS tests_RHCOS_uefi = []string{ "iso-fips.uefi", } - - // The iso-as-disk tests are only supported in x86_64 because other - // architectures don't have the required hybrid partition table. - tests_x86_64 = []string{ - "iso-as-disk.bios", - "iso-as-disk.uefi", - "iso-as-disk.uefi-secure", - "iso-as-disk.4k.uefi", - } ) const ( @@ -116,24 +99,6 @@ ExecStart=/bin/sh -c '/usr/bin/echo %s >/dev/virtio-ports/testisocompletion && s RequiredBy=emergency.target `, signalEmergencyString) -// This test is broken. Please fix! -// https://github.com/coreos/coreos-assembler/issues/3554 -var verifyNoEFIBootEntry = `[Unit] -Description=TestISO Verify No EFI Boot Entry -OnFailure=emergency.target -OnFailureJobMode=isolate -ConditionPathExists=/sys/firmware/efi -Before=live-signal-ok.service -[Service] -Type=oneshot -RemainAfterExit=yes -ExecStart=/bin/sh -c '! efibootmgr -v | grep -E "(HD|CDROM)\("' -[Install] -# for install tests -RequiredBy=coreos-installer.target -# for iso-as-disk -RequiredBy=multi-user.target` - func init() { cmdTestIso.Flags().BoolVarP(&instInsecure, "inst-insecure", "S", false, "Do not verify signature on metal image") cmdTestIso.Flags().BoolVar(&console, "console", false, "Connect qemu console to terminal, turn off automatic initramfs failure checking") @@ -150,26 +115,17 @@ func liveArtifactExistsInBuild() error { return nil } -func getAllTests(build *util.LocalBuild) []string { +func getAllTests() []string { arch := coreosarch.CurrentRpmArch() - var tests []string - switch arch { - case "x86_64": - tests = tests_x86_64 - default: - return []string{} - } if kola.CosaBuild.Meta.Name == "rhcos" && arch != "s390x" && arch != "ppc64le" { - tests = append(tests, tests_RHCOS_uefi...) + return tests_RHCOS_uefi } - return tests + return []string{} } func newBaseQemuBuilder(outdir string) (*platform.QemuBuilder, error) { builder := qemu.NewMetalQemuBuilderDefault() - if enableUefiSecure { - builder.Firmware = "uefi-secure" - } else if enableUefi { + if enableUefi { builder.Firmware = "uefi" } @@ -249,7 +205,7 @@ func runTestIso(cmd *cobra.Command, args []string) (err error) { if kola.CosaBuild == nil { return fmt.Errorf("Must provide --build") } - tests := getAllTests(kola.CosaBuild) + tests := getAllTests() if len(args) != 0 { if tests, err = filterTests(tests, args); err != nil { return err @@ -300,23 +256,6 @@ func runTestIso(cmd *cobra.Command, args []string) (err error) { } }() - baseInst := qemu.Install{ - CosaBuild: kola.CosaBuild, - NmKeyfiles: make(map[string]string), - } - - if instInsecure { - baseInst.Insecure = true - fmt.Printf("Ignoring verification of signature on metal image\n") - } - - // Ignore signing verification by default when running with development build - // https://github.com/coreos/fedora-coreos-tracker/issues/908 - if !baseInst.Insecure && strings.Contains(kola.CosaBuild.Meta.BuildID, ".dev.") { - baseInst.Insecure = true - fmt.Printf("Detected development build; disabling signature verification\n") - } - var duration time.Duration atLeastOneFailed := false @@ -328,29 +267,16 @@ func runTestIso(cmd *cobra.Command, args []string) (err error) { return err } - enable4k = false enableUefi = false - enableUefiSecure = false - inst := baseInst // Pretend this is Rust and I wrote .copy() fmt.Printf("Running test: %s\n", test) components := strings.Split(test, ".") - inst.PxeAppendRootfs = kola.HasString("rootfs-appended", components) - - if kola.HasString("4k", components) { - enable4k = true - inst.Native4k = true - } - if kola.HasString("uefi-secure", components) { - enableUefiSecure = true - } else if kola.HasString("uefi", components) { + if kola.HasString("uefi", components) { enableUefi = true } switch components[0] { - case "iso-as-disk": - duration, err = testAsDisk(ctx, filepath.Join(outputDir, test)) case "iso-fips": duration, err = testLiveFIPS(ctx, filepath.Join(outputDir, test)) default: @@ -568,34 +494,3 @@ RequiredBy=fips-signal-ok.service return awaitCompletion(ctx, mach, outdir, completionChannel, nil, []string{liveOKSignal}) } - -func testAsDisk(ctx context.Context, outdir string) (time.Duration, error) { - builddir := kola.CosaBuild.Dir - isopath := filepath.Join(builddir, kola.CosaBuild.Meta.BuildArtifacts.LiveIso.Path) - builder, config, err := newQemuBuilder(outdir) - if err != nil { - return 0, err - } - defer builder.Close() - // Drop the bootindex bit (applicable to all arches except s390x and ppc64le); we want it to be the default - if err := builder.AddIso(isopath, "", true); err != nil { - return 0, err - } - - completionChannel, err := builder.VirtioChannelRead("testisocompletion") - if err != nil { - return 0, err - } - - config.AddSystemdUnit("live-signal-ok.service", liveSignalOKUnit, conf.Enable) - config.AddSystemdUnit("verify-no-efi-boot-entry.service", verifyNoEFIBootEntry, conf.Enable) - builder.SetConfig(config) - - mach, err := builder.Exec() - if err != nil { - return 0, errors.Wrapf(err, "running iso") - } - defer mach.Destroy() - - return awaitCompletion(ctx, mach, outdir, completionChannel, nil, []string{liveOKSignal}) -} diff --git a/mantle/kola/tests/iso/live-iso.go b/mantle/kola/tests/iso/live-iso.go index 42ee9e672b..36466ee880 100644 --- a/mantle/kola/tests/iso/live-iso.go +++ b/mantle/kola/tests/iso/live-iso.go @@ -105,6 +105,15 @@ var ( //"iso-offline-install-iscsi.ibft-with-mpath.uefi", //"iso-offline-install-iscsi.manual.uefi", } + + // The iso-as-disk tests are only supported in x86_64 because other + // architectures don't have the required hybrid partition table. + tests_as_disk_x86_64 = []string{ + "iso-as-disk.bios", + "iso-as-disk.uefi", + "iso-as-disk.uefi-secure", + "iso-as-disk.4k.uefi", + } ) func getAllLiveIsoTests() []string { @@ -241,6 +250,23 @@ func init() { Platforms: []string{"qemu"}, }) } + + // as-disk tests + for _, testName := range tests_as_disk_x86_64 { + register.RegisterTest(®ister.Test{ + Run: func(c cluster.TestCluster) { + opts := getIsoTestOpts(testName) + isoTestAsDisk(c, opts) + }, + ClusterSize: 0, + Name: "iso." + testName, + Description: "Verify ISO-as-disk install works.", + Timeout: 12 * time.Minute, + Flags: []register.Flag{}, + Platforms: []string{"qemu"}, + }) + } + } var liveOKSignal = "live-test-OK" @@ -1004,3 +1030,46 @@ func awaitCompletion(c cluster.TestCluster, inst *platform.QemuInstance, console } return err } + +func isoTestAsDisk(c cluster.TestCluster, opts IsoTestOpts) { + var outdir string + //var qc *qemu.Cluster + switch pc := c.Cluster.(type) { + case *qemu.Cluster: + outdir = pc.RuntimeConf().OutputDir + //qc = pc + default: + c.Fatalf("Unsupported cluster type") + } + + isopath := filepath.Join(kola.CosaBuild.Dir, kola.CosaBuild.Meta.BuildArtifacts.LiveIso.Path) + builder, config, err := newQemuBuilder(opts, outdir) + if err != nil { + c.Fatal(err) + } + defer builder.Close() + // Drop the bootindex bit (applicable to all arches except s390x and ppc64le); we want it to be the default + if err := builder.AddIso(isopath, "", true); err != nil { + c.Fatal(err) + } + + completionChannel, err := builder.VirtioChannelRead("testisocompletion") + if err != nil { + c.Fatal(err) + } + + config.AddSystemdUnit("live-signal-ok.service", liveSignalOKUnit, conf.Enable) + config.AddSystemdUnit("verify-no-efi-boot-entry.service", verifyNoEFIBootEntry, conf.Enable) + builder.SetConfig(config) + + mach, err := builder.Exec() + if err != nil { + c.Fatal(err) + } + defer mach.Destroy() + + err = awaitCompletion(c, mach, opts.console, outdir, completionChannel, nil, []string{liveOKSignal}) + if err != nil { + c.Fatal(err) + } +} From 548e343cb3165ea70d7488c33cd932b408e8b58f Mon Sep 17 00:00:00 2001 From: Nikita Dubrovskii Date: Fri, 21 Nov 2025 11:29:06 +0100 Subject: [PATCH 09/31] kola/tests/iso: refactor live-iso.go into independent test files Split the monolithic live-iso.go file into several focused files where each file contains a specific set of tests: - live-pxe.go: PXE network boot tests - live-iscsi.go: iSCSI boot and installation tests - live-as-disk.go: ISO-as-disk installation tests - common.go: Shared test utilities and helper functions This improves code organization and makes it easier to locate and maintain specific test suites. --- mantle/kola/tests/iso/common.go | 498 ++++++++++++++ mantle/kola/tests/iso/live-as-disk.go | 81 +++ mantle/kola/tests/iso/live-iscsi.go | 243 +++++++ mantle/kola/tests/iso/live-iso.go | 905 +------------------------- mantle/kola/tests/iso/live-login.go | 2 +- mantle/kola/tests/iso/live-pxe.go | 172 +++++ 6 files changed, 1000 insertions(+), 901 deletions(-) create mode 100644 mantle/kola/tests/iso/common.go create mode 100644 mantle/kola/tests/iso/live-as-disk.go create mode 100644 mantle/kola/tests/iso/live-iscsi.go create mode 100644 mantle/kola/tests/iso/live-pxe.go diff --git a/mantle/kola/tests/iso/common.go b/mantle/kola/tests/iso/common.go new file mode 100644 index 0000000000..5d2fc0bca6 --- /dev/null +++ b/mantle/kola/tests/iso/common.go @@ -0,0 +1,498 @@ +package iso + +import ( + "bufio" + "fmt" + "io" + "os" + "path/filepath" + "strconv" + "strings" + "time" + + "github.com/coreos/coreos-assembler/mantle/kola" + "github.com/coreos/coreos-assembler/mantle/kola/cluster" + "github.com/coreos/coreos-assembler/mantle/platform" + "github.com/coreos/coreos-assembler/mantle/platform/conf" + "github.com/coreos/coreos-assembler/mantle/platform/machine/qemu" + coreosarch "github.com/coreos/stream-metadata-go/arch" + "github.com/pkg/errors" +) + +const ( + installTimeoutMins = 12 + // https://github.com/coreos/fedora-coreos-config/pull/2544 + liveISOFromRAMKarg = "coreos.liveiso.fromram" +) + +type IsoTestOpts struct { + // Flags().BoolVarP(&instInsecure, "inst-insecure", "S", false, "Do not verify signature on metal image") + instInsecure bool + // Flags().StringSliceVar(&pxeKernelArgs, "pxe-kargs", nil, "Additional kernel arguments for PXE") + pxeKernelArgs []string + // Flags().BoolVar(&console, "console", false, "Connect qemu console to terminal, turn off automatic initramfs failure checking") + console bool + addNmKeyfile bool + enable4k bool + enableMultipath bool + isOffline bool + isISOFromRAM bool + isMiniso bool + enableUefi bool + enableUefiSecure bool + enableIbft bool + manual bool + pxeAppendRootfs bool +} + +func getIsoTestOpts(testName string) IsoTestOpts { + opts := IsoTestOpts{} + + // Parse test name to determine options + if strings.Contains(testName, "4k") { + opts.enable4k = true + } + if strings.Contains(testName, "uefi-secure") { + opts.enableUefiSecure = true + } else if strings.Contains(testName, "uefi") { + opts.enableUefi = true + } + if strings.Contains(testName, "mpath") { + opts.enableMultipath = true + } + if strings.Contains(testName, "offline") { + opts.isOffline = true + } + if strings.Contains(testName, "fromram") { + opts.isISOFromRAM = true + } + if strings.Contains(testName, "miniso") { + opts.isMiniso = true + } + if strings.Contains(testName, ".nm") { + opts.addNmKeyfile = true + } + if strings.Contains(testName, "rootfs-appended") { + opts.pxeAppendRootfs = true + } + if strings.Contains(testName, "ibft") { + opts.enableIbft = true + } + if strings.Contains(testName, "manual") { + opts.manual = true + } + + opts.SetInsecureOnDevBuild() + return opts +} + +func (o *IsoTestOpts) SetInsecureOnDevBuild() { + // Ignore signing verification by default when running with development build + // https://github.com/coreos/fedora-coreos-tracker/issues/908 + if strings.Contains(kola.CosaBuild.Meta.BuildID, ".dev.") { + o.instInsecure = true + //fmt.Printf("Detected development build; disabling signature verification\n") + } +} + +func newBaseQemuBuilder(opts IsoTestOpts, outdir string) (*platform.QemuBuilder, error) { + builder := qemu.NewMetalQemuBuilderDefault() + if opts.enableUefiSecure { + builder.Firmware = "uefi-secure" + } else if opts.enableUefi { + builder.Firmware = "uefi" + } + + if err := os.MkdirAll(outdir, 0755); err != nil { + return nil, err + } + + builder.InheritConsole = opts.console + if !opts.console { + builder.ConsoleFile = filepath.Join(outdir, "console.txt") + } + + if kola.QEMUOptions.Memory != "" { + parsedMem, err := strconv.ParseInt(kola.QEMUOptions.Memory, 10, 32) + if err != nil { + return nil, err + } + builder.MemoryMiB = int(parsedMem) + } + + // increase the memory for pxe tests with appended rootfs in the initrd + // we were bumping up into the 4GiB limit in RHCOS/c9s + // pxe-offline-install.rootfs-appended.bios tests + if opts.pxeAppendRootfs && builder.MemoryMiB < 5120 { + builder.MemoryMiB = 5120 + } + + return builder, nil +} + +func newQemuBuilder(opts IsoTestOpts, outdir string) (*platform.QemuBuilder, *conf.Conf, error) { + builder, err := newBaseQemuBuilder(opts, outdir) + if err != nil { + return nil, nil, err + } + + config, err := conf.EmptyIgnition().Render(conf.FailWarnings) + if err != nil { + return nil, nil, err + } + + err = forwardJournal(outdir, builder, config) + if err != nil { + return nil, nil, err + } + + return builder, config, nil +} + +func forwardJournal(outdir string, builder *platform.QemuBuilder, config *conf.Conf) error { + journalPipe, err := builder.VirtioJournal(config, "") + if err != nil { + return err + } + journalOut, err := os.OpenFile(filepath.Join(outdir, "journal.txt"), os.O_WRONLY|os.O_CREATE, 0644) + if err != nil { + return err + } + + go func() { + _, err := io.Copy(journalOut, journalPipe) + if err != nil && err != io.EOF { + panic(err) + } + }() + + return nil +} + +func newQemuBuilderWithDisk(opts IsoTestOpts, outdir string) (*platform.QemuBuilder, *conf.Conf, error) { + builder, config, err := newQemuBuilder(opts, outdir) + + if err != nil { + return nil, nil, err + } + + sectorSize := 0 + if opts.enable4k { + sectorSize = 4096 + } + + disk := platform.Disk{ + Size: "12G", // Arbitrary + SectorSize: sectorSize, + MultiPathDisk: opts.enableMultipath, + } + + //TBD: see if we can remove this and just use AddDisk and inject bootindex during startup + if coreosarch.CurrentRpmArch() == "s390x" || coreosarch.CurrentRpmArch() == "aarch64" { + // s390x and aarch64 need to use bootindex as they don't support boot once + if err := builder.AddDisk(&disk); err != nil { + return nil, nil, err + } + } else { + if err := builder.AddPrimaryDisk(&disk); err != nil { + return nil, nil, err + } + } + + return builder, config, nil +} + +func awaitCompletion(c cluster.TestCluster, inst *platform.QemuInstance, console bool, outdir string, qchan *os.File, booterrchan chan error, expected []string) error { + ctx := c.Context() + + errchan := make(chan error) + go func() { + timeout := (time.Duration(installTimeoutMins*(100+kola.Options.ExtendTimeoutPercent)) * time.Minute) / 100 + time.Sleep(timeout) + errchan <- fmt.Errorf("timed out after %v", timeout) + }() + if !console { + go func() { + errBuf, err := inst.WaitIgnitionError(ctx) + if err == nil { + if errBuf != "" { + c.Logf("entered emergency.target in initramfs") + path := filepath.Join(outdir, "ignition-virtio-dump.txt") + if err := os.WriteFile(path, []byte(errBuf), 0644); err != nil { + c.Errorf("Failed to write journal: %v", err) + } + err = platform.ErrInitramfsEmergency + } + } + if err != nil { + errchan <- err + } + }() + } + go func() { + err := inst.Wait() + // only one Wait() gets process data, so also manually check for signal + //plog.Debugf("qemu exited err=%v", err) + if err == nil && inst.Signaled() { + err = errors.New("process killed") + } + if err != nil { + errchan <- errors.Wrapf(err, "QEMU unexpectedly exited while awaiting completion") + } + time.Sleep(1 * time.Minute) + errchan <- fmt.Errorf("QEMU exited; timed out waiting for completion") + }() + go func() { + r := bufio.NewReader(qchan) + for _, exp := range expected { + l, err := r.ReadString('\n') + if err != nil { + if err == io.EOF { + // this may be from QEMU getting killed or exiting; wait a bit + // to give a chance for .Wait() above to feed the channel with a + // better error + time.Sleep(1 * time.Second) + errchan <- fmt.Errorf("Got EOF from completion channel, %s expected", exp) + } else { + errchan <- errors.Wrapf(err, "reading from completion channel") + } + return + } + line := strings.TrimSpace(l) + if line != exp { + errchan <- fmt.Errorf("Unexpected string from completion channel: %s expected: %s", line, exp) + return + } + } + // OK! + errchan <- nil + }() + go func() { + //check for error when switching boot order + if booterrchan != nil { + if err := <-booterrchan; err != nil { + errchan <- err + } + } + }() + err := <-errchan + if err == nil { + // No error so far, check the console and journal files + consoleFile := filepath.Join(outdir, "console.txt") + journalFile := filepath.Join(outdir, "journal.txt") + files := []string{consoleFile, journalFile} + for _, file := range files { + fileName := filepath.Base(file) + // Check if the file exists + _, err := os.Stat(file) + if os.IsNotExist(err) { + fmt.Printf("The file: %v does not exist\n", fileName) + continue + } else if err != nil { + fmt.Println(err) + return err + } + // Read the contents of the file + fileContent, err := os.ReadFile(file) + if err != nil { + fmt.Println(err) + return err + } + // Check for badness with CheckConsole + warnOnly, badlines := kola.CheckConsole([]byte(fileContent), nil) + if len(badlines) > 0 { + for _, badline := range badlines { + if warnOnly { + c.Errorf("bad log line detected: %v", badline) + } else { + c.Logf("bad log line detected: %v", badline) + } + } + if !warnOnly { + err = fmt.Errorf("errors found in log files") + return err + } + } + } + } + return err +} + +var liveOKSignal = "live-test-OK" +var liveSignalOKUnit = fmt.Sprintf(` +[Unit] +Description=TestISO Signal Live ISO Completion +Requires=dev-virtio\\x2dports-testisocompletion.device +OnFailure=emergency.target +OnFailureJobMode=isolate +Before=coreos-installer.service +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=/bin/sh -c '/usr/bin/echo %s >/dev/virtio-ports/testisocompletion' +[Install] +# for install tests +RequiredBy=coreos-installer.target +# for iso-as-disk +RequiredBy=multi-user.target`, liveOKSignal) + +var signalCompleteString = "coreos-installer-test-OK" +var signalCompletionUnit = fmt.Sprintf(` +[Unit] +Description=TestISO Signal Completion +Requires=dev-virtio\\x2dports-testisocompletion.device +OnFailure=emergency.target +OnFailureJobMode=isolate +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=/bin/sh -c '/usr/bin/echo %s >/dev/virtio-ports/testisocompletion && systemctl poweroff' +[Install] +RequiredBy=multi-user.target`, signalCompleteString) + +var signalEmergencyString = "coreos-installer-test-entered-emergency-target" +var signalFailureUnit = fmt.Sprintf(` +[Unit] +Description=TestISO Signal Failure +Requires=dev-virtio\\x2dports-testisocompletion.device +DefaultDependencies=false +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=/bin/sh -c '/usr/bin/echo %s >/dev/virtio-ports/testisocompletion && systemctl poweroff' +[Install] +RequiredBy=emergency.target`, signalEmergencyString) + +var multipathedRoot = `[Unit] +Description=TestISO Verify Multipathed Root +OnFailure=emergency.target +OnFailureJobMode=isolate +Before=coreos-test-installer.service +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=/bin/bash -c 'lsblk -pno NAME "/dev/mapper/$(multipath -l -v 1)" | grep -qw "$(findmnt -nvr /sysroot -o SOURCE)"' +[Install] +RequiredBy=multi-user.target` + +var checkNoIgnition = ` +[Unit] +Description=TestISO Verify No Ignition Config +OnFailure=emergency.target +OnFailureJobMode=isolate +Before=coreos-test-installer.service +After=coreos-ignition-firstboot-complete.service +RequiresMountsFor=/boot +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=/bin/sh -c '[ ! -e /boot/ignition ]' +[Install] +RequiredBy=multi-user.target` + +// This test is broken. Please fix! +// https://github.com/coreos/coreos-assembler/issues/3554 +var verifyNoEFIBootEntry = ` +[Unit] +Description=TestISO Verify No EFI Boot Entry +OnFailure=emergency.target +OnFailureJobMode=isolate +ConditionPathExists=/sys/firmware/efi +Before=live-signal-ok.service +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=/bin/sh -c '! efibootmgr -v | grep -E "(HD|CDROM)\("' +[Install] +# for install tests +RequiredBy=coreos-installer.target +# for iso-as-disk +RequiredBy=multi-user.target` + +// Verify that the volume ID is the OS name. See also +// https://github.com/openshift/assisted-image-service/pull/477. +// This is the same as the LABEL of the block device for ISO9660. See +// https://github.com/util-linux/util-linux/blob/643bdae8e38055e36acf2963c3416de206081507/libblkid/src/superblocks/iso9660.c#L366-L377 +var verifyIsoVolumeId = ` +[Unit] +Description=Verify ISO Volume ID +OnFailure=emergency.target +OnFailureJobMode=isolate +# only if we're actually mounting the ISO +ConditionPathIsMountPoint=/run/media/iso +[Service] +Type=oneshot +RemainAfterExit=yes +# the backing device name is arch-dependent, but we know it's mounted on /run/media/iso +ExecStart=bash -c "[[ $(findmnt -no LABEL /run/media/iso) == %s-* ]]" +[Install] +RequiredBy=coreos-installer.target` + +// Unit to check that /run/media/iso is not mounted when +// coreos.liveiso.fromram kernel argument is passed +var isoNotMountedUnit = ` +[Unit] +Description=Verify ISO is not mounted when coreos.liveiso.fromram +OnFailure=emergency.target +OnFailureJobMode=isolate +ConditionKernelCommandLine=coreos.liveiso.fromram +[Service] +Type=oneshot +StandardOutput=kmsg+console +StandardError=kmsg+console +RemainAfterExit=yes +# Would like to use SuccessExitStatus but it doesn't support what +# we want: https://github.com/systemd/systemd/issues/10297#issuecomment-1672002635 +ExecStart=bash -c "if mountpoint /run/media/iso 2>/dev/null; then exit 1; fi" +[Install] +RequiredBy=coreos-installer.target` + +var nmConnectionId = "CoreOS DHCP" +var nmConnectionFile = "coreos-dhcp.nmconnection" +var nmConnection = fmt.Sprintf(`[connection] +id=%s +type=ethernet +# add wait-device-timeout here so we make sure NetworkManager-wait-online.service will +# wait for a device to be present before exiting. See +# https://github.com/coreos/fedora-coreos-tracker/issues/1275#issuecomment-1231605438 +wait-device-timeout=20000 + +[ipv4] +method=auto +`, nmConnectionId) + +var nmstateConfigFile = "/etc/nmstate/br-ex.yml" +var nmstateConfig = `interfaces: + - name: br-ex + type: linux-bridge + state: up + ipv4: + enabled: false + ipv6: + enabled: false + bridge: + port: [] +` + +// This is used to verify *both* the live and the target system in the `--add-nm-keyfile` path. +var verifyNmKeyfile = fmt.Sprintf(`[Unit] +Description=TestISO Verify NM Keyfile Propagation +OnFailure=emergency.target +OnFailureJobMode=isolate +Wants=network-online.target +After=network-online.target +Before=live-signal-ok.service +Before=coreos-test-installer.service +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=/usr/bin/journalctl -u nm-initrd --no-pager --grep "policy: set '%[1]s' (.*) as default .* routing and DNS" +ExecStart=/usr/bin/journalctl -u NetworkManager --no-pager --grep "policy: set '%[1]s' (.*) as default .* routing and DNS" +ExecStart=/usr/bin/grep "%[1]s" /etc/NetworkManager/system-connections/%[2]s +# Also verify nmstate config +ExecStart=/usr/bin/nmcli c show br-ex +[Install] +# for live system +RequiredBy=coreos-installer.target +# for target system +RequiredBy=multi-user.target`, nmConnectionId, nmConnectionFile) diff --git a/mantle/kola/tests/iso/live-as-disk.go b/mantle/kola/tests/iso/live-as-disk.go new file mode 100644 index 0000000000..7d9fe6f803 --- /dev/null +++ b/mantle/kola/tests/iso/live-as-disk.go @@ -0,0 +1,81 @@ +package iso + +import ( + "path/filepath" + "time" + + "github.com/coreos/coreos-assembler/mantle/kola" + "github.com/coreos/coreos-assembler/mantle/kola/cluster" + "github.com/coreos/coreos-assembler/mantle/kola/register" + "github.com/coreos/coreos-assembler/mantle/platform/conf" + "github.com/coreos/coreos-assembler/mantle/platform/machine/qemu" +) + +func init() { + // The iso-as-disk tests are only supported in x86_64 because other + // architectures don't have the required hybrid partition table. + var tests_as_disk_x86_64 = []string{ + "iso-as-disk.bios", + "iso-as-disk.uefi", + "iso-as-disk.uefi-secure", + "iso-as-disk.4k.uefi", + } + + for _, testName := range tests_as_disk_x86_64 { + register.RegisterTest(®ister.Test{ + Run: func(c cluster.TestCluster) { + opts := getIsoTestOpts(testName) + isoTestAsDisk(c, opts) + }, + ClusterSize: 0, + Name: "iso." + testName, + Description: "Verify ISO-as-disk install works.", + Timeout: installTimeoutMins * time.Minute, + Flags: []register.Flag{}, + Platforms: []string{"qemu"}, + }) + } +} + +func isoTestAsDisk(c cluster.TestCluster, opts IsoTestOpts) { + var outdir string + //var qc *qemu.Cluster + switch pc := c.Cluster.(type) { + case *qemu.Cluster: + outdir = pc.RuntimeConf().OutputDir + //qc = pc + default: + c.Fatalf("Unsupported cluster type") + } + + isopath := filepath.Join(kola.CosaBuild.Dir, kola.CosaBuild.Meta.BuildArtifacts.LiveIso.Path) + builder, config, err := newQemuBuilder(opts, outdir) + if err != nil { + c.Fatal(err) + } + defer builder.Close() + // Drop the bootindex bit (applicable to all arches except s390x and ppc64le); we want it to be the default + if err := builder.AddIso(isopath, "", true); err != nil { + c.Fatal(err) + } + + completionChannel, err := builder.VirtioChannelRead("testisocompletion") + if err != nil { + c.Fatal(err) + } + + config.AddSystemdUnit("live-signal-ok.service", liveSignalOKUnit, conf.Enable) + config.AddSystemdUnit("verify-no-efi-boot-entry.service", verifyNoEFIBootEntry, conf.Enable) + builder.SetConfig(config) + + mach, err := builder.Exec() + if err != nil { + c.Fatal(err) + } + defer mach.Destroy() + + err = awaitCompletion(c, mach, opts.console, outdir, completionChannel, nil, []string{liveOKSignal}) + if err != nil { + c.Fatal(err) + } +} diff --git a/mantle/kola/tests/iso/live-iscsi.go b/mantle/kola/tests/iso/live-iscsi.go new file mode 100644 index 0000000000..593692b123 --- /dev/null +++ b/mantle/kola/tests/iso/live-iscsi.go @@ -0,0 +1,243 @@ +package iso + +import ( + _ "embed" + "io" + "os" + "path/filepath" + "strings" + "time" + + "github.com/coreos/coreos-assembler/mantle/kola" + "github.com/coreos/coreos-assembler/mantle/kola/cluster" + "github.com/coreos/coreos-assembler/mantle/kola/register" + "github.com/coreos/coreos-assembler/mantle/platform" + "github.com/coreos/coreos-assembler/mantle/platform/conf" + "github.com/coreos/coreos-assembler/mantle/platform/machine/qemu" + coreosarch "github.com/coreos/stream-metadata-go/arch" +) + +var ( + tests_iscsi_x86_64 = []string{ + "iso-offline-install-iscsi.ibft.uefi", + "iso-offline-install-iscsi.ibft-with-mpath.bios", + "iso-offline-install-iscsi.manual.bios", + } + tests_iscsi_s390x = []string{ + // FIXME https://github.com/coreos/fedora-coreos-tracker/issues/1657 + //"iso-offline-install-iscsi.ibft.s390fw", + //"iso-offline-install-iscsi.ibft-with-mpath.s390fw", + //"iso-offline-install-iscsi.manual.s390fw", + } + tests_iscsi_ppc64le = []string{ + // FIXME https://github.com/coreos/fedora-coreos-tracker/issues/1657 + //"iso-offline-install-iscsi.ibft.ppcfw", + //"iso-offline-install-iscsi.ibft-with-mpath.ppcfw", + //"iso-offline-install-iscsi.manual.ppcfw", + } + tests_iscsi_aarch64 = []string{ + // FIXME https://github.com/coreos/fedora-coreos-tracker/issues/1657 + //"iso-offline-install-iscsi.ibft.uefi", + //"iso-offline-install-iscsi.ibft-with-mpath.uefi", + //"iso-offline-install-iscsi.manual.uefi", + } +) + +func getAllIscsiTests() []string { + arch := coreosarch.CurrentRpmArch() + switch arch { + case "x86_64": + return tests_iscsi_x86_64 + case "aarch64": + return tests_iscsi_aarch64 + case "ppc64le": + return tests_iscsi_ppc64le + case "s390x": + return tests_iscsi_s390x + default: + return []string{} + } +} + +func init() { + for _, testName := range getAllIscsiTests() { + register.RegisterTest(®ister.Test{ + Run: func(c cluster.TestCluster) { + opts := getIsoTestOpts(testName) + isoInstalliScsi(c, opts) + }, + ClusterSize: 0, + Name: "iso." + testName, + Description: "Verify iSCSI install works.", + Timeout: installTimeoutMins * time.Minute, + Flags: []register.Flag{}, + Platforms: []string{"qemu"}, + }) + } +} + +func isoOfflineInstallIscsiIbftUefi(c cluster.TestCluster) { + opts := IsoTestOpts{ + enableUefi: true, + isOffline: true, + enableIbft: true, + } + opts.SetInsecureOnDevBuild() + isoInstalliScsi(c, opts) +} + +func isoOfflineInstallIscsiIbftMpath(c cluster.TestCluster) { + opts := IsoTestOpts{ + enableUefi: true, + isOffline: true, + enableMultipath: true, + enableIbft: true, + } + opts.SetInsecureOnDevBuild() + isoInstalliScsi(c, opts) +} + +func isoOfflineInstallIscsiManual(c cluster.TestCluster) { + opts := IsoTestOpts{ + isOffline: true, + manual: true, + enableIbft: true, + } + opts.SetInsecureOnDevBuild() + isoInstalliScsi(c, opts) +} + +//go:embed iscsi_butane_setup.yaml +var iscsi_butane_config string + +// iscsi_butane_setup.yaml contains the full butane config but here is an overview of the setup +// 1 - Boot a live ISO with two extra 10G disks with labels "target" and "var" +// - Format and mount `virtio-var` to /var +// +// 2 - target.container -> start an iscsi target, using quay.io/coreos-assembler/targetcli +// 3 - setup-targetcli.service calls /usr/local/bin/targetcli_script: +// - instructs targetcli to serve /dev/disk/by-id/virtio-target as an iscsi target +// - disables authentication +// - verifies the iscsi service is active and reachable +// +// 4 - install-coreos-to-iscsi-target.service calls /usr/local/bin/install-coreos-iscsi: +// - mount iscsi target +// - run coreos-installer on the mounted block device +// - unmount iscsi +// +// 5 - coreos-iscsi-vm.container -> start a coreos-assembler conainer: +// - launch kola qemuexec instructing it to boot from an iPXE script +// wich in turns mount the iscsi target and load kernel +// - note the virtserial port device: we pass through the serial port +// that was created by kola for test completion +// +// 6 - /var/nested-ign.json contains an ignition config: +// - when the system is booted, write a success string to /dev/virtio-ports/testisocompletion +// - as this serial device is mapped to the host serial device, the test concludes +func isoInstalliScsi(c cluster.TestCluster, opts IsoTestOpts) { + var outdir string + //var qc *qemu.Cluster + switch pc := c.Cluster.(type) { + case *qemu.Cluster: + outdir = pc.RuntimeConf().OutputDir + //qc = pc + default: + c.Fatalf("Unsupported cluster type") + } + + var butane string + if opts.enableIbft && opts.enableMultipath { + butane = strings.ReplaceAll(iscsi_butane_config, "COREOS_INSTALLER_KARGS", "--append-karg rd.iscsi.firmware=1 --append-karg rd.multipath=default --append-karg root=/dev/disk/by-label/dm-mpath-root --append-karg rw") + } else if opts.enableIbft { + butane = strings.ReplaceAll(iscsi_butane_config, "COREOS_INSTALLER_KARGS", "--append-karg rd.iscsi.firmware=1") + } else if opts.manual { + butane = strings.ReplaceAll(iscsi_butane_config, "COREOS_INSTALLER_KARGS", "--append-karg netroot=iscsi:10.0.2.15::::iqn.2024-05.com.coreos:0") + } + + if kola.CosaBuild.Meta.BuildArtifacts.LiveIso == nil || kola.CosaBuild.Meta.BuildArtifacts.LiveKernel == nil { + c.Fatalf("build %s is missing live artifacts", kola.CosaBuild.Meta.Name) + } + builddir := kola.CosaBuild.Dir + isopath := filepath.Join(builddir, kola.CosaBuild.Meta.BuildArtifacts.LiveIso.Path) + builder, err := newBaseQemuBuilder(opts, outdir) + if err != nil { + c.Fatal(err) + } + defer builder.Close() + if err := builder.AddIso(isopath, "", false); err != nil { + c.Fatal(err) + } + + completionChannel, err := builder.VirtioChannelRead("testisocompletion") + if err != nil { + c.Fatal(err) + } + + // Create a serial channel to read the logs from the nested VM + nestedVmLogsChannel, err := builder.VirtioChannelRead("nestedvmlogs") + if err != nil { + c.Fatal(err) + } + + // Create a file to write the contents of the serial channel into + nestedVMConsole, err := os.OpenFile(filepath.Join(outdir, "nested_vm_console.txt"), os.O_WRONLY|os.O_CREATE, 0644) + if err != nil { + c.Fatal(err) + } + + go func() { + _, err := io.Copy(nestedVMConsole, nestedVmLogsChannel) + if err != nil && err != io.EOF { + panic(err) + } + }() + + // empty disk to use as an iscsi target to install coreOS on and subseqently boot + // Also add a 10G disk that we will mount on /var, to increase space available when pulling containers + err = builder.AddDisksFromSpecs([]string{"10G:serial=target", "10G:serial=var"}) + if err != nil { + c.Fatal(err) + } + + // We need more memory to start another VM within ! + builder.MemoryMiB = 2048 + + var iscsiTargetConfig = conf.Butane(butane) + + config, err := iscsiTargetConfig.Render(conf.FailWarnings) + if err != nil { + c.Fatal(err) + } + err = forwardJournal(outdir, builder, config) + if err != nil { + c.Fatal(err) + } + + // Add a failure target to stop the test if something go wrong rather than waiting for the 10min timeout + config.AddSystemdUnit("coreos-test-entered-emergency-target.service", signalFailureUnit, conf.Enable) + + // enable network + builder.EnableUsermodeNetworking([]platform.HostForwardPort{}, "") + + // keep auto-login enabled for easier debug when running console + config.AddAutoLogin() + + builder.SetConfig(config) + + // Bind mount in the COSA rootfs into the VM so we can use it as a + // read-only rootfs for quickly starting the container to kola + // qemuexec the nested VM for the test. See resources/iscsi_butane_setup.yaml + builder.MountHost("/", "/var/cosaroot", true) + config.MountHost("/var/cosaroot", true) + + mach, err := builder.Exec() + if err != nil { + c.Fatal(err) + } + defer mach.Destroy() + + err = awaitCompletion(c, mach, opts.console, outdir, completionChannel, nil, []string{"iscsi-boot-ok"}) + if err != nil { + c.Fatal(err) + } +} diff --git a/mantle/kola/tests/iso/live-iso.go b/mantle/kola/tests/iso/live-iso.go index 36466ee880..29a414a4f6 100644 --- a/mantle/kola/tests/iso/live-iso.go +++ b/mantle/kola/tests/iso/live-iso.go @@ -1,27 +1,19 @@ -package testiso +package iso import ( - "bufio" _ "embed" "fmt" - "io" "os" - "path/filepath" - "strconv" "strings" "time" - coreosarch "github.com/coreos/stream-metadata-go/arch" - "github.com/pkg/errors" - "github.com/coreos/coreos-assembler/mantle/kola" - "github.com/coreos/coreos-assembler/mantle/platform" - "github.com/coreos/coreos-assembler/mantle/platform/machine/qemu" - "github.com/coreos/coreos-assembler/mantle/util" - "github.com/coreos/coreos-assembler/mantle/kola/cluster" "github.com/coreos/coreos-assembler/mantle/kola/register" "github.com/coreos/coreos-assembler/mantle/platform/conf" + "github.com/coreos/coreos-assembler/mantle/platform/machine/qemu" + "github.com/coreos/coreos-assembler/mantle/util" + coreosarch "github.com/coreos/stream-metadata-go/arch" ) var ( @@ -62,58 +54,6 @@ var ( "miniso-install.nm.s390fw", "miniso-install.4k.nm.s390fw", } - tests_pxe_x86_64 = []string{ - "pxe-offline-install.rootfs-appended.bios", - "pxe-offline-install.4k.uefi", - "pxe-online-install.bios", - "pxe-online-install.4k.uefi", - } - tests_pxe_aarch64 = []string{ - "pxe-offline-install.uefi", - "pxe-offline-install.rootfs-appended.4k.uefi", - "pxe-online-install.uefi", - "pxe-online-install.4k.uefi", - } - tests_pxe_ppc64le = []string{ - "pxe-online-install.rootfs-appended.ppcfw", - "pxe-offline-install.4k.ppcfw", - } - tests_pxe_s390x = []string{ - "pxe-online-install.rootfs-appended.s390fw", - "pxe-offline-install.s390fw", - } - tests_iscsi_x86_64 = []string{ - "iso-offline-install-iscsi.ibft.uefi", - "iso-offline-install-iscsi.ibft-with-mpath.bios", - "iso-offline-install-iscsi.manual.bios", - } - tests_iscsi_s390x = []string{ - // FIXME https://github.com/coreos/fedora-coreos-tracker/issues/1657 - //"iso-offline-install-iscsi.ibft.s390fw", - //"iso-offline-install-iscsi.ibft-with-mpath.s390fw", - //"iso-offline-install-iscsi.manual.s390fw", - } - tests_iscsi_ppc64le = []string{ - // FIXME https://github.com/coreos/fedora-coreos-tracker/issues/1657 - //"iso-offline-install-iscsi.ibft.ppcfw", - //"iso-offline-install-iscsi.ibft-with-mpath.ppcfw", - //"iso-offline-install-iscsi.manual.ppcfw", - } - tests_iscsi_aarch64 = []string{ - // FIXME https://github.com/coreos/fedora-coreos-tracker/issues/1657 - //"iso-offline-install-iscsi.ibft.uefi", - //"iso-offline-install-iscsi.ibft-with-mpath.uefi", - //"iso-offline-install-iscsi.manual.uefi", - } - - // The iso-as-disk tests are only supported in x86_64 because other - // architectures don't have the required hybrid partition table. - tests_as_disk_x86_64 = []string{ - "iso-as-disk.bios", - "iso-as-disk.uefi", - "iso-as-disk.uefi-secure", - "iso-as-disk.4k.uefi", - } ) func getAllLiveIsoTests() []string { @@ -132,77 +72,6 @@ func getAllLiveIsoTests() []string { } } -func getAllPxeTests() []string { - arch := coreosarch.CurrentRpmArch() - switch arch { - case "x86_64": - return tests_pxe_x86_64 - case "aarch64": - return tests_pxe_aarch64 - case "ppc64le": - return tests_pxe_ppc64le - case "s390x": - return tests_pxe_s390x - default: - return []string{} - } -} - -func getAllIscsiTests() []string { - arch := coreosarch.CurrentRpmArch() - switch arch { - case "x86_64": - return tests_iscsi_x86_64 - case "aarch64": - return tests_iscsi_aarch64 - case "ppc64le": - return tests_iscsi_ppc64le - case "s390x": - return tests_iscsi_s390x - default: - return []string{} - } -} - -func getIsoTestOpts(testName string) IsoTestOpts { - opts := IsoTestOpts{} - - // Parse test name to determine options - if strings.Contains(testName, "4k") { - opts.enable4k = true - } - if strings.Contains(testName, "uefi") { - opts.enableUefi = true - } - if strings.Contains(testName, "mpath") { - opts.enableMultipath = true - } - if strings.Contains(testName, "offline") { - opts.isOffline = true - } - if strings.Contains(testName, "fromram") { - opts.isISOFromRAM = true - } - if strings.Contains(testName, "miniso") { - opts.isMiniso = true - } - if strings.Contains(testName, ".nm") { - opts.addNmKeyfile = true - } - if strings.Contains(testName, "rootfs-appended") { - opts.pxeAppendRootfs = true - } - if strings.Contains(testName, "ibft") { - opts.enableIbft = true - } - if strings.Contains(testName, "manual") { - opts.manual = true - } - - opts.SetInsecureOnDevBuild() - return opts -} - func init() { for _, testName := range getAllLiveIsoTests() { register.RegisterTest(®ister.Test{ @@ -213,381 +82,11 @@ func init() { ClusterSize: 0, Name: "iso." + testName, Description: "Verify ISO live install works.", - Timeout: 12 * time.Minute, + Timeout: installTimeoutMins * time.Minute, Flags: []register.Flag{}, Platforms: []string{"qemu"}, }) } - - // PXE tests - for _, testName := range getAllPxeTests() { - register.RegisterTest(®ister.Test{ - Run: func(c cluster.TestCluster) { - opts := getIsoTestOpts(testName) - testPXE(c, opts) - }, - ClusterSize: 0, - Name: "iso." + testName, - Description: "Verify PXE install works.", - Timeout: 12 * time.Minute, - Flags: []register.Flag{}, - Platforms: []string{"qemu"}, - }) - } - - // iSCSI tests - for _, testName := range getAllIscsiTests() { - register.RegisterTest(®ister.Test{ - Run: func(c cluster.TestCluster) { - opts := getIsoTestOpts(testName) - isoInstalliScsi(c, opts) - }, - ClusterSize: 0, - Name: "iso." + testName, - Description: "Verify iSCSI install works.", - Timeout: 12 * time.Minute, - Flags: []register.Flag{}, - Platforms: []string{"qemu"}, - }) - } - - // as-disk tests - for _, testName := range tests_as_disk_x86_64 { - register.RegisterTest(®ister.Test{ - Run: func(c cluster.TestCluster) { - opts := getIsoTestOpts(testName) - isoTestAsDisk(c, opts) - }, - ClusterSize: 0, - Name: "iso." + testName, - Description: "Verify ISO-as-disk install works.", - Timeout: 12 * time.Minute, - Flags: []register.Flag{}, - Platforms: []string{"qemu"}, - }) - } - -} - -var liveOKSignal = "live-test-OK" -var liveSignalOKUnit = fmt.Sprintf(` -[Unit] -Description=TestISO Signal Live ISO Completion -Requires=dev-virtio\\x2dports-testisocompletion.device -OnFailure=emergency.target -OnFailureJobMode=isolate -Before=coreos-installer.service -[Service] -Type=oneshot -RemainAfterExit=yes -ExecStart=/bin/sh -c '/usr/bin/echo %s >/dev/virtio-ports/testisocompletion' -[Install] -# for install tests -RequiredBy=coreos-installer.target -# for iso-as-disk -RequiredBy=multi-user.target`, liveOKSignal) - -var signalCompleteString = "coreos-installer-test-OK" -var signalCompletionUnit = fmt.Sprintf(` -[Unit] -Description=TestISO Signal Completion -Requires=dev-virtio\\x2dports-testisocompletion.device -OnFailure=emergency.target -OnFailureJobMode=isolate -[Service] -Type=oneshot -RemainAfterExit=yes -ExecStart=/bin/sh -c '/usr/bin/echo %s >/dev/virtio-ports/testisocompletion && systemctl poweroff' -[Install] -RequiredBy=multi-user.target`, signalCompleteString) - -var signalEmergencyString = "coreos-installer-test-entered-emergency-target" -var signalFailureUnit = fmt.Sprintf(` -[Unit] -Description=TestISO Signal Failure -Requires=dev-virtio\\x2dports-testisocompletion.device -DefaultDependencies=false -[Service] -Type=oneshot -RemainAfterExit=yes -ExecStart=/bin/sh -c '/usr/bin/echo %s >/dev/virtio-ports/testisocompletion && systemctl poweroff' -[Install] -RequiredBy=emergency.target`, signalEmergencyString) - -var multipathedRoot = `[Unit] -Description=TestISO Verify Multipathed Root -OnFailure=emergency.target -OnFailureJobMode=isolate -Before=coreos-test-installer.service -[Service] -Type=oneshot -RemainAfterExit=yes -ExecStart=/bin/bash -c 'lsblk -pno NAME "/dev/mapper/$(multipath -l -v 1)" | grep -qw "$(findmnt -nvr /sysroot -o SOURCE)"' -[Install] -RequiredBy=multi-user.target` - -var checkNoIgnition = ` -[Unit] -Description=TestISO Verify No Ignition Config -OnFailure=emergency.target -OnFailureJobMode=isolate -Before=coreos-test-installer.service -After=coreos-ignition-firstboot-complete.service -RequiresMountsFor=/boot -[Service] -Type=oneshot -RemainAfterExit=yes -ExecStart=/bin/sh -c '[ ! -e /boot/ignition ]' -[Install] -RequiredBy=multi-user.target` - -// This test is broken. Please fix! -// https://github.com/coreos/coreos-assembler/issues/3554 -var verifyNoEFIBootEntry = ` -[Unit] -Description=TestISO Verify No EFI Boot Entry -OnFailure=emergency.target -OnFailureJobMode=isolate -ConditionPathExists=/sys/firmware/efi -Before=live-signal-ok.service -[Service] -Type=oneshot -RemainAfterExit=yes -ExecStart=/bin/sh -c '! efibootmgr -v | grep -E "(HD|CDROM)\("' -[Install] -# for install tests -RequiredBy=coreos-installer.target -# for iso-as-disk -RequiredBy=multi-user.target` - -// Verify that the volume ID is the OS name. See also -// https://github.com/openshift/assisted-image-service/pull/477. -// This is the same as the LABEL of the block device for ISO9660. See -// https://github.com/util-linux/util-linux/blob/643bdae8e38055e36acf2963c3416de206081507/libblkid/src/superblocks/iso9660.c#L366-L377 -var verifyIsoVolumeId = ` -[Unit] -Description=Verify ISO Volume ID -OnFailure=emergency.target -OnFailureJobMode=isolate -# only if we're actually mounting the ISO -ConditionPathIsMountPoint=/run/media/iso -[Service] -Type=oneshot -RemainAfterExit=yes -# the backing device name is arch-dependent, but we know it's mounted on /run/media/iso -ExecStart=bash -c "[[ $(findmnt -no LABEL /run/media/iso) == %s-* ]]" -[Install] -RequiredBy=coreos-installer.target` - -// Unit to check that /run/media/iso is not mounted when -// coreos.liveiso.fromram kernel argument is passed -var isoNotMountedUnit = ` -[Unit] -Description=Verify ISO is not mounted when coreos.liveiso.fromram -OnFailure=emergency.target -OnFailureJobMode=isolate -ConditionKernelCommandLine=coreos.liveiso.fromram -[Service] -Type=oneshot -StandardOutput=kmsg+console -StandardError=kmsg+console -RemainAfterExit=yes -# Would like to use SuccessExitStatus but it doesn't support what -# we want: https://github.com/systemd/systemd/issues/10297#issuecomment-1672002635 -ExecStart=bash -c "if mountpoint /run/media/iso 2>/dev/null; then exit 1; fi" -[Install] -RequiredBy=coreos-installer.target` - -var nmConnectionId = "CoreOS DHCP" -var nmConnectionFile = "coreos-dhcp.nmconnection" -var nmConnection = fmt.Sprintf(`[connection] -id=%s -type=ethernet -# add wait-device-timeout here so we make sure NetworkManager-wait-online.service will -# wait for a device to be present before exiting. See -# https://github.com/coreos/fedora-coreos-tracker/issues/1275#issuecomment-1231605438 -wait-device-timeout=20000 - -[ipv4] -method=auto -`, nmConnectionId) - -var nmstateConfigFile = "/etc/nmstate/br-ex.yml" -var nmstateConfig = `interfaces: - - name: br-ex - type: linux-bridge - state: up - ipv4: - enabled: false - ipv6: - enabled: false - bridge: - port: [] -` - -// This is used to verify *both* the live and the target system in the `--add-nm-keyfile` path. -var verifyNmKeyfile = fmt.Sprintf(`[Unit] -Description=TestISO Verify NM Keyfile Propagation -OnFailure=emergency.target -OnFailureJobMode=isolate -Wants=network-online.target -After=network-online.target -Before=live-signal-ok.service -Before=coreos-test-installer.service -[Service] -Type=oneshot -RemainAfterExit=yes -ExecStart=/usr/bin/journalctl -u nm-initrd --no-pager --grep "policy: set '%[1]s' (.*) as default .* routing and DNS" -ExecStart=/usr/bin/journalctl -u NetworkManager --no-pager --grep "policy: set '%[1]s' (.*) as default .* routing and DNS" -ExecStart=/usr/bin/grep "%[1]s" /etc/NetworkManager/system-connections/%[2]s -# Also verify nmstate config -ExecStart=/usr/bin/nmcli c show br-ex -[Install] -# for live system -RequiredBy=coreos-installer.target -# for target system -RequiredBy=multi-user.target`, nmConnectionId, nmConnectionFile) - -type IsoTestOpts struct { - // Flags().BoolVarP(&instInsecure, "inst-insecure", "S", false, "Do not verify signature on metal image") - instInsecure bool - // Flags().StringSliceVar(&pxeKernelArgs, "pxe-kargs", nil, "Additional kernel arguments for PXE") - pxeKernelArgs []string - // Flags().BoolVar(&console, "console", false, "Connect qemu console to terminal, turn off automatic initramfs failure checking") - console bool - addNmKeyfile bool - enable4k bool - enableMultipath bool - isOffline bool - isISOFromRAM bool - isMiniso bool - enableUefi bool - enableUefiSecure bool - enableIbft bool - manual bool - pxeAppendRootfs bool -} - -func (o *IsoTestOpts) SetInsecureOnDevBuild() { - // Ignore signing verification by default when running with development build - // https://github.com/coreos/fedora-coreos-tracker/issues/908 - if strings.Contains(kola.CosaBuild.Meta.BuildID, ".dev.") { - o.instInsecure = true - //fmt.Printf("Detected development build; disabling signature verification\n") - } -} - -const ( - installTimeoutMins = 12 - // https://github.com/coreos/fedora-coreos-config/pull/2544 - liveISOFromRAMKarg = "coreos.liveiso.fromram" -) - -func newBaseQemuBuilder(opts IsoTestOpts, outdir string) (*platform.QemuBuilder, error) { - builder := qemu.NewMetalQemuBuilderDefault() - if opts.enableUefiSecure { - builder.Firmware = "uefi-secure" - } else if opts.enableUefi { - builder.Firmware = "uefi" - } - - if err := os.MkdirAll(outdir, 0755); err != nil { - return nil, err - } - - builder.InheritConsole = opts.console - if !opts.console { - builder.ConsoleFile = filepath.Join(outdir, "console.txt") - } - - if kola.QEMUOptions.Memory != "" { - parsedMem, err := strconv.ParseInt(kola.QEMUOptions.Memory, 10, 32) - if err != nil { - return nil, err - } - builder.MemoryMiB = int(parsedMem) - } - - // increase the memory for pxe tests with appended rootfs in the initrd - // we were bumping up into the 4GiB limit in RHCOS/c9s - // pxe-offline-install.rootfs-appended.bios tests - if opts.pxeAppendRootfs && builder.MemoryMiB < 5120 { - builder.MemoryMiB = 5120 - } - - return builder, nil -} - -func newQemuBuilder(opts IsoTestOpts, outdir string) (*platform.QemuBuilder, *conf.Conf, error) { - builder, err := newBaseQemuBuilder(opts, outdir) - if err != nil { - return nil, nil, err - } - - config, err := conf.EmptyIgnition().Render(conf.FailWarnings) - if err != nil { - return nil, nil, err - } - - err = forwardJournal(outdir, builder, config) - if err != nil { - return nil, nil, err - } - - return builder, config, nil -} - -func forwardJournal(outdir string, builder *platform.QemuBuilder, config *conf.Conf) error { - journalPipe, err := builder.VirtioJournal(config, "") - if err != nil { - return err - } - journalOut, err := os.OpenFile(filepath.Join(outdir, "journal.txt"), os.O_WRONLY|os.O_CREATE, 0644) - if err != nil { - return err - } - - go func() { - _, err := io.Copy(journalOut, journalPipe) - if err != nil && err != io.EOF { - panic(err) - } - }() - - return nil -} - -func newQemuBuilderWithDisk(opts IsoTestOpts, outdir string) (*platform.QemuBuilder, *conf.Conf, error) { - builder, config, err := newQemuBuilder(opts, outdir) - - if err != nil { - return nil, nil, err - } - - sectorSize := 0 - if opts.enable4k { - sectorSize = 4096 - } - - disk := platform.Disk{ - Size: "12G", // Arbitrary - SectorSize: sectorSize, - MultiPathDisk: opts.enableMultipath, - } - - //TBD: see if we can remove this and just use AddDisk and inject bootindex during startup - if coreosarch.CurrentRpmArch() == "s390x" || coreosarch.CurrentRpmArch() == "aarch64" { - // s390x and aarch64 need to use bootindex as they don't support boot once - if err := builder.AddDisk(&disk); err != nil { - return nil, nil, err - } - } else { - if err := builder.AddPrimaryDisk(&disk); err != nil { - return nil, nil, err - } - } - - return builder, config, nil } func isoLiveIso(c cluster.TestCluster, opts IsoTestOpts) { @@ -679,397 +178,3 @@ func isoLiveIso(c cluster.TestCluster, opts IsoTestOpts) { c.Fatal(err) } } - -var downloadCheck = `[Unit] -Description=TestISO Verify CoreOS Installer Download -After=coreos-installer.service -Before=coreos-installer.target -[Service] -Type=oneshot -StandardOutput=kmsg+console -StandardError=kmsg+console -ExecStart=/bin/sh -c "journalctl -t coreos-installer-service | /usr/bin/awk '/[Dd]ownload/ {exit 1}'" -ExecStart=/bin/sh -c "/usr/bin/udevadm settle" -ExecStart=/bin/sh -c "/usr/bin/mount /dev/disk/by-label/root /mnt" -ExecStart=/bin/sh -c "/usr/bin/jq -er '.[\"build\"]? + .[\"version\"]? == \"%s\"' /mnt/.coreos-aleph-version.json" -[Install] -RequiredBy=coreos-installer.target -` - -func testPXE(c cluster.TestCluster, opts IsoTestOpts) { - var outdir string - var qc *qemu.Cluster - - switch pc := c.Cluster.(type) { - case *qemu.Cluster: - outdir = pc.RuntimeConf().OutputDir - qc = pc - default: - c.Fatalf("Unsupported cluster type") - } - - if opts.addNmKeyfile { - c.Fatal("--add-nm-keyfile not yet supported for PXE") - } - - inst := qemu.Install{ - CosaBuild: kola.CosaBuild, - NmKeyfiles: make(map[string]string), - Insecure: opts.instInsecure, - Native4k: opts.enable4k, - MultiPathDisk: opts.enableMultipath, - PxeAppendRootfs: opts.pxeAppendRootfs, - } - - tmpd, err := os.MkdirTemp("", "kola-iso.pxe") - if err != nil { - c.Fatal(err) - } - defer os.RemoveAll(tmpd) - - sshPubKeyBuf, _, err := util.CreateSSHAuthorizedKey(tmpd) - if err != nil { - c.Fatal(err) - } - - builder, virtioJournalConfig, err := newQemuBuilderWithDisk(opts, outdir) - if err != nil { - c.Fatal(err) - } - - // increase the memory for pxe tests with appended rootfs in the initrd - // we were bumping up into the 4GiB limit in RHCOS/c9s - // pxe-offline-install.rootfs-appended.bios tests - if inst.PxeAppendRootfs && builder.MemoryMiB < 5120 { - builder.MemoryMiB = 5120 - } - - inst.Builder = builder - completionChannel, err := inst.Builder.VirtioChannelRead("testisocompletion") - if err != nil { - c.Fatal(err) // , "setting up virtio-serial channel") - } - - var keys []string - keys = append(keys, strings.TrimSpace(string(sshPubKeyBuf))) - virtioJournalConfig.AddAuthorizedKeys("core", keys) - - liveConfig := *virtioJournalConfig - liveConfig.AddSystemdUnit("live-signal-ok.service", liveSignalOKUnit, conf.Enable) - liveConfig.AddSystemdUnit("coreos-test-entered-emergency-target.service", signalFailureUnit, conf.Enable) - - if opts.isOffline { - contents := fmt.Sprintf(downloadCheck, kola.CosaBuild.Meta.OstreeVersion) - liveConfig.AddSystemdUnit("coreos-installer-offline-check.service", contents, conf.Enable) - } - - targetConfig := *virtioJournalConfig - targetConfig.AddSystemdUnit("coreos-test-installer.service", signalCompletionUnit, conf.Enable) - targetConfig.AddSystemdUnit("coreos-test-entered-emergency-target.service", signalFailureUnit, conf.Enable) - targetConfig.AddSystemdUnit("coreos-test-installer-no-ignition.service", checkNoIgnition, conf.Enable) - - mach, err := inst.PXE(opts.pxeKernelArgs, liveConfig, targetConfig, opts.isOffline) - if err != nil { - c.Fatal(err) - } - qc.AddMach(mach) - - err = awaitCompletion(c, mach.Instance(), opts.console, outdir, completionChannel, mach.BootStartedErrorChannel(), []string{liveOKSignal, signalCompleteString}) - if err != nil { - c.Fatal(err) - } -} - -//go:embed iscsi_butane_setup.yaml -var iscsi_butane_config string - -// iscsi_butane_setup.yaml contains the full butane config but here is an overview of the setup -// 1 - Boot a live ISO with two extra 10G disks with labels "target" and "var" -// - Format and mount `virtio-var` to /var -// -// 2 - target.container -> start an iscsi target, using quay.io/coreos-assembler/targetcli -// 3 - setup-targetcli.service calls /usr/local/bin/targetcli_script: -// - instructs targetcli to serve /dev/disk/by-id/virtio-target as an iscsi target -// - disables authentication -// - verifies the iscsi service is active and reachable -// -// 4 - install-coreos-to-iscsi-target.service calls /usr/local/bin/install-coreos-iscsi: -// - mount iscsi target -// - run coreos-installer on the mounted block device -// - unmount iscsi -// -// 5 - coreos-iscsi-vm.container -> start a coreos-assembler conainer: -// - launch kola qemuexec instructing it to boot from an iPXE script -// wich in turns mount the iscsi target and load kernel -// - note the virtserial port device: we pass through the serial port -// that was created by kola for test completion -// -// 6 - /var/nested-ign.json contains an ignition config: -// - when the system is booted, write a success string to /dev/virtio-ports/testisocompletion -// - as this serial device is mapped to the host serial device, the test concludes -func isoInstalliScsi(c cluster.TestCluster, opts IsoTestOpts) { - var outdir string - //var qc *qemu.Cluster - switch pc := c.Cluster.(type) { - case *qemu.Cluster: - outdir = pc.RuntimeConf().OutputDir - //qc = pc - default: - c.Fatalf("Unsupported cluster type") - } - - var butane string - if opts.enableIbft && opts.enableMultipath { - butane = strings.ReplaceAll(iscsi_butane_config, "COREOS_INSTALLER_KARGS", "--append-karg rd.iscsi.firmware=1 --append-karg rd.multipath=default --append-karg root=/dev/disk/by-label/dm-mpath-root --append-karg rw") - } else if opts.enableIbft { - butane = strings.ReplaceAll(iscsi_butane_config, "COREOS_INSTALLER_KARGS", "--append-karg rd.iscsi.firmware=1") - } else if opts.manual { - butane = strings.ReplaceAll(iscsi_butane_config, "COREOS_INSTALLER_KARGS", "--append-karg netroot=iscsi:10.0.2.15::::iqn.2024-05.com.coreos:0") - } - - if kola.CosaBuild.Meta.BuildArtifacts.LiveIso == nil || kola.CosaBuild.Meta.BuildArtifacts.LiveKernel == nil { - c.Fatalf("build %s is missing live artifacts", kola.CosaBuild.Meta.Name) - } - builddir := kola.CosaBuild.Dir - isopath := filepath.Join(builddir, kola.CosaBuild.Meta.BuildArtifacts.LiveIso.Path) - builder, err := newBaseQemuBuilder(opts, outdir) - if err != nil { - c.Fatal(err) - } - defer builder.Close() - if err := builder.AddIso(isopath, "", false); err != nil { - c.Fatal(err) - } - - completionChannel, err := builder.VirtioChannelRead("testisocompletion") - if err != nil { - c.Fatal(err) - } - - // Create a serial channel to read the logs from the nested VM - nestedVmLogsChannel, err := builder.VirtioChannelRead("nestedvmlogs") - if err != nil { - c.Fatal(err) - } - - // Create a file to write the contents of the serial channel into - nestedVMConsole, err := os.OpenFile(filepath.Join(outdir, "nested_vm_console.txt"), os.O_WRONLY|os.O_CREATE, 0644) - if err != nil { - c.Fatal(err) - } - - go func() { - _, err := io.Copy(nestedVMConsole, nestedVmLogsChannel) - if err != nil && err != io.EOF { - panic(err) - } - }() - - // empty disk to use as an iscsi target to install coreOS on and subseqently boot - // Also add a 10G disk that we will mount on /var, to increase space available when pulling containers - err = builder.AddDisksFromSpecs([]string{"10G:serial=target", "10G:serial=var"}) - if err != nil { - c.Fatal(err) - } - - // We need more memory to start another VM within ! - builder.MemoryMiB = 2048 - - var iscsiTargetConfig = conf.Butane(butane) - - config, err := iscsiTargetConfig.Render(conf.FailWarnings) - if err != nil { - c.Fatal(err) - } - err = forwardJournal(outdir, builder, config) - if err != nil { - c.Fatal(err) - } - - // Add a failure target to stop the test if something go wrong rather than waiting for the 10min timeout - config.AddSystemdUnit("coreos-test-entered-emergency-target.service", signalFailureUnit, conf.Enable) - - // enable network - builder.EnableUsermodeNetworking([]platform.HostForwardPort{}, "") - - // keep auto-login enabled for easier debug when running console - config.AddAutoLogin() - - builder.SetConfig(config) - - // Bind mount in the COSA rootfs into the VM so we can use it as a - // read-only rootfs for quickly starting the container to kola - // qemuexec the nested VM for the test. See resources/iscsi_butane_setup.yaml - builder.MountHost("/", "/var/cosaroot", true) - config.MountHost("/var/cosaroot", true) - - mach, err := builder.Exec() - if err != nil { - c.Fatal(err) - } - defer mach.Destroy() - - err = awaitCompletion(c, mach, opts.console, outdir, completionChannel, nil, []string{"iscsi-boot-ok"}) - if err != nil { - c.Fatal(err) - } -} - -func awaitCompletion(c cluster.TestCluster, inst *platform.QemuInstance, console bool, outdir string, qchan *os.File, booterrchan chan error, expected []string) error { - ctx := c.Context() - - errchan := make(chan error) - go func() { - timeout := (time.Duration(installTimeoutMins*(100+kola.Options.ExtendTimeoutPercent)) * time.Minute) / 100 - time.Sleep(timeout) - errchan <- fmt.Errorf("timed out after %v", timeout) - }() - if !console { - go func() { - errBuf, err := inst.WaitIgnitionError(ctx) - if err == nil { - if errBuf != "" { - c.Logf("entered emergency.target in initramfs") - path := filepath.Join(outdir, "ignition-virtio-dump.txt") - if err := os.WriteFile(path, []byte(errBuf), 0644); err != nil { - c.Errorf("Failed to write journal: %v", err) - } - err = platform.ErrInitramfsEmergency - } - } - if err != nil { - errchan <- err - } - }() - } - go func() { - err := inst.Wait() - // only one Wait() gets process data, so also manually check for signal - //plog.Debugf("qemu exited err=%v", err) - if err == nil && inst.Signaled() { - err = errors.New("process killed") - } - if err != nil { - errchan <- errors.Wrapf(err, "QEMU unexpectedly exited while awaiting completion") - } - time.Sleep(1 * time.Minute) - errchan <- fmt.Errorf("QEMU exited; timed out waiting for completion") - }() - go func() { - r := bufio.NewReader(qchan) - for _, exp := range expected { - l, err := r.ReadString('\n') - if err != nil { - if err == io.EOF { - // this may be from QEMU getting killed or exiting; wait a bit - // to give a chance for .Wait() above to feed the channel with a - // better error - time.Sleep(1 * time.Second) - errchan <- fmt.Errorf("Got EOF from completion channel, %s expected", exp) - } else { - errchan <- errors.Wrapf(err, "reading from completion channel") - } - return - } - line := strings.TrimSpace(l) - if line != exp { - errchan <- fmt.Errorf("Unexpected string from completion channel: %s expected: %s", line, exp) - return - } - } - // OK! - errchan <- nil - }() - go func() { - //check for error when switching boot order - if booterrchan != nil { - if err := <-booterrchan; err != nil { - errchan <- err - } - } - }() - err := <-errchan - if err == nil { - // No error so far, check the console and journal files - consoleFile := filepath.Join(outdir, "console.txt") - journalFile := filepath.Join(outdir, "journal.txt") - files := []string{consoleFile, journalFile} - for _, file := range files { - fileName := filepath.Base(file) - // Check if the file exists - _, err := os.Stat(file) - if os.IsNotExist(err) { - fmt.Printf("The file: %v does not exist\n", fileName) - continue - } else if err != nil { - fmt.Println(err) - return err - } - // Read the contents of the file - fileContent, err := os.ReadFile(file) - if err != nil { - fmt.Println(err) - return err - } - // Check for badness with CheckConsole - warnOnly, badlines := kola.CheckConsole([]byte(fileContent), nil) - if len(badlines) > 0 { - for _, badline := range badlines { - if warnOnly { - c.Errorf("bad log line detected: %v", badline) - } else { - c.Logf("bad log line detected: %v", badline) - } - } - if !warnOnly { - err = fmt.Errorf("errors found in log files") - return err - } - } - } - } - return err -} - -func isoTestAsDisk(c cluster.TestCluster, opts IsoTestOpts) { - var outdir string - //var qc *qemu.Cluster - switch pc := c.Cluster.(type) { - case *qemu.Cluster: - outdir = pc.RuntimeConf().OutputDir - //qc = pc - default: - c.Fatalf("Unsupported cluster type") - } - - isopath := filepath.Join(kola.CosaBuild.Dir, kola.CosaBuild.Meta.BuildArtifacts.LiveIso.Path) - builder, config, err := newQemuBuilder(opts, outdir) - if err != nil { - c.Fatal(err) - } - defer builder.Close() - // Drop the bootindex bit (applicable to all arches except s390x and ppc64le); we want it to be the default - if err := builder.AddIso(isopath, "", true); err != nil { - c.Fatal(err) - } - - completionChannel, err := builder.VirtioChannelRead("testisocompletion") - if err != nil { - c.Fatal(err) - } - - config.AddSystemdUnit("live-signal-ok.service", liveSignalOKUnit, conf.Enable) - config.AddSystemdUnit("verify-no-efi-boot-entry.service", verifyNoEFIBootEntry, conf.Enable) - builder.SetConfig(config) - - mach, err := builder.Exec() - if err != nil { - c.Fatal(err) - } - defer mach.Destroy() - - err = awaitCompletion(c, mach, opts.console, outdir, completionChannel, nil, []string{liveOKSignal}) - if err != nil { - c.Fatal(err) - } -} diff --git a/mantle/kola/tests/iso/live-login.go b/mantle/kola/tests/iso/live-login.go index 1d41baef1a..52a928641e 100644 --- a/mantle/kola/tests/iso/live-login.go +++ b/mantle/kola/tests/iso/live-login.go @@ -1,4 +1,4 @@ -package testiso +package iso import ( "bufio" diff --git a/mantle/kola/tests/iso/live-pxe.go b/mantle/kola/tests/iso/live-pxe.go new file mode 100644 index 0000000000..4d75ba85e2 --- /dev/null +++ b/mantle/kola/tests/iso/live-pxe.go @@ -0,0 +1,172 @@ +package iso + +import ( + "fmt" + "os" + "strings" + "time" + + "github.com/coreos/coreos-assembler/mantle/kola" + "github.com/coreos/coreos-assembler/mantle/kola/cluster" + "github.com/coreos/coreos-assembler/mantle/kola/register" + "github.com/coreos/coreos-assembler/mantle/platform/conf" + "github.com/coreos/coreos-assembler/mantle/platform/machine/qemu" + "github.com/coreos/coreos-assembler/mantle/util" + coreosarch "github.com/coreos/stream-metadata-go/arch" +) + +var ( + tests_pxe_x86_64 = []string{ + "pxe-offline-install.rootfs-appended.bios", + "pxe-offline-install.4k.uefi", + "pxe-online-install.bios", + "pxe-online-install.4k.uefi", + } + tests_pxe_aarch64 = []string{ + "pxe-offline-install.uefi", + "pxe-offline-install.rootfs-appended.4k.uefi", + "pxe-online-install.uefi", + "pxe-online-install.4k.uefi", + } + tests_pxe_ppc64le = []string{ + "pxe-online-install.rootfs-appended.ppcfw", + "pxe-offline-install.4k.ppcfw", + } + tests_pxe_s390x = []string{ + "pxe-online-install.rootfs-appended.s390fw", + "pxe-offline-install.s390fw", + } +) + +func getAllPxeTests() []string { + arch := coreosarch.CurrentRpmArch() + switch arch { + case "x86_64": + return tests_pxe_x86_64 + case "aarch64": + return tests_pxe_aarch64 + case "ppc64le": + return tests_pxe_ppc64le + case "s390x": + return tests_pxe_s390x + default: + return []string{} + } +} + +func init() { + for _, testName := range getAllPxeTests() { + register.RegisterTest(®ister.Test{ + Run: func(c cluster.TestCluster) { + opts := getIsoTestOpts(testName) + testPXE(c, opts) + }, + ClusterSize: 0, + Name: "iso." + testName, + Description: "Verify PXE install works.", + Timeout: installTimeoutMins * time.Minute, + Flags: []register.Flag{}, + Platforms: []string{"qemu"}, + }) + } +} + +var downloadCheck = `[Unit] +Description=TestISO Verify CoreOS Installer Download +After=coreos-installer.service +Before=coreos-installer.target +[Service] +Type=oneshot +StandardOutput=kmsg+console +StandardError=kmsg+console +ExecStart=/bin/sh -c "journalctl -t coreos-installer-service | /usr/bin/awk '/[Dd]ownload/ {exit 1}'" +ExecStart=/bin/sh -c "/usr/bin/udevadm settle" +ExecStart=/bin/sh -c "/usr/bin/mount /dev/disk/by-label/root /mnt" +ExecStart=/bin/sh -c "/usr/bin/jq -er '.[\"build\"]? + .[\"version\"]? == \"%s\"' /mnt/.coreos-aleph-version.json" +[Install] +RequiredBy=coreos-installer.target +` + +func testPXE(c cluster.TestCluster, opts IsoTestOpts) { + var outdir string + var qc *qemu.Cluster + + switch pc := c.Cluster.(type) { + case *qemu.Cluster: + outdir = pc.RuntimeConf().OutputDir + qc = pc + default: + c.Fatalf("Unsupported cluster type") + } + + if opts.addNmKeyfile { + c.Fatal("--add-nm-keyfile not yet supported for PXE") + } + + inst := qemu.Install{ + CosaBuild: kola.CosaBuild, + NmKeyfiles: make(map[string]string), + Insecure: opts.instInsecure, + Native4k: opts.enable4k, + MultiPathDisk: opts.enableMultipath, + PxeAppendRootfs: opts.pxeAppendRootfs, + } + + tmpd, err := os.MkdirTemp("", "kola-iso.pxe") + if err != nil { + c.Fatal(err) + } + defer os.RemoveAll(tmpd) + + sshPubKeyBuf, _, err := util.CreateSSHAuthorizedKey(tmpd) + if err != nil { + c.Fatal(err) + } + + builder, virtioJournalConfig, err := newQemuBuilderWithDisk(opts, outdir) + if err != nil { + c.Fatal(err) + } + + // increase the memory for pxe tests with appended rootfs in the initrd + // we were bumping up into the 4GiB limit in RHCOS/c9s + // pxe-offline-install.rootfs-appended.bios tests + if inst.PxeAppendRootfs && builder.MemoryMiB < 5120 { + builder.MemoryMiB = 5120 + } + + inst.Builder = builder + completionChannel, err := inst.Builder.VirtioChannelRead("testisocompletion") + if err != nil { + c.Fatal(err) // , "setting up virtio-serial channel") + } + + var keys []string + keys = append(keys, strings.TrimSpace(string(sshPubKeyBuf))) + virtioJournalConfig.AddAuthorizedKeys("core", keys) + + liveConfig := *virtioJournalConfig + liveConfig.AddSystemdUnit("live-signal-ok.service", liveSignalOKUnit, conf.Enable) + liveConfig.AddSystemdUnit("coreos-test-entered-emergency-target.service", signalFailureUnit, conf.Enable) + + if opts.isOffline { + contents := fmt.Sprintf(downloadCheck, kola.CosaBuild.Meta.OstreeVersion) + liveConfig.AddSystemdUnit("coreos-installer-offline-check.service", contents, conf.Enable) + } + + targetConfig := *virtioJournalConfig + targetConfig.AddSystemdUnit("coreos-test-installer.service", signalCompletionUnit, conf.Enable) + targetConfig.AddSystemdUnit("coreos-test-entered-emergency-target.service", signalFailureUnit, conf.Enable) + targetConfig.AddSystemdUnit("coreos-test-installer-no-ignition.service", checkNoIgnition, conf.Enable) + + mach, err := inst.PXE(opts.pxeKernelArgs, liveConfig, targetConfig, opts.isOffline) + if err != nil { + c.Fatal(err) + } + qc.AddMach(mach) + + err = awaitCompletion(c, mach.Instance(), opts.console, outdir, completionChannel, mach.BootStartedErrorChannel(), []string{liveOKSignal, signalCompleteString}) + if err != nil { + c.Fatal(err) + } +} From c9a7fb412adb4b6729a46302df592573ae08a253 Mon Sep 17 00:00:00 2001 From: Nikita Dubrovskii Date: Thu, 2 Apr 2026 12:42:42 +0200 Subject: [PATCH 10/31] kola/tests/iso: Dump journal on emergency.target in ISO tests Add journal dumping to signalFailureUnit to capture diagnostic information when the system enters emergency.target during ISO tests. After switch root occurs during live ISO testing, coreos-installer runs from the real root filesystem rather than the initramfs, which means ignition-virtio-dump-journal.service is no longer enabled. This change ensures that when emergency.target is reached, the journal is still dumped to the virtio port, allowing platform.StartMachine to capture errors. The unit now checks if ignition-virtio-dump-journal.service is enabled, and if not, manually executes the journal dump script from the dracut emergency shell setup module. --- mantle/kola/tests/iso/common.go | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/mantle/kola/tests/iso/common.go b/mantle/kola/tests/iso/common.go index 5d2fc0bca6..78405698a2 100644 --- a/mantle/kola/tests/iso/common.go +++ b/mantle/kola/tests/iso/common.go @@ -350,6 +350,11 @@ ExecStart=/bin/sh -c '/usr/bin/echo %s >/dev/virtio-ports/testisocompletion && s [Install] RequiredBy=multi-user.target`, signalCompleteString) +// signalFailureUnit also ensures that the journal is dumped to the virtio port if the system +// enters emergency.target. This is needed when running from the live ISO and coreos-installer +// fails, because ignition-virtio-dump-journal.service is no longer enabled in that context: +// after switch root occurs, coreos-installer runs from the real root filesystem rather than +// the initramfs. Using this unit guarantees we can still catch errors in platform.StartMachine. var signalEmergencyString = "coreos-installer-test-entered-emergency-target" var signalFailureUnit = fmt.Sprintf(` [Unit] @@ -359,7 +364,11 @@ DefaultDependencies=false [Service] Type=oneshot RemainAfterExit=yes -ExecStart=/bin/sh -c '/usr/bin/echo %s >/dev/virtio-ports/testisocompletion && systemctl poweroff' +ExecStart=/bin/sh -c '/usr/bin/echo %s >/dev/virtio-ports/testisocompletion' +ExecStart=/bin/bash -c '\ +if ! systemctl is-enabled ignition-virtio-dump-journal.service >/dev/null 2>&1; then \ + exec /usr/lib/dracut/modules.d/99emergency-shell-setup/ignition-virtio-dump-journal.sh; \ +fi' [Install] RequiredBy=emergency.target`, signalEmergencyString) From c50d43fde9a6f32fbb087f4e92bedacb73e9faea Mon Sep 17 00:00:00 2001 From: Nikita Dubrovskii Date: Fri, 21 Nov 2025 12:28:56 +0100 Subject: [PATCH 11/31] mantle/cmd/kola: move inst-insecure and pxe-kargs to global QEMUOptions Move the 'inst-insecure' and 'pxe-kargs' parameters from local test options to global QEMUOptions structure, making them available as command-line flags for all ISO tests: - --inst-insecure: Skip signature verification on metal image - --pxe-kargs: Additional kernel arguments for PXE boot tests --- mantle/cmd/kola/options.go | 4 ++++ mantle/kola/tests/iso/common.go | 13 ++++++------- mantle/kola/tests/iso/live-pxe.go | 2 +- mantle/platform/machine/qemu/flight.go | 5 +++++ 4 files changed, 16 insertions(+), 8 deletions(-) diff --git a/mantle/cmd/kola/options.go b/mantle/cmd/kola/options.go index 146005c576..b90570452d 100644 --- a/mantle/cmd/kola/options.go +++ b/mantle/cmd/kola/options.go @@ -162,6 +162,10 @@ func init() { sv(&kola.QEMUOptions.SecureExecutionHostKey, "qemu-secex-hostkey", "", "Path to Secure Execution HKD certificate") // s390x CEX-specific options bv(&kola.QEMUOptions.Cex, "qemu-cex", false, "Attach CEX device to guest") + + // kola run iso.* options + bv(&kola.QEMUOptions.InstInsecure, "inst-insecure", false, "Do not verify signature on metal image") + ssv(&kola.QEMUOptions.PxeKernelArgs, "pxe-kargs", nil, "Additional kernel arguments for PXE") } // Sync up the command line options if there is dependency diff --git a/mantle/kola/tests/iso/common.go b/mantle/kola/tests/iso/common.go index 78405698a2..f090e5ceed 100644 --- a/mantle/kola/tests/iso/common.go +++ b/mantle/kola/tests/iso/common.go @@ -28,8 +28,6 @@ const ( type IsoTestOpts struct { // Flags().BoolVarP(&instInsecure, "inst-insecure", "S", false, "Do not verify signature on metal image") instInsecure bool - // Flags().StringSliceVar(&pxeKernelArgs, "pxe-kargs", nil, "Additional kernel arguments for PXE") - pxeKernelArgs []string // Flags().BoolVar(&console, "console", false, "Connect qemu console to terminal, turn off automatic initramfs failure checking") console bool addNmKeyfile bool @@ -82,17 +80,18 @@ func getIsoTestOpts(testName string) IsoTestOpts { opts.manual = true } - opts.SetInsecureOnDevBuild() + opts.instInsecure = kola.QEMUOptions.InstInsecure || IsDevBuild() return opts } -func (o *IsoTestOpts) SetInsecureOnDevBuild() { +func IsDevBuild() bool { // Ignore signing verification by default when running with development build // https://github.com/coreos/fedora-coreos-tracker/issues/908 - if strings.Contains(kola.CosaBuild.Meta.BuildID, ".dev.") { - o.instInsecure = true - //fmt.Printf("Detected development build; disabling signature verification\n") + if kola.CosaBuild != nil && strings.Contains(kola.CosaBuild.Meta.BuildID, ".dev.") { + fmt.Printf("Detected development build; disabling signature verification\n") + return true } + return false } func newBaseQemuBuilder(opts IsoTestOpts, outdir string) (*platform.QemuBuilder, error) { diff --git a/mantle/kola/tests/iso/live-pxe.go b/mantle/kola/tests/iso/live-pxe.go index 4d75ba85e2..086e0df735 100644 --- a/mantle/kola/tests/iso/live-pxe.go +++ b/mantle/kola/tests/iso/live-pxe.go @@ -159,7 +159,7 @@ func testPXE(c cluster.TestCluster, opts IsoTestOpts) { targetConfig.AddSystemdUnit("coreos-test-entered-emergency-target.service", signalFailureUnit, conf.Enable) targetConfig.AddSystemdUnit("coreos-test-installer-no-ignition.service", checkNoIgnition, conf.Enable) - mach, err := inst.PXE(opts.pxeKernelArgs, liveConfig, targetConfig, opts.isOffline) + mach, err := inst.PXE(kola.QEMUOptions.PxeKernelArgs, liveConfig, targetConfig, opts.isOffline) if err != nil { c.Fatal(err) } diff --git a/mantle/platform/machine/qemu/flight.go b/mantle/platform/machine/qemu/flight.go index 4a008b1a0a..bc3bd90a54 100644 --- a/mantle/platform/machine/qemu/flight.go +++ b/mantle/platform/machine/qemu/flight.go @@ -55,6 +55,11 @@ type Options struct { SecureExecutionIgnitionPubKey string SecureExecutionHostKey string + // kola run iso.* options + // Do not verify signature on metal image + InstInsecure bool + PxeKernelArgs []string + // Option to create IBM cex based luks encryption Cex bool From 6d4d3947239e5a3fdbfb8bb816fdf5b53632b420 Mon Sep 17 00:00:00 2001 From: Nikita Dubrovskii Date: Fri, 21 Nov 2025 12:40:10 +0100 Subject: [PATCH 12/31] kola/tests/iso: fold testiso iso-fips.uefi test Migrate the last remaining test from the legacy 'kola testiso' command into the standard 'kola run' This completes the migration of all ISO tests and removes the now-obsolete testiso.go file. --- mantle/cmd/kola/testiso.go | 496 ----------------------------- mantle/kola/tests/iso/live-fips.go | 99 ++++++ 2 files changed, 99 insertions(+), 496 deletions(-) delete mode 100644 mantle/cmd/kola/testiso.go create mode 100644 mantle/kola/tests/iso/live-fips.go diff --git a/mantle/cmd/kola/testiso.go b/mantle/cmd/kola/testiso.go deleted file mode 100644 index 6f78e2da59..0000000000 --- a/mantle/cmd/kola/testiso.go +++ /dev/null @@ -1,496 +0,0 @@ -// Copyright 2020 Red Hat, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// TODO: -// - Support testing the "just run Live" case - maybe try to figure out -// how to have main `kola` tests apply? -// - Test `coreos-install iso embed` path - -package main - -import ( - "bufio" - "context" - "fmt" - "io" - "os" - "path/filepath" - "strconv" - "strings" - "time" - - "github.com/coreos/coreos-assembler/mantle/harness" - "github.com/coreos/coreos-assembler/mantle/harness/reporters" - "github.com/coreos/coreos-assembler/mantle/harness/testresult" - "github.com/coreos/coreos-assembler/mantle/platform/conf" - "github.com/coreos/coreos-assembler/mantle/platform/machine/qemu" - coreosarch "github.com/coreos/stream-metadata-go/arch" - "github.com/pkg/errors" - - "github.com/spf13/cobra" - - "github.com/coreos/coreos-assembler/mantle/kola" - "github.com/coreos/coreos-assembler/mantle/platform" -) - -var ( - cmdTestIso = &cobra.Command{ - RunE: runTestIso, - PreRunE: preRun, - Use: "testiso [glob pattern...]", - Short: "Test a CoreOS PXE boot or ISO install path", - - SilenceUsage: true, - } - - instInsecure bool - pxeKernelArgs []string - console bool - enableUefi bool - // These tests only run on RHCOS - tests_RHCOS_uefi = []string{ - "iso-fips.uefi", - } -) - -const ( - installTimeoutMins = 12 -) - -var liveOKSignal = "live-test-OK" -var liveSignalOKUnit = fmt.Sprintf(`[Unit] -Description=TestISO Signal Live ISO Completion -Requires=dev-virtio\\x2dports-testisocompletion.device -OnFailure=emergency.target -OnFailureJobMode=isolate -Before=coreos-installer.service -[Service] -Type=oneshot -RemainAfterExit=yes -ExecStart=/bin/sh -c '/usr/bin/echo %s >/dev/virtio-ports/testisocompletion' -[Install] -# for install tests -RequiredBy=coreos-installer.target -# for iso-as-disk -RequiredBy=multi-user.target -`, liveOKSignal) - -var signalEmergencyString = "coreos-installer-test-entered-emergency-target" -var signalFailureUnit = fmt.Sprintf(`[Unit] -Description=TestISO Signal Failure -Requires=dev-virtio\\x2dports-testisocompletion.device -DefaultDependencies=false -[Service] -Type=oneshot -RemainAfterExit=yes -ExecStart=/bin/sh -c '/usr/bin/echo %s >/dev/virtio-ports/testisocompletion && systemctl poweroff' -[Install] -RequiredBy=emergency.target -`, signalEmergencyString) - -func init() { - cmdTestIso.Flags().BoolVarP(&instInsecure, "inst-insecure", "S", false, "Do not verify signature on metal image") - cmdTestIso.Flags().BoolVar(&console, "console", false, "Connect qemu console to terminal, turn off automatic initramfs failure checking") - cmdTestIso.Flags().StringSliceVar(&pxeKernelArgs, "pxe-kargs", nil, "Additional kernel arguments for PXE") - - root.AddCommand(cmdTestIso) -} - -func liveArtifactExistsInBuild() error { - - if kola.CosaBuild.Meta.BuildArtifacts.LiveIso == nil || kola.CosaBuild.Meta.BuildArtifacts.LiveKernel == nil { - return fmt.Errorf("build %s is missing live artifacts", kola.CosaBuild.Meta.Name) - } - return nil -} - -func getAllTests() []string { - arch := coreosarch.CurrentRpmArch() - if kola.CosaBuild.Meta.Name == "rhcos" && arch != "s390x" && arch != "ppc64le" { - return tests_RHCOS_uefi - } - return []string{} -} - -func newBaseQemuBuilder(outdir string) (*platform.QemuBuilder, error) { - builder := qemu.NewMetalQemuBuilderDefault() - if enableUefi { - builder.Firmware = "uefi" - } - - if err := os.MkdirAll(outdir, 0755); err != nil { - return nil, err - } - - builder.InheritConsole = console - if !console { - builder.ConsoleFile = filepath.Join(outdir, "console.txt") - } - - if kola.QEMUOptions.Memory != "" { - parsedMem, err := strconv.ParseInt(kola.QEMUOptions.Memory, 10, 32) - if err != nil { - return nil, err - } - builder.MemoryMiB = int(parsedMem) - } - - return builder, nil -} - -func newQemuBuilder(outdir string) (*platform.QemuBuilder, *conf.Conf, error) { - builder, err := newBaseQemuBuilder(outdir) - if err != nil { - return nil, nil, err - } - - config, err := conf.EmptyIgnition().Render(conf.FailWarnings) - if err != nil { - return nil, nil, err - } - - err = forwardJournal(outdir, builder, config) - if err != nil { - return nil, nil, err - } - - return builder, config, nil -} - -func forwardJournal(outdir string, builder *platform.QemuBuilder, config *conf.Conf) error { - journalPipe, err := builder.VirtioJournal(config, "") - if err != nil { - return err - } - journalOut, err := os.OpenFile(filepath.Join(outdir, "journal.txt"), os.O_WRONLY|os.O_CREATE, 0644) - if err != nil { - return err - } - - go func() { - _, err := io.Copy(journalOut, journalPipe) - if err != nil && err != io.EOF { - panic(err) - } - }() - - return nil -} - -// See similar semantics in the `filterTests` of `kola.go`. -func filterTests(tests []string, patterns []string) ([]string, error) { - r := []string{} - for _, test := range tests { - if matches, err := kola.MatchesPatterns(test, patterns); err != nil { - return nil, err - } else if matches { - r = append(r, test) - } - } - return r, nil -} - -func runTestIso(cmd *cobra.Command, args []string) (err error) { - if kola.CosaBuild == nil { - return fmt.Errorf("Must provide --build") - } - tests := getAllTests() - if len(args) != 0 { - if tests, err = filterTests(tests, args); err != nil { - return err - } else if len(tests) == 0 { - return harness.SuiteEmpty - } - } - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - // Call `ParseDenyListYaml` to populate the `kola.DenylistedTests` var - err = kola.ParseDenyListYaml("qemu") - if err != nil { - plog.Fatal(err) - } - - finalTests := []string{} - for _, test := range tests { - if !kola.HasString(test, kola.DenylistedTests) { - matchTest, err := kola.MatchesPatterns(test, kola.DenylistedTests) - if err != nil { - return err - - } - if !matchTest { - finalTests = append(finalTests, test) - } - } - } - - // note this reassigns a *global* - outputDir, err = kola.SetupOutputDir(outputDir, "testiso") - if err != nil { - return err - } - - // see similar code in suite.go - reportDir := filepath.Join(outputDir, "reports") - if err := os.Mkdir(reportDir, 0777); err != nil { - return err - } - - reporter := reporters.NewJSONReporter("report.json", "testiso", "") - defer func() { - if reportErr := reporter.Output(reportDir); reportErr != nil && err != nil { - err = reportErr - } - }() - - var duration time.Duration - - atLeastOneFailed := false - for _, test := range finalTests { - - // All of these tests require buildextend-live to have been run - err = liveArtifactExistsInBuild() - if err != nil { - return err - } - - enableUefi = false - - fmt.Printf("Running test: %s\n", test) - components := strings.Split(test, ".") - - if kola.HasString("uefi", components) { - enableUefi = true - } - - switch components[0] { - case "iso-fips": - duration, err = testLiveFIPS(ctx, filepath.Join(outputDir, test)) - default: - plog.Fatalf("Unknown test name:%s", test) - } - - result := testresult.Pass - output := []byte{} - if err != nil { - result = testresult.Fail - output = []byte(err.Error()) - } - reporter.ReportTest(test, []string{}, result, duration, output) - if printResult(test, duration, err) { - atLeastOneFailed = true - } - } - - reporter.SetResult(testresult.Pass) - if atLeastOneFailed { - reporter.SetResult(testresult.Fail) - return harness.SuiteFailed - } - - return nil -} - -func awaitCompletion(ctx context.Context, inst *platform.QemuInstance, outdir string, qchan *os.File, booterrchan chan error, expected []string) (time.Duration, error) { - start := time.Now() - errchan := make(chan error) - go func() { - timeout := (time.Duration(installTimeoutMins*(100+kola.Options.ExtendTimeoutPercent)) * time.Minute) / 100 - time.Sleep(timeout) - errchan <- fmt.Errorf("timed out after %v", timeout) - }() - if !console { - go func() { - errBuf, err := inst.WaitIgnitionError(ctx) - if err == nil { - if errBuf != "" { - plog.Info("entered emergency.target in initramfs") - path := filepath.Join(outdir, "ignition-virtio-dump.txt") - if err := os.WriteFile(path, []byte(errBuf), 0644); err != nil { - plog.Errorf("Failed to write journal: %v", err) - } - err = platform.ErrInitramfsEmergency - } - } - if err != nil { - errchan <- err - } - }() - } - go func() { - err := inst.Wait() - // only one Wait() gets process data, so also manually check for signal - plog.Debugf("qemu exited err=%v", err) - if err == nil && inst.Signaled() { - err = errors.New("process killed") - } - if err != nil { - errchan <- errors.Wrapf(err, "QEMU unexpectedly exited while awaiting completion") - } - time.Sleep(1 * time.Minute) - errchan <- fmt.Errorf("QEMU exited; timed out waiting for completion") - }() - go func() { - r := bufio.NewReader(qchan) - for _, exp := range expected { - l, err := r.ReadString('\n') - if err != nil { - if err == io.EOF { - // this may be from QEMU getting killed or exiting; wait a bit - // to give a chance for .Wait() above to feed the channel with a - // better error - time.Sleep(1 * time.Second) - errchan <- fmt.Errorf("Got EOF from completion channel, %s expected", exp) - } else { - errchan <- errors.Wrapf(err, "reading from completion channel") - } - return - } - line := strings.TrimSpace(l) - if line != exp { - errchan <- fmt.Errorf("Unexpected string from completion channel: %s expected: %s", line, exp) - return - } - plog.Debugf("Matched expected message %s", exp) - } - plog.Debugf("Matched all expected messages") - // OK! - errchan <- nil - }() - go func() { - //check for error when switching boot order - if booterrchan != nil { - if err := <-booterrchan; err != nil { - errchan <- err - } - } - }() - err := <-errchan - elapsed := time.Since(start) - if err == nil { - // No error so far, check the console and journal files - consoleFile := filepath.Join(outdir, "console.txt") - journalFile := filepath.Join(outdir, "journal.txt") - files := []string{consoleFile, journalFile} - for _, file := range files { - fileName := filepath.Base(file) - // Check if the file exists - _, err := os.Stat(file) - if os.IsNotExist(err) { - fmt.Printf("The file: %v does not exist\n", fileName) - continue - } else if err != nil { - fmt.Println(err) - return elapsed, err - } - // Read the contents of the file - fileContent, err := os.ReadFile(file) - if err != nil { - fmt.Println(err) - return elapsed, err - } - // Check for badness with CheckConsole - warnOnly, badlines := kola.CheckConsole([]byte(fileContent), nil) - if len(badlines) > 0 { - for _, badline := range badlines { - if warnOnly { - plog.Errorf("bad log line detected: %v", badline) - } else { - plog.Warningf("bad log line detected: %v", badline) - } - } - if !warnOnly { - err = fmt.Errorf("errors found in log files") - return elapsed, err - } - } - } - } - return elapsed, err -} - -func printResult(test string, duration time.Duration, err error) bool { - result := "PASS" - if err != nil { - result = "FAIL" - } - fmt.Printf("%s: %s (%s)\n", result, test, duration.Round(time.Millisecond).String()) - if err != nil { - fmt.Printf(" %s\n", err) - return true - } - return false -} - -// testLiveFIPS verifies that adding fips=1 to the ISO results in a FIPS mode system -func testLiveFIPS(ctx context.Context, outdir string) (time.Duration, error) { - tmpd, err := os.MkdirTemp("", "kola-testiso") - if err != nil { - return 0, err - } - defer os.RemoveAll(tmpd) - - builddir := kola.CosaBuild.Dir - isopath := filepath.Join(builddir, kola.CosaBuild.Meta.BuildArtifacts.LiveIso.Path) - builder, config, err := newQemuBuilder(outdir) - if err != nil { - return 0, err - } - defer builder.Close() - if err := builder.AddIso(isopath, "", false); err != nil { - return 0, err - } - - // This is the core change under test - adding the `fips=1` kernel argument via - // coreos-installer iso kargs modify should enter fips mode. - // Removing this line should cause this test to fail. - builder.AppendKernelArgs = "fips=1" - - completionChannel, err := builder.VirtioChannelRead("testisocompletion") - if err != nil { - return 0, err - } - - config.AddSystemdUnit("fips-verify.service", ` -[Unit] -OnFailure=emergency.target -OnFailureJobMode=isolate -Before=fips-signal-ok.service - -[Service] -Type=oneshot -RemainAfterExit=yes -ExecStart=grep 1 /proc/sys/crypto/fips_enabled -ExecStart=grep FIPS etc/crypto-policies/config - -[Install] -RequiredBy=fips-signal-ok.service -`, conf.Enable) - config.AddSystemdUnit("fips-signal-ok.service", liveSignalOKUnit, conf.Enable) - config.AddSystemdUnit("fips-emergency-target.service", signalFailureUnit, conf.Enable) - - // Just for reliability, we'll run fully offline - builder.Append("-net", "none") - - builder.SetConfig(config) - mach, err := builder.Exec() - if err != nil { - return 0, errors.Wrapf(err, "running iso") - } - defer mach.Destroy() - - return awaitCompletion(ctx, mach, outdir, completionChannel, nil, []string{liveOKSignal}) -} diff --git a/mantle/kola/tests/iso/live-fips.go b/mantle/kola/tests/iso/live-fips.go new file mode 100644 index 0000000000..c1e8732e0c --- /dev/null +++ b/mantle/kola/tests/iso/live-fips.go @@ -0,0 +1,99 @@ +package iso + +import ( + "path/filepath" + "time" + + "github.com/coreos/coreos-assembler/mantle/kola" + "github.com/coreos/coreos-assembler/mantle/kola/cluster" + "github.com/coreos/coreos-assembler/mantle/kola/register" + "github.com/coreos/coreos-assembler/mantle/platform/conf" + "github.com/coreos/coreos-assembler/mantle/platform/machine/qemu" +) + +func init() { + // These tests only run on RHCOS + var tests_RHCOS_uefi = []string{ + "iso-fips.uefi", + } + for _, testName := range tests_RHCOS_uefi { + register.RegisterTest(®ister.Test{ + Run: func(c cluster.TestCluster) { + opts := getIsoTestOpts(testName) + testLiveFIPS(c, opts) + }, + ClusterSize: 0, + Name: "iso." + testName, + Timeout: installTimeoutMins * time.Minute, + Distros: []string{"rhcos"}, + Platforms: []string{"qemu"}, + Architectures: []string{"x86_64", "aarch64"}, + }) + } +} + +// testLiveFIPS verifies that adding fips=1 to the ISO results in a FIPS mode system +func testLiveFIPS(c cluster.TestCluster, opts IsoTestOpts) { + var outdir string + //var qc *qemu.Cluster + switch pc := c.Cluster.(type) { + case *qemu.Cluster: + outdir = pc.RuntimeConf().OutputDir + //qc = pc + default: + c.Fatalf("Unsupported cluster type") + } + + isopath := filepath.Join(kola.CosaBuild.Dir, kola.CosaBuild.Meta.BuildArtifacts.LiveIso.Path) + builder, config, err := newQemuBuilder(opts, outdir) + if err != nil { + c.Fatal(err) + } + defer builder.Close() + if err := builder.AddIso(isopath, "", false); err != nil { + c.Fatal(err) + } + + // This is the core change under test - adding the `fips=1` kernel argument via + // coreos-installer iso kargs modify should enter fips mode. + // Removing this line should cause this test to fail. + builder.AppendKernelArgs = "fips=1" + + completionChannel, err := builder.VirtioChannelRead("testisocompletion") + if err != nil { + c.Fatal(err) + } + + config.AddSystemdUnit("fips-verify.service", ` +[Unit] +OnFailure=emergency.target +OnFailureJobMode=isolate +Before=fips-signal-ok.service + +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=grep 1 /proc/sys/crypto/fips_enabled +ExecStart=grep FIPS etc/crypto-policies/config + +[Install] +RequiredBy=fips-signal-ok.service +`, conf.Enable) + config.AddSystemdUnit("fips-signal-ok.service", liveSignalOKUnit, conf.Enable) + config.AddSystemdUnit("fips-emergency-target.service", signalFailureUnit, conf.Enable) + + // Just for reliability, we'll run fully offline + builder.Append("-net", "none") + + builder.SetConfig(config) + mach, err := builder.Exec() + if err != nil { + c.Fatal(err) + } + defer mach.Destroy() + + err = awaitCompletion(c, mach, opts.console, outdir, completionChannel, nil, []string{liveOKSignal}) + if err != nil { + c.Fatal(err) + } +} From 42986603d1a8a1e91a0222db8c369a2c7c456faf Mon Sep 17 00:00:00 2001 From: Nikita Dubrovskii Date: Mon, 24 Nov 2025 11:04:34 +0100 Subject: [PATCH 13/31] kola/tests/iso: extract CheckTestOutput helper function Extract duplicate completion channel reading logic into a reusable CheckTestOutput helper function. This reduces code duplication across iso tests. --- mantle/kola/tests/iso/common.go | 50 ++++++++++++++++------------- mantle/kola/tests/iso/live-login.go | 29 ++--------------- 2 files changed, 29 insertions(+), 50 deletions(-) diff --git a/mantle/kola/tests/iso/common.go b/mantle/kola/tests/iso/common.go index f090e5ceed..249829bf9c 100644 --- a/mantle/kola/tests/iso/common.go +++ b/mantle/kola/tests/iso/common.go @@ -201,6 +201,32 @@ func newQemuBuilderWithDisk(opts IsoTestOpts, outdir string) (*platform.QemuBuil return builder, config, nil } +// Reads from a virtio channel and validates that the expected +// strings are received in order. Returns an error if EOF is encountered, a read +// error occurs, or an unexpected string is received. +func CheckTestOutput(output *os.File, expected []string) error { + reader := bufio.NewReader(output) + for _, exp := range expected { + line, err := reader.ReadString('\n') + if err != nil { + if err == io.EOF { + // this may be from QEMU getting killed or exiting; wait a bit + // to give a chance for .Wait() above to feed the channel with a + // better error + time.Sleep(1 * time.Second) + return fmt.Errorf("got EOF from completion channel, %s expected", exp) + } else { + return errors.Wrapf(err, "reading from completion channel") + } + } + line = strings.TrimSpace(line) + if line != exp { + return fmt.Errorf("unexpected string from completion channel: %q, expected: %q", line, exp) + } + } + return nil +} + func awaitCompletion(c cluster.TestCluster, inst *platform.QemuInstance, console bool, outdir string, qchan *os.File, booterrchan chan error, expected []string) error { ctx := c.Context() @@ -242,29 +268,7 @@ func awaitCompletion(c cluster.TestCluster, inst *platform.QemuInstance, console errchan <- fmt.Errorf("QEMU exited; timed out waiting for completion") }() go func() { - r := bufio.NewReader(qchan) - for _, exp := range expected { - l, err := r.ReadString('\n') - if err != nil { - if err == io.EOF { - // this may be from QEMU getting killed or exiting; wait a bit - // to give a chance for .Wait() above to feed the channel with a - // better error - time.Sleep(1 * time.Second) - errchan <- fmt.Errorf("Got EOF from completion channel, %s expected", exp) - } else { - errchan <- errors.Wrapf(err, "reading from completion channel") - } - return - } - line := strings.TrimSpace(l) - if line != exp { - errchan <- fmt.Errorf("Unexpected string from completion channel: %s expected: %s", line, exp) - return - } - } - // OK! - errchan <- nil + errchan <- CheckTestOutput(qchan, expected) }() go func() { //check for error when switching boot order diff --git a/mantle/kola/tests/iso/live-login.go b/mantle/kola/tests/iso/live-login.go index 52a928641e..8335f11c86 100644 --- a/mantle/kola/tests/iso/live-login.go +++ b/mantle/kola/tests/iso/live-login.go @@ -1,12 +1,8 @@ package iso import ( - "bufio" - "fmt" - "io" "path/filepath" "strings" - "time" "github.com/coreos/coreos-assembler/mantle/kola" "github.com/coreos/coreos-assembler/mantle/platform" @@ -94,27 +90,7 @@ version: 1.1.0`) // Read line in a goroutine and send errors to channel go func() { - exp := "coreos-liveiso-success" - line, err := bufio.NewReader(output).ReadString('\n') - if err != nil { - if err == io.EOF { - // this may be from QEMU getting killed or exiting; wait a bit - // to give a chance for .Wait() above to feed the channel with a - // better error - time.Sleep(1 * time.Second) - errchan <- fmt.Errorf("Got EOF from completion channel, %s expected", exp) - } else { - errchan <- errors.Wrapf(err, "reading from completion channel") - } - return - } - line = strings.TrimSpace(line) - if line != exp { - errchan <- fmt.Errorf("Unexpected string from completion channel: %q, expected: %q", line, exp) - return - } - // OK! - errchan <- nil + errchan <- CheckTestOutput(output, []string{"coreos-liveiso-success"}) }() isopath := filepath.Join(kola.CosaBuild.Dir, kola.CosaBuild.Meta.BuildArtifacts.LiveIso.Path) @@ -136,8 +112,7 @@ version: 1.1.0`) c.Fatalf("Unsupported cluster type") } - err := <-errchan - if err != nil { + if err := <-errchan; err != nil { c.Fatal(err) } } From 4f2092cf0b700131e5d5099e7d41b158a1ad72df Mon Sep 17 00:00:00 2001 From: Nikita Dubrovskii Date: Mon, 24 Nov 2025 11:24:50 +0100 Subject: [PATCH 14/31] kola/tests/iso: remove redundant console and journal checks The kola test harness already performs console and journal checks automatically after each test completes, making the duplicate checks in awaitCompletion unnecessary. Call flow in mantle/kola/harness.go: ``` runProvidedTests -> runTest -> handleConsoleChecks("console", ...) -> CheckConsole -> handleConsoleChecks("journal", ...) -> CheckConsole ``` --- mantle/kola/tests/iso/common.go | 39 --------------------------------- 1 file changed, 39 deletions(-) diff --git a/mantle/kola/tests/iso/common.go b/mantle/kola/tests/iso/common.go index 249829bf9c..8e782eb164 100644 --- a/mantle/kola/tests/iso/common.go +++ b/mantle/kola/tests/iso/common.go @@ -279,45 +279,6 @@ func awaitCompletion(c cluster.TestCluster, inst *platform.QemuInstance, console } }() err := <-errchan - if err == nil { - // No error so far, check the console and journal files - consoleFile := filepath.Join(outdir, "console.txt") - journalFile := filepath.Join(outdir, "journal.txt") - files := []string{consoleFile, journalFile} - for _, file := range files { - fileName := filepath.Base(file) - // Check if the file exists - _, err := os.Stat(file) - if os.IsNotExist(err) { - fmt.Printf("The file: %v does not exist\n", fileName) - continue - } else if err != nil { - fmt.Println(err) - return err - } - // Read the contents of the file - fileContent, err := os.ReadFile(file) - if err != nil { - fmt.Println(err) - return err - } - // Check for badness with CheckConsole - warnOnly, badlines := kola.CheckConsole([]byte(fileContent), nil) - if len(badlines) > 0 { - for _, badline := range badlines { - if warnOnly { - c.Errorf("bad log line detected: %v", badline) - } else { - c.Logf("bad log line detected: %v", badline) - } - } - if !warnOnly { - err = fmt.Errorf("errors found in log files") - return err - } - } - } - } return err } From 65b2d627a6ed5b0245fe6452d64d89fc8d40696c Mon Sep 17 00:00:00 2001 From: Nikita Dubrovskii Date: Fri, 21 Nov 2025 14:27:33 +0100 Subject: [PATCH 15/31] kola/tests/iso: refactor *fips* tests Update the test to use the existing MachineBuilder API after the testiso migration. --- mantle/kola/tests/iso/live-fips.go | 101 +++++++++++++++-------------- 1 file changed, 53 insertions(+), 48 deletions(-) diff --git a/mantle/kola/tests/iso/live-fips.go b/mantle/kola/tests/iso/live-fips.go index c1e8732e0c..fac149656e 100644 --- a/mantle/kola/tests/iso/live-fips.go +++ b/mantle/kola/tests/iso/live-fips.go @@ -7,15 +7,18 @@ import ( "github.com/coreos/coreos-assembler/mantle/kola" "github.com/coreos/coreos-assembler/mantle/kola/cluster" "github.com/coreos/coreos-assembler/mantle/kola/register" + "github.com/coreos/coreos-assembler/mantle/platform" "github.com/coreos/coreos-assembler/mantle/platform/conf" "github.com/coreos/coreos-assembler/mantle/platform/machine/qemu" + "github.com/pkg/errors" ) +// These tests only run on RHCOS +var tests_RHCOS_uefi = []string{ + "iso-fips.uefi", +} + func init() { - // These tests only run on RHCOS - var tests_RHCOS_uefi = []string{ - "iso-fips.uefi", - } for _, testName := range tests_RHCOS_uefi { register.RegisterTest(®ister.Test{ Run: func(c cluster.TestCluster) { @@ -24,6 +27,7 @@ func init() { }, ClusterSize: 0, Name: "iso." + testName, + Description: "verifies that adding fips=1 to the ISO results in a FIPS mode system", Timeout: installTimeoutMins * time.Minute, Distros: []string{"rhcos"}, Platforms: []string{"qemu"}, @@ -32,40 +36,7 @@ func init() { } } -// testLiveFIPS verifies that adding fips=1 to the ISO results in a FIPS mode system -func testLiveFIPS(c cluster.TestCluster, opts IsoTestOpts) { - var outdir string - //var qc *qemu.Cluster - switch pc := c.Cluster.(type) { - case *qemu.Cluster: - outdir = pc.RuntimeConf().OutputDir - //qc = pc - default: - c.Fatalf("Unsupported cluster type") - } - - isopath := filepath.Join(kola.CosaBuild.Dir, kola.CosaBuild.Meta.BuildArtifacts.LiveIso.Path) - builder, config, err := newQemuBuilder(opts, outdir) - if err != nil { - c.Fatal(err) - } - defer builder.Close() - if err := builder.AddIso(isopath, "", false); err != nil { - c.Fatal(err) - } - - // This is the core change under test - adding the `fips=1` kernel argument via - // coreos-installer iso kargs modify should enter fips mode. - // Removing this line should cause this test to fail. - builder.AppendKernelArgs = "fips=1" - - completionChannel, err := builder.VirtioChannelRead("testisocompletion") - if err != nil { - c.Fatal(err) - } - - config.AddSystemdUnit("fips-verify.service", ` -[Unit] +var fipsVerify = `[Unit] OnFailure=emergency.target OnFailureJobMode=isolate Before=fips-signal-ok.service @@ -77,23 +48,57 @@ ExecStart=grep 1 /proc/sys/crypto/fips_enabled ExecStart=grep FIPS etc/crypto-policies/config [Install] -RequiredBy=fips-signal-ok.service -`, conf.Enable) - config.AddSystemdUnit("fips-signal-ok.service", liveSignalOKUnit, conf.Enable) - config.AddSystemdUnit("fips-emergency-target.service", signalFailureUnit, conf.Enable) +RequiredBy=fips-signal-ok.service` - // Just for reliability, we'll run fully offline - builder.Append("-net", "none") +func testLiveFIPS(c cluster.TestCluster, opts IsoTestOpts) { + qc, ok := c.Cluster.(*qemu.Cluster) + if !ok { + c.Fatalf("Unsupported cluster type") + } - builder.SetConfig(config) - mach, err := builder.Exec() + config, err := conf.EmptyIgnition().Render(conf.FailWarnings) if err != nil { c.Fatal(err) } - defer mach.Destroy() + config.AddSystemdUnit("fips-verify.service", fipsVerify, conf.Enable) + config.AddSystemdUnit("fips-signal-ok.service", liveSignalOKUnit, conf.Enable) + config.AddSystemdUnit("fips-emergency-target.service", signalFailureUnit, conf.Enable) + keys, err := qc.Keys() + if err != nil { + c.Fatal(err) + } + config.CopyKeys(keys) + + errchan := make(chan error) + setupDisks := func(_ platform.MachineOptions, builder *platform.QemuBuilder) error { + isopath := filepath.Join(kola.CosaBuild.Dir, kola.CosaBuild.Meta.BuildArtifacts.LiveIso.Path) + if err := builder.AddIso(isopath, "", false); err != nil { + return err + } + completionChannel, err := builder.VirtioChannelRead("testisocompletion") + if err != nil { + return errors.Wrap(err, "setting up virtio-serial channel") + } + go func() { + errchan <- CheckTestOutput(completionChannel, []string{liveOKSignal}) + }() + return nil + } + + options := platform.MachineOptions{AppendKernelArgs: "fips=1"} + if opts.enableUefi { + options.Firmware = "uefi" + } - err = awaitCompletion(c, mach, opts.console, outdir, completionChannel, nil, []string{liveOKSignal}) + machineBuilder := &qemu.MachineBuilder{ + SetupDisks: setupDisks, + } + _, err = qc.NewMachineWithBuilder(config, options, machineBuilder) if err != nil { + c.Fatalf("Unable to create test machine: %v", err) + } + + if err := <-errchan; err != nil { c.Fatal(err) } } From c437fb40c0f070611d1b961cd949b1d536e7256c Mon Sep 17 00:00:00 2001 From: Nikita Dubrovskii Date: Mon, 24 Nov 2025 10:23:55 +0100 Subject: [PATCH 16/31] kola/tests/iso: refactor *iscsi* tests --- mantle/kola/tests/iso/live-iscsi.go | 166 ++++++++++------------------ 1 file changed, 60 insertions(+), 106 deletions(-) diff --git a/mantle/kola/tests/iso/live-iscsi.go b/mantle/kola/tests/iso/live-iscsi.go index 593692b123..c8a285ba0b 100644 --- a/mantle/kola/tests/iso/live-iscsi.go +++ b/mantle/kola/tests/iso/live-iscsi.go @@ -15,6 +15,7 @@ import ( "github.com/coreos/coreos-assembler/mantle/platform/conf" "github.com/coreos/coreos-assembler/mantle/platform/machine/qemu" coreosarch "github.com/coreos/stream-metadata-go/arch" + "github.com/pkg/errors" ) var ( @@ -69,6 +70,7 @@ func init() { ClusterSize: 0, Name: "iso." + testName, Description: "Verify iSCSI install works.", + Tags: []string{kola.NeedsInternetTag}, Timeout: installTimeoutMins * time.Minute, Flags: []register.Flag{}, Platforms: []string{"qemu"}, @@ -76,37 +78,6 @@ func init() { } } -func isoOfflineInstallIscsiIbftUefi(c cluster.TestCluster) { - opts := IsoTestOpts{ - enableUefi: true, - isOffline: true, - enableIbft: true, - } - opts.SetInsecureOnDevBuild() - isoInstalliScsi(c, opts) -} - -func isoOfflineInstallIscsiIbftMpath(c cluster.TestCluster) { - opts := IsoTestOpts{ - enableUefi: true, - isOffline: true, - enableMultipath: true, - enableIbft: true, - } - opts.SetInsecureOnDevBuild() - isoInstalliScsi(c, opts) -} - -func isoOfflineInstallIscsiManual(c cluster.TestCluster) { - opts := IsoTestOpts{ - isOffline: true, - manual: true, - enableIbft: true, - } - opts.SetInsecureOnDevBuild() - isoInstalliScsi(c, opts) -} - //go:embed iscsi_butane_setup.yaml var iscsi_butane_config string @@ -135,16 +106,12 @@ var iscsi_butane_config string // - when the system is booted, write a success string to /dev/virtio-ports/testisocompletion // - as this serial device is mapped to the host serial device, the test concludes func isoInstalliScsi(c cluster.TestCluster, opts IsoTestOpts) { - var outdir string - //var qc *qemu.Cluster - switch pc := c.Cluster.(type) { - case *qemu.Cluster: - outdir = pc.RuntimeConf().OutputDir - //qc = pc - default: + qc, ok := c.Cluster.(*qemu.Cluster) + if !ok { c.Fatalf("Unsupported cluster type") } + // Prepare config var butane string if opts.enableIbft && opts.enableMultipath { butane = strings.ReplaceAll(iscsi_butane_config, "COREOS_INSTALLER_KARGS", "--append-karg rd.iscsi.firmware=1 --append-karg rd.multipath=default --append-karg root=/dev/disk/by-label/dm-mpath-root --append-karg rw") @@ -153,91 +120,78 @@ func isoInstalliScsi(c cluster.TestCluster, opts IsoTestOpts) { } else if opts.manual { butane = strings.ReplaceAll(iscsi_butane_config, "COREOS_INSTALLER_KARGS", "--append-karg netroot=iscsi:10.0.2.15::::iqn.2024-05.com.coreos:0") } - - if kola.CosaBuild.Meta.BuildArtifacts.LiveIso == nil || kola.CosaBuild.Meta.BuildArtifacts.LiveKernel == nil { - c.Fatalf("build %s is missing live artifacts", kola.CosaBuild.Meta.Name) - } - builddir := kola.CosaBuild.Dir - isopath := filepath.Join(builddir, kola.CosaBuild.Meta.BuildArtifacts.LiveIso.Path) - builder, err := newBaseQemuBuilder(opts, outdir) - if err != nil { - c.Fatal(err) - } - defer builder.Close() - if err := builder.AddIso(isopath, "", false); err != nil { - c.Fatal(err) - } - - completionChannel, err := builder.VirtioChannelRead("testisocompletion") - if err != nil { - c.Fatal(err) - } - - // Create a serial channel to read the logs from the nested VM - nestedVmLogsChannel, err := builder.VirtioChannelRead("nestedvmlogs") + var iscsiTargetConfig = conf.Butane(butane) + config, err := iscsiTargetConfig.Render(conf.FailWarnings) if err != nil { c.Fatal(err) } - - // Create a file to write the contents of the serial channel into - nestedVMConsole, err := os.OpenFile(filepath.Join(outdir, "nested_vm_console.txt"), os.O_WRONLY|os.O_CREATE, 0644) + keys, err := qc.Keys() if err != nil { c.Fatal(err) } + config.CopyKeys(keys) + // Add a failure target to stop the test if something go wrong rather than waiting for the 10min timeout + config.AddSystemdUnit("coreos-test-entered-emergency-target.service", signalFailureUnit, conf.Enable) + config.MountHost("/var/cosaroot", true) - go func() { - _, err := io.Copy(nestedVMConsole, nestedVmLogsChannel) - if err != nil && err != io.EOF { - panic(err) + errchan := make(chan error) + setupDisks := func(_ platform.MachineOptions, builder *platform.QemuBuilder) error { + isopath := filepath.Join(kola.CosaBuild.Dir, kola.CosaBuild.Meta.BuildArtifacts.LiveIso.Path) + if err := builder.AddIso(isopath, "", false); err != nil { + return err } - }() - // empty disk to use as an iscsi target to install coreOS on and subseqently boot - // Also add a 10G disk that we will mount on /var, to increase space available when pulling containers - err = builder.AddDisksFromSpecs([]string{"10G:serial=target", "10G:serial=var"}) - if err != nil { - c.Fatal(err) + completionChannel, err := builder.VirtioChannelRead("testisocompletion") + if err != nil { + return errors.Wrap(err, "setting up virtio-serial channel") + } + go func() { + errchan <- CheckTestOutput(completionChannel, []string{"iscsi-boot-ok"}) + }() + + // Create a serial channel to read the logs from the nested VM + nestedVmLogsChannel, err := builder.VirtioChannelRead("nestedvmlogs") + if err != nil { + return err + } + // Create a file to write the contents of the serial channel into + path := filepath.Join(filepath.Dir(builder.ConsoleFile), "nested_vm_console.txt") + nestedVMConsole, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE, 0644) + if err != nil { + return err + } + go func() { + _, err := io.Copy(nestedVMConsole, nestedVmLogsChannel) + if err != nil && err != io.EOF { + errchan <- errors.Wrap(err, "copying nested VM logs") + } + }() + // empty disk to use as an iscsi target to install coreOS on and subseqently boot + // Also add a 10G disk that we will mount on /var, to increase space available when pulling containers + if err := builder.AddDisksFromSpecs([]string{"10G:serial=target", "10G:serial=var"}); err != nil { + return err + } + // Bind mount in the COSA rootfs into the VM so we can use it as a + // read-only rootfs for quickly starting the container to kola + // qemuexec the nested VM for the test. See resources/iscsi_butane_setup.yaml + builder.MountHost("/", "/var/cosaroot", true) + return nil } // We need more memory to start another VM within ! - builder.MemoryMiB = 2048 - - var iscsiTargetConfig = conf.Butane(butane) - - config, err := iscsiTargetConfig.Render(conf.FailWarnings) - if err != nil { - c.Fatal(err) + options := platform.MachineOptions{MinMemory: 2048} + if opts.enableUefi { + options.Firmware = "uefi" } - err = forwardJournal(outdir, builder, config) - if err != nil { - c.Fatal(err) + machineBuilder := &qemu.MachineBuilder{ + SetupDisks: setupDisks, } - - // Add a failure target to stop the test if something go wrong rather than waiting for the 10min timeout - config.AddSystemdUnit("coreos-test-entered-emergency-target.service", signalFailureUnit, conf.Enable) - - // enable network - builder.EnableUsermodeNetworking([]platform.HostForwardPort{}, "") - - // keep auto-login enabled for easier debug when running console - config.AddAutoLogin() - - builder.SetConfig(config) - - // Bind mount in the COSA rootfs into the VM so we can use it as a - // read-only rootfs for quickly starting the container to kola - // qemuexec the nested VM for the test. See resources/iscsi_butane_setup.yaml - builder.MountHost("/", "/var/cosaroot", true) - config.MountHost("/var/cosaroot", true) - - mach, err := builder.Exec() + _, err = qc.NewMachineWithBuilder(config, options, machineBuilder) if err != nil { - c.Fatal(err) + c.Fatalf("Unable to create test machine: %v", err) } - defer mach.Destroy() - err = awaitCompletion(c, mach, opts.console, outdir, completionChannel, nil, []string{"iscsi-boot-ok"}) - if err != nil { + if err := <-errchan; err != nil { c.Fatal(err) } } From 98c7cef12e79a6da319a718f7f0e56baea41f41e Mon Sep 17 00:00:00 2001 From: Nikita Dubrovskii Date: Mon, 24 Nov 2025 12:13:20 +0100 Subject: [PATCH 17/31] kola/tests/iso: refactor iso-as-disk* tests Also remove the 4k sector size test variant as it's technically impossible to configure. The AddIso() function and iso-as-disk implementation do not support sector size configuration - the ISO is attached as a raw, readonly device without any sector size options. The Disk.SectorSize field only applies to regular disk images added via AddDisk() or AddDisksFromSpecs(), not to ISOs attached via AddIso(). Since iso-as-disk tests boot directly from the ISO file (not a disk image), sector size configuration is not applicable. --- mantle/kola/tests/iso/live-as-disk.go | 80 ++++++++++++++++----------- 1 file changed, 49 insertions(+), 31 deletions(-) diff --git a/mantle/kola/tests/iso/live-as-disk.go b/mantle/kola/tests/iso/live-as-disk.go index 7d9fe6f803..700c24a92c 100644 --- a/mantle/kola/tests/iso/live-as-disk.go +++ b/mantle/kola/tests/iso/live-as-disk.go @@ -7,20 +7,26 @@ import ( "github.com/coreos/coreos-assembler/mantle/kola" "github.com/coreos/coreos-assembler/mantle/kola/cluster" "github.com/coreos/coreos-assembler/mantle/kola/register" + "github.com/coreos/coreos-assembler/mantle/platform" "github.com/coreos/coreos-assembler/mantle/platform/conf" "github.com/coreos/coreos-assembler/mantle/platform/machine/qemu" + coreosarch "github.com/coreos/stream-metadata-go/arch" + "github.com/pkg/errors" ) +// The iso-as-disk tests are only supported in x86_64 because other +// architectures don't have the required hybrid partition table. +var tests_as_disk_x86_64 = []string{ + "iso-as-disk.bios", + "iso-as-disk.uefi", + "iso-as-disk.uefi-secure", +} + func init() { - // The iso-as-disk tests are only supported in x86_64 because other - // architectures don't have the required hybrid partition table. - var tests_as_disk_x86_64 = []string{ - "iso-as-disk.bios", - "iso-as-disk.uefi", - "iso-as-disk.uefi-secure", - "iso-as-disk.4k.uefi", + arch := coreosarch.CurrentRpmArch() + if arch != "x86_64" { + return } - for _, testName := range tests_as_disk_x86_64 { register.RegisterTest(®ister.Test{ Run: func(c cluster.TestCluster) { @@ -38,44 +44,56 @@ func init() { } func isoTestAsDisk(c cluster.TestCluster, opts IsoTestOpts) { - var outdir string - //var qc *qemu.Cluster - switch pc := c.Cluster.(type) { - case *qemu.Cluster: - outdir = pc.RuntimeConf().OutputDir - //qc = pc - default: + qc, ok := c.Cluster.(*qemu.Cluster) + if !ok { c.Fatalf("Unsupported cluster type") } - isopath := filepath.Join(kola.CosaBuild.Dir, kola.CosaBuild.Meta.BuildArtifacts.LiveIso.Path) - builder, config, err := newQemuBuilder(opts, outdir) + config, err := conf.EmptyIgnition().Render(conf.FailWarnings) if err != nil { c.Fatal(err) } - defer builder.Close() - // Drop the bootindex bit (applicable to all arches except s390x and ppc64le); we want it to be the default - if err := builder.AddIso(isopath, "", true); err != nil { + config.AddSystemdUnit("live-signal-ok.service", liveSignalOKUnit, conf.Enable) + config.AddSystemdUnit("verify-no-efi-boot-entry.service", verifyNoEFIBootEntry, conf.Enable) + keys, err := qc.Keys() + if err != nil { c.Fatal(err) } + config.CopyKeys(keys) - completionChannel, err := builder.VirtioChannelRead("testisocompletion") - if err != nil { - c.Fatal(err) + errchan := make(chan error) + setupDisks := func(_ platform.MachineOptions, builder *platform.QemuBuilder) error { + isopath := filepath.Join(kola.CosaBuild.Dir, kola.CosaBuild.Meta.BuildArtifacts.LiveIso.Path) + if err := builder.AddIso(isopath, "", true); err != nil { + return err + } + completionChannel, err := builder.VirtioChannelRead("testisocompletion") + if err != nil { + return errors.Wrap(err, "setting up virtio-serial channel") + } + go func() { + errchan <- CheckTestOutput(completionChannel, []string{liveOKSignal}) + }() + return nil } - config.AddSystemdUnit("live-signal-ok.service", liveSignalOKUnit, conf.Enable) - config.AddSystemdUnit("verify-no-efi-boot-entry.service", verifyNoEFIBootEntry, conf.Enable) - builder.SetConfig(config) + options := platform.MachineOptions{} + switch { + case opts.enableUefiSecure: + options.Firmware = "uefi-secure" + case opts.enableUefi: + options.Firmware = "uefi" + } - mach, err := builder.Exec() + machineBuilder := &qemu.MachineBuilder{ + SetupDisks: setupDisks, + } + _, err = qc.NewMachineWithBuilder(config, options, machineBuilder) if err != nil { - c.Fatal(err) + c.Fatalf("Unable to create test machine: %v", err) } - defer mach.Destroy() - err = awaitCompletion(c, mach, opts.console, outdir, completionChannel, nil, []string{liveOKSignal}) - if err != nil { + if err := <-errchan; err != nil { c.Fatal(err) } } From c939c2cec0634ccacead716e86b146b354348a84 Mon Sep 17 00:00:00 2001 From: Nikita Dubrovskii Date: Wed, 1 Apr 2026 14:00:48 +0200 Subject: [PATCH 18/31] mantle/qemu: conditionally wait for SSH address Only wait for SSH address when usermode networking is enabled. This allows ISO tests without networking to proceed without unnecessary SSH address waiting. --- mantle/platform/machine/qemu/cluster.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/mantle/platform/machine/qemu/cluster.go b/mantle/platform/machine/qemu/cluster.go index 162a96e891..647537471e 100644 --- a/mantle/platform/machine/qemu/cluster.go +++ b/mantle/platform/machine/qemu/cluster.go @@ -129,8 +129,10 @@ func (qc *Cluster) NewMachineWithBuilder(userdata any, options platform.MachineO } qm.inst = inst - if err := qc.waitForSSHAddress(qm, inst); err != nil { - return nil, err + if qemuBuilder.UsermodeNetworking { + if err := qc.waitForSSHAddress(qm, inst); err != nil { + return nil, err + } } // Run StartMachine, which blocks on the machine being booted up enough From 3dd9de643a5177fe1abb37f89ed6c22ffcf9a3e5 Mon Sep 17 00:00:00 2001 From: Nikita Dubrovskii Date: Wed, 1 Apr 2026 14:11:59 +0200 Subject: [PATCH 19/31] mantle/qemu: add Instance() method to access QemuInstance Add Instance() method to expose the underlying QemuInstance from a platform.Machine. This allows tests to access QEMU-specific functionality not available through the generic interface. --- mantle/platform/machine/qemu/cluster.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/mantle/platform/machine/qemu/cluster.go b/mantle/platform/machine/qemu/cluster.go index 647537471e..72d4b702e3 100644 --- a/mantle/platform/machine/qemu/cluster.go +++ b/mantle/platform/machine/qemu/cluster.go @@ -335,3 +335,13 @@ func (qc *Cluster) SetupDefaultNetwork(options platform.MachineOptions, builder } return nil } + +// Instance returns the underlying QemuInstance for a given Machine. +// This allows tests to access QEMU-specific functionality. +func (qc *Cluster) Instance(m platform.Machine) *platform.QemuInstance { + qm, ok := m.(*machine) + if !ok { + return nil + } + return qm.inst +} From ea32701d18771a889e8c8888134f7c69e685eb05 Mon Sep 17 00:00:00 2001 From: Nikita Dubrovskii Date: Wed, 26 Nov 2025 08:38:37 +0100 Subject: [PATCH 20/31] kola/tests/iso: refactor (iso|miniso)-(offline-)?install* tests --- mantle/kola/tests/iso/common.go | 89 ++++++- mantle/kola/tests/iso/live-iso.go | 326 ++++++++++++++++++++++---- mantle/platform/machine/qemu/metal.go | 261 --------------------- 3 files changed, 360 insertions(+), 316 deletions(-) diff --git a/mantle/kola/tests/iso/common.go b/mantle/kola/tests/iso/common.go index 8e782eb164..3c1a082f87 100644 --- a/mantle/kola/tests/iso/common.go +++ b/mantle/kola/tests/iso/common.go @@ -4,6 +4,8 @@ import ( "bufio" "fmt" "io" + "net" + "net/http" "os" "path/filepath" "strconv" @@ -21,10 +23,63 @@ import ( const ( installTimeoutMins = 12 - // https://github.com/coreos/fedora-coreos-config/pull/2544 - liveISOFromRAMKarg = "coreos.liveiso.fromram" + // defaultQemuHostIPv4 is documented in `man qemu-kvm`, under the `-netdev` option + defaultQemuHostIPv4 = "10.0.2.2" ) +// This object gets serialized to YAML and fed to coreos-installer: +// https://coreos.github.io/coreos-installer/customizing-install/#config-file-format +type coreosInstallerConfig struct { + ImageURL string `yaml:"image-url,omitempty"` + IgnitionFile string `yaml:"ignition-file,omitempty"` + Insecure bool `yaml:"insecure,omitempty"` + AppendKargs []string `yaml:"append-karg,omitempty"` + CopyNetwork bool `yaml:"copy-network,omitempty"` + DestDevice string `yaml:"dest-device,omitempty"` + Console []string `yaml:"console,omitempty"` +} + +// Sometimes the logs that stream from various virtio streams can be +// incomplete because they depend on services inside the guest. +// When you are debugging earlyboot/initramfs issues this can be +// problematic. Let's add a hook here to enable more debugging. +func renderCosaTestIsoDebugKargs() []string { + if _, ok := os.LookupEnv("COSA_TESTISO_DEBUG"); ok { + return []string{"systemd.log_color=0", "systemd.log_level=debug", + "systemd.journald.forward_to_console=1", + "systemd.journald.max_level_console=debug"} + } else { + return []string{} + } +} + +func absSymlink(src, dest string) error { + src, err := filepath.Abs(src) + if err != nil { + return err + } + return os.Symlink(src, dest) +} + +func setupMetalImage(builddir, metalimg, destdir string) (string, error) { + if err := absSymlink(filepath.Join(builddir, metalimg), filepath.Join(destdir, metalimg)); err != nil { + return "", err + } + return metalimg, nil +} + +// startHTTPServer starts an HTTP file server in a goroutine and returns the server. +// The caller is responsible for closing the server. +func startHTTPServer(listener net.Listener, dir string) *http.Server { + mux := http.NewServeMux() + mux.Handle("/", http.FileServer(http.Dir(dir))) + server := &http.Server{Handler: mux} + go func() { + server.Serve(listener) + }() + return server +} + type IsoTestOpts struct { // Flags().BoolVarP(&instInsecure, "inst-insecure", "S", false, "Do not verify signature on metal image") instInsecure bool @@ -310,7 +365,7 @@ OnFailureJobMode=isolate [Service] Type=oneshot RemainAfterExit=yes -ExecStart=/bin/sh -c '/usr/bin/echo %s >/dev/virtio-ports/testisocompletion && systemctl poweroff' +ExecStart=/bin/sh -c '/usr/bin/echo %s >/dev/virtio-ports/testisocompletion' [Install] RequiredBy=multi-user.target`, signalCompleteString) @@ -469,3 +524,31 @@ ExecStart=/usr/bin/nmcli c show br-ex RequiredBy=coreos-installer.target # for target system RequiredBy=multi-user.target`, nmConnectionId, nmConnectionFile) + +var bootStartedSignal = "boot-started-OK" +var bootStartedUnit = fmt.Sprintf(`[Unit] +Description=TestISO Boot Started +Requires=dev-virtio\\x2dports-bootstarted.device +OnFailure=emergency.target +OnFailureJobMode=isolate +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=/bin/sh -c '/usr/bin/echo %s >/dev/virtio-ports/bootstarted' +[Install] +RequiredBy=coreos-installer.target`, bootStartedSignal) + +var coreosInstallerMultipathUnit = `[Unit] +Description=TestISO Enable Multipath +Before=multipathd.service +DefaultDependencies=no +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=/usr/sbin/mpathconf --enable +[Install] +WantedBy=coreos-installer.target` + +var waitForMpathTargetConf = `[Unit] +Requires=dev-mapper-mpatha.device +After=dev-mapper-mpatha.device` diff --git a/mantle/kola/tests/iso/live-iso.go b/mantle/kola/tests/iso/live-iso.go index 29a414a4f6..a3b05f8559 100644 --- a/mantle/kola/tests/iso/live-iso.go +++ b/mantle/kola/tests/iso/live-iso.go @@ -3,17 +3,22 @@ package iso import ( _ "embed" "fmt" + "net" "os" + "os/exec" + "path/filepath" "strings" "time" "github.com/coreos/coreos-assembler/mantle/kola" "github.com/coreos/coreos-assembler/mantle/kola/cluster" "github.com/coreos/coreos-assembler/mantle/kola/register" + "github.com/coreos/coreos-assembler/mantle/platform" "github.com/coreos/coreos-assembler/mantle/platform/conf" "github.com/coreos/coreos-assembler/mantle/platform/machine/qemu" - "github.com/coreos/coreos-assembler/mantle/util" coreosarch "github.com/coreos/stream-metadata-go/arch" + "github.com/pkg/errors" + "gopkg.in/yaml.v2" ) var ( @@ -74,107 +79,324 @@ func getAllLiveIsoTests() []string { func init() { for _, testName := range getAllLiveIsoTests() { + tags := []string{} + if !strings.Contains(testName, "offline") { + tags = append(tags, kola.NeedsInternetTag) + } register.RegisterTest(®ister.Test{ Run: func(c cluster.TestCluster) { opts := getIsoTestOpts(testName) - isoLiveIso(c, opts) + runLiveIsoInstallTest(c, opts) }, ClusterSize: 0, Name: "iso." + testName, Description: "Verify ISO live install works.", Timeout: installTimeoutMins * time.Minute, + Tags: tags, Flags: []register.Flag{}, Platforms: []string{"qemu"}, }) } } -func isoLiveIso(c cluster.TestCluster, opts IsoTestOpts) { - var outdir string - var qc *qemu.Cluster - switch pc := c.Cluster.(type) { - case *qemu.Cluster: - outdir = pc.RuntimeConf().OutputDir - qc = pc - default: - c.Fatalf("Unsupported cluster type") +func runLiveIsoInstallTest(c cluster.TestCluster, opts IsoTestOpts) { + if opts.isMiniso && opts.isOffline { // ideally this'd be one enum parameter + c.Fatal("Can't run minimal install offline") } - - if kola.CosaBuild.Meta.BuildArtifacts.LiveIso == nil || kola.CosaBuild.Meta.BuildArtifacts.LiveKernel == nil { - c.Fatalf("build %s is missing live artifacts", kola.CosaBuild.Meta.Name) + if opts.isOffline && opts.addNmKeyfile { + c.Fatal("Cannot use `--add-nm-keyfile` with offline mode") } - inst := qemu.Install{ - CosaBuild: kola.CosaBuild, - NmKeyfiles: make(map[string]string), - Insecure: opts.instInsecure, - Native4k: opts.enable4k, - MultiPathDisk: opts.enableMultipath, + qc, ok := c.Cluster.(*qemu.Cluster) + if !ok { + c.Fatalf("Unsupported cluster type") } - tmpd, err := os.MkdirTemp("", "kola-iso.live") + tempdir, err := os.MkdirTemp("/var/tmp", "iso") if err != nil { c.Fatal(err) } - defer os.RemoveAll(tmpd) + defer func() { + os.RemoveAll(tempdir) + }() - sshPubKeyBuf, _, err := util.CreateSSHAuthorizedKey(tmpd) - if err != nil { + if err := runIsoTest(qc, opts, tempdir); err != nil { c.Fatal(err) } +} - builder, virtioJournalConfig, err := newQemuBuilderWithDisk(opts, outdir) +func runIsoTest(qc *qemu.Cluster, opts IsoTestOpts, tempdir string) error { + targetConfig, err := conf.EmptyIgnition().Render(conf.FailWarnings) if err != nil { - c.Fatal(err) + return err } - inst.Builder = builder - completionChannel, err := inst.Builder.VirtioChannelRead("testisocompletion") + keys, err := qc.Keys() if err != nil { - c.Fatal(err) + return err + } + + targetConfig.CopyKeys(keys) + targetConfig.AddSystemdUnit("coreos-test-installer.service", signalCompletionUnit, conf.Enable) + targetConfig.AddSystemdUnit("coreos-test-entered-emergency-target.service", signalFailureUnit, conf.Enable) + targetConfig.AddSystemdUnit("coreos-test-installer-no-ignition.service", checkNoIgnition, conf.Enable) + if opts.enableMultipath { + targetConfig.AddSystemdUnit("coreos-test-installer-multipathed.service", multipathedRoot, conf.Enable) + } + if opts.addNmKeyfile { + targetConfig.AddSystemdUnit("coreos-test-nm-keyfile.service", verifyNmKeyfile, conf.Enable) } - var isoKernelArgs []string - var keys []string - keys = append(keys, strings.TrimSpace(string(sshPubKeyBuf))) - virtioJournalConfig.AddAuthorizedKeys("core", keys) + isopath := filepath.Join(kola.CosaBuild.Dir, kola.CosaBuild.Meta.BuildArtifacts.LiveIso.Path) - liveConfig := *virtioJournalConfig + installerConfig := coreosInstallerConfig{ + IgnitionFile: "/var/opt/pointer.ign", + DestDevice: "/dev/vda", + AppendKargs: renderCosaTestIsoDebugKargs(), + Insecure: opts.instInsecure, + CopyNetwork: opts.addNmKeyfile, // force networking on in the initrd to verify the keyfile was used + } + + var serializedTargetConfig string + if opts.isOffline { + // note we leave ImageURL empty here; offline installs should now be the + // default! + + // we want to test that a full offline install works; that includes the + // final installed host booting offline + serializedTargetConfig = targetConfig.String() + } else { + listener, err := net.Listen("tcp", ":0") + if err != nil { + return err + } + port := listener.Addr().(*net.TCPAddr).Port + baseurl := fmt.Sprintf("http://%s:%d", defaultQemuHostIPv4, port) + + // This is subtle but: for the minimal case, while we need networking to fetch the + // rootfs, the primary install flow will still rely on osmet. So let's keep ImageURL + // empty to exercise that path. In the future, this could be a separate scenario + // (likely we should drop the "offline" naming and have a "remote" tag on the + // opposite scenarios instead which fetch the metal image, so then we'd have + // "[min]iso-install" and "[min]iso-remote-install"). + if opts.isMiniso { + isopath, err = createMiniso(tempdir, isopath, baseurl) + if err != nil { + return err + } + } else { + var metalimg string + if opts.enable4k { + metalimg = kola.CosaBuild.Meta.BuildArtifacts.Metal4KNative.Path + } else { + metalimg = kola.CosaBuild.Meta.BuildArtifacts.Metal.Path + } + metalname, err := setupMetalImage(kola.CosaBuild.Dir, metalimg, tempdir) + if err != nil { + return err + } + installerConfig.ImageURL = fmt.Sprintf("%s/%s", baseurl, metalname) + } + + if opts.addNmKeyfile { + nmKeyfiles := make(map[string]string) + nmKeyfiles[nmConnectionFile] = nmConnection + if err := embedNmkeyfiles(tempdir, nmKeyfiles, isopath); err != nil { + return err + } + } + + // In this case; the target config is jut a tiny wrapper that wants to + // fetch our hosted target.ign config + // TODO also use https://github.com/coreos/coreos-installer/issues/118#issuecomment-585572952 + // when it arrives + if err := targetConfig.WriteFile(filepath.Join(tempdir, "target.ign")); err != nil { + return err + } + // Create a new config that fetches the target config + pointerConfig, err := conf.EmptyIgnition().Render(conf.FailWarnings) + if err != nil { + return err + } + pointerConfig.AddConfigSource(baseurl + "/target.ign") + serializedTargetConfig = pointerConfig.String() + + server := startHTTPServer(listener, tempdir) + defer server.Close() + } + + // XXX: https://github.com/coreos/coreos-installer/issues/1171 + if coreosarch.CurrentRpmArch() != "s390x" { + installerConfig.Console = []string{platform.ConsoleKernelArgument[coreosarch.CurrentRpmArch()]} + } + if opts.enableMultipath { + // we only have one multipath device so it has to be that + installerConfig.DestDevice = "/dev/mapper/mpatha" + installerConfig.AppendKargs = append(installerConfig.AppendKargs, "rd.multipath=default", "root=/dev/disk/by-label/dm-mpath-root", "rw") + } + + installerConfigData, err := yaml.Marshal(installerConfig) + if err != nil { + return err + } + mode := 0644 + + liveConfig, err := conf.EmptyIgnition().Render(conf.FailWarnings) + if err != nil { + return err + } liveConfig.AddSystemdUnit("live-signal-ok.service", liveSignalOKUnit, conf.Enable) liveConfig.AddSystemdUnit("verify-no-efi-boot-entry.service", verifyNoEFIBootEntry, conf.Enable) liveConfig.AddSystemdUnit("iso-not-mounted-when-fromram.service", isoNotMountedUnit, conf.Enable) liveConfig.AddSystemdUnit("coreos-test-entered-emergency-target.service", signalFailureUnit, conf.Enable) volumeIdUnitContents := fmt.Sprintf(verifyIsoVolumeId, kola.CosaBuild.Meta.Name) liveConfig.AddSystemdUnit("verify-iso-volume-id.service", volumeIdUnitContents, conf.Enable) - - targetConfig := *virtioJournalConfig - targetConfig.AddSystemdUnit("coreos-test-installer.service", signalCompletionUnit, conf.Enable) - targetConfig.AddSystemdUnit("coreos-test-entered-emergency-target.service", signalFailureUnit, conf.Enable) - targetConfig.AddSystemdUnit("coreos-test-installer-no-ignition.service", checkNoIgnition, conf.Enable) - if inst.MultiPathDisk { - targetConfig.AddSystemdUnit("coreos-test-installer-multipathed.service", multipathedRoot, conf.Enable) + liveConfig.AddSystemdUnit("boot-started.service", bootStartedUnit, conf.Enable) + liveConfig.AddFile(installerConfig.IgnitionFile, serializedTargetConfig, mode) + liveConfig.AddFile("/etc/coreos/installer.d/mantle.yaml", string(installerConfigData), mode) + liveConfig.AddAutoLogin() + if opts.enableMultipath { + liveConfig.AddSystemdUnit("coreos-installer-multipath.service", coreosInstallerMultipathUnit, conf.Enable) + liveConfig.AddSystemdUnitDropin("coreos-installer.service", "wait-for-mpath-target.conf", waitForMpathTargetConf) } - if opts.addNmKeyfile { liveConfig.AddSystemdUnit("coreos-test-nm-keyfile.service", verifyNmKeyfile, conf.Enable) - targetConfig.AddSystemdUnit("coreos-test-nm-keyfile.service", verifyNmKeyfile, conf.Enable) - // NM keyfile via `iso network embed` - inst.NmKeyfiles[nmConnectionFile] = nmConnection // nmstate config via live Ignition config, propagated via // --copy-network, which is enabled by inst.NmKeyfiles liveConfig.AddFile(nmstateConfigFile, nmstateConfig, 0644) } + setupNet := func(o platform.MachineOptions, builder *platform.QemuBuilder) error { + if !opts.isOffline { + // also save pointer config into the output dir for debugging + path := filepath.Join(qc.RuntimeConf().OutputDir, builder.UUID, "config-target-pointer.ign") + if err := targetConfig.WriteFile(path); err != nil { + return err + } + } + // for basic network with ssh access + return qc.SetupDefaultNetwork(o, builder) + } + + errchan := make(chan error) + var bootStartedOutput *os.File + setupDisks := func(_ platform.MachineOptions, builder *platform.QemuBuilder) error { + sectorSize := 0 + if opts.enable4k { + sectorSize = 4096 + } + disk := platform.Disk{ + Size: "12G", // Arbitrary + SectorSize: sectorSize, + MultiPathDisk: opts.enableMultipath, + } + //TBD: see if we can remove this and just use AddDisk and inject bootindex during startup + if coreosarch.CurrentRpmArch() == "s390x" || coreosarch.CurrentRpmArch() == "aarch64" { + // s390x and aarch64 need to use bootindex as they don't support boot once + if err := builder.AddDisk(&disk); err != nil { + return err + } + } else { + if err := builder.AddPrimaryDisk(&disk); err != nil { + return err + } + } + isoCompletionOutput, err := builder.VirtioChannelRead("testisocompletion") + if err != nil { + return errors.Wrap(err, "setting up testisocompletion virtio-serial channel") + } + go func() { + errchan <- CheckTestOutput(isoCompletionOutput, []string{liveOKSignal, signalCompleteString}) + }() + + bootStartedOutput, err = builder.VirtioChannelRead("bootstarted") + if err != nil { + return errors.Wrap(err, "setting up bootstarted virtio-serial channel") + } + + return builder.AddIso(isopath, "bootindex=3", false) + } + kargs := renderCosaTestIsoDebugKargs() if opts.isISOFromRAM { - isoKernelArgs = append(isoKernelArgs, liveISOFromRAMKarg) + // https://github.com/coreos/fedora-coreos-config/pull/2544 + kargs = append(kargs, "coreos.liveiso.fromram") + } + if opts.addNmKeyfile { + kargs = append(kargs, "rd.neednet=1") } - mach, err := inst.InstallViaISOEmbed(isoKernelArgs, liveConfig, targetConfig, outdir, opts.isOffline, opts.isMiniso) - if err != nil { - c.Fatal(err) + options := platform.MachineOptions{ + MinMemory: 4096, + MultiPathDisk: opts.enableMultipath, + AppendKernelArgs: strings.Join(kargs, " "), } - qc.AddMach(mach) - err = awaitCompletion(c, mach.Instance(), opts.console, outdir, completionChannel, mach.BootStartedErrorChannel(), []string{liveOKSignal, signalCompleteString}) + if opts.enableUefi { + options.Firmware = "uefi" + } + + machineBuilder := &qemu.MachineBuilder{ + SetupDisks: setupDisks, + SetupNetwork: setupNet, + } + + qm, err := qc.NewMachineWithBuilder(liveConfig, options, machineBuilder) if err != nil { - c.Fatal(err) + return errors.Wrap(err, "unable to create test machine") + } + + inst := qc.Instance(qm) + if inst == nil { + return errors.New("failed to get QemuInstance from machine") + } + + //check for error when switching boot order + go func() { + if err := CheckTestOutput(bootStartedOutput, []string{bootStartedSignal}); err != nil { + errchan <- err + return + } + if err := inst.SwitchBootOrder(); err != nil { + errchan <- errors.Wrapf(err, "switching boot order failed") + return + } + }() + + return <-errchan +} + +func createMiniso(tempd string, isopath string, url string) (string, error) { + minisopath := filepath.Join(tempd, "minimal.iso") + // This is obviously also available in the build dir, but to be realistic, + // let's take it from --rootfs-output + rootfs_path := filepath.Join(tempd, "rootfs.img") + // Ideally we'd use the coreos-installer of the target build here, because it's part + // of the test workflow, but that's complex... Sadly, probably easiest is to spin up + // a VM just to get the minimal ISO. + cmd := exec.Command("coreos-installer", "iso", "extract", "minimal-iso", isopath, + minisopath, "--output-rootfs", rootfs_path, "--rootfs-url", url+"/rootfs.img") + cmd.Stderr = os.Stderr + if err := cmd.Run(); err != nil { + return "", errors.Wrapf(err, "running coreos-installer iso extract minimal") + } + return minisopath, nil +} + +func embedNmkeyfiles(tempd string, nmKeyfiles map[string]string, isopath string) error { + var keyfileArgs []string + for nmName, nmContents := range nmKeyfiles { + path := filepath.Join(tempd, nmName) + if err := os.WriteFile(path, []byte(nmContents), 0600); err != nil { + return err + } + keyfileArgs = append(keyfileArgs, "--keyfile", path) + } + if len(keyfileArgs) > 0 { + args := []string{"iso", "network", "embed", isopath} + args = append(args, keyfileArgs...) + cmd := exec.Command("coreos-installer", args...) + cmd.Stderr = os.Stderr + if err := cmd.Run(); err != nil { + return errors.Wrapf(err, "running coreos-installer iso network embed") + } } + return nil } diff --git a/mantle/platform/machine/qemu/metal.go b/mantle/platform/machine/qemu/metal.go index d49e489bae..d4730bfe87 100644 --- a/mantle/platform/machine/qemu/metal.go +++ b/mantle/platform/machine/qemu/metal.go @@ -574,264 +574,3 @@ type installerConfig struct { DestDevice string `yaml:"dest-device,omitempty"` Console []string `yaml:"console,omitempty"` } - -func (inst *Install) InstallViaISOEmbed(kargs []string, liveIgnition, targetIgnition conf.Conf, outdir string, offline, minimal bool) (*machine, error) { - artifacts := []string{"live-iso"} - if !offline { - if inst.Native4k { - artifacts = append(artifacts, "metal4k") - } else { - artifacts = append(artifacts, "metal") - } - } - if err := inst.checkArtifactsExist(artifacts); err != nil { - return nil, err - } - if minimal && offline { // ideally this'd be one enum parameter - panic("Can't run minimal install offline") - } - if offline && len(inst.NmKeyfiles) > 0 { - return nil, fmt.Errorf("Cannot use `--add-nm-keyfile` with offline mode") - } - - installerConfig := installerConfig{ - IgnitionFile: "/var/opt/pointer.ign", - DestDevice: "/dev/vda", - AppendKargs: renderCosaTestIsoDebugKargs(), - } - - // XXX: https://github.com/coreos/coreos-installer/issues/1171 - if coreosarch.CurrentRpmArch() != "s390x" { - installerConfig.Console = []string{platform.ConsoleKernelArgument[coreosarch.CurrentRpmArch()]} - } - - if inst.MultiPathDisk { - // we only have one multipath device so it has to be that - installerConfig.DestDevice = "/dev/mapper/mpatha" - installerConfig.AppendKargs = append(installerConfig.AppendKargs, "rd.multipath=default", "root=/dev/disk/by-label/dm-mpath-root", "rw") - } - - inst.kargs = append(renderCosaTestIsoDebugKargs(), kargs...) - inst.ignition = targetIgnition - inst.liveIgnition = liveIgnition - - tempdir, err := os.MkdirTemp("/var/tmp", "mantle-metal") - if err != nil { - return nil, err - } - cleanupTempdir := true - defer func() { - if cleanupTempdir { - os.RemoveAll(tempdir) - } - }() - - if err := inst.ignition.WriteFile(filepath.Join(tempdir, "target.ign")); err != nil { - return nil, err - } - // and write it once more in the output dir for debugging - if err := inst.ignition.WriteFile(filepath.Join(outdir, "config-target.ign")); err != nil { - return nil, err - } - - builddir := inst.CosaBuild.Dir - srcisopath := filepath.Join(builddir, inst.CosaBuild.Meta.BuildArtifacts.LiveIso.Path) - - // Copy the ISO to a new location for modification. - // This is a bit awkward; we copy here, but QemuBuilder will also copy - // again (in `setupIso()`). I didn't want to lower the NM keyfile stuff - // into QemuBuilder. And plus, both tempdirs should be in /var/tmp so - // the `cp --reflink=auto` that QemuBuilder does should just reflink. - newIso := filepath.Join(tempdir, "install.iso") - cmd := exec.Command("cp", "--reflink=auto", srcisopath, newIso) - cmd.Stderr = os.Stderr - if err := cmd.Run(); err != nil { - return nil, errors.Wrapf(err, "copying iso") - } - // Make it writable so we can modify it - if err := os.Chmod(newIso, 0644); err != nil { - return nil, errors.Wrapf(err, "setting permissions on iso") - } - srcisopath = newIso - - var serializedTargetConfig string - if offline { - // note we leave ImageURL empty here; offline installs should now be the - // default! - - // we want to test that a full offline install works; that includes the - // final installed host booting offline - serializedTargetConfig = inst.ignition.String() - } else { - var metalimg string - if inst.Native4k { - metalimg = inst.CosaBuild.Meta.BuildArtifacts.Metal4KNative.Path - } else { - metalimg = inst.CosaBuild.Meta.BuildArtifacts.Metal.Path - } - metalname, err := setupMetalImage(builddir, metalimg, tempdir) - if err != nil { - return nil, errors.Wrapf(err, "setting up metal image") - } - - mux := http.NewServeMux() - mux.Handle("/", http.FileServer(http.Dir(tempdir))) - listener, err := net.Listen("tcp", ":0") - if err != nil { - return nil, err - } - port := listener.Addr().(*net.TCPAddr).Port - //nolint // Yeah this leaks - go func() { - http.Serve(listener, mux) - }() - baseurl := fmt.Sprintf("http://%s:%d", defaultQemuHostIPv4, port) - - // This is subtle but: for the minimal case, while we need networking to fetch the - // rootfs, the primary install flow will still rely on osmet. So let's keep ImageURL - // empty to exercise that path. In the future, this could be a separate scenario - // (likely we should drop the "offline" naming and have a "remote" tag on the - // opposite scenarios instead which fetch the metal image, so then we'd have - // "[min]iso-install" and "[min]iso-remote-install"). - if !minimal { - installerConfig.ImageURL = fmt.Sprintf("%s/%s", baseurl, metalname) - } - - if minimal { - minisopath := filepath.Join(tempdir, "minimal.iso") - // This is obviously also available in the build dir, but to be realistic, - // let's take it from --rootfs-output - rootfs_path := filepath.Join(tempdir, "rootfs.img") - // Ideally we'd use the coreos-installer of the target build here, because it's part - // of the test workflow, but that's complex... Sadly, probably easiest is to spin up - // a VM just to get the minimal ISO. - cmd := exec.Command("coreos-installer", "iso", "extract", "minimal-iso", srcisopath, - minisopath, "--output-rootfs", rootfs_path, "--rootfs-url", baseurl+"/rootfs.img") - cmd.Stderr = os.Stderr - if err := cmd.Run(); err != nil { - return nil, errors.Wrapf(err, "running coreos-installer iso extract minimal") - } - srcisopath = minisopath - } - - // In this case; the target config is jut a tiny wrapper that wants to - // fetch our hosted target.ign config - - // TODO also use https://github.com/coreos/coreos-installer/issues/118#issuecomment-585572952 - // when it arrives - targetConfig, err := conf.EmptyIgnition().Render(conf.FailWarnings) - if err != nil { - return nil, err - } - targetConfig.AddConfigSource(baseurl + "/target.ign") - serializedTargetConfig = targetConfig.String() - - // also save pointer config into the output dir for debugging - if err := targetConfig.WriteFile(filepath.Join(outdir, "config-target-pointer.ign")); err != nil { - return nil, err - } - } - - var keyfileArgs []string - for nmName, nmContents := range inst.NmKeyfiles { - path := filepath.Join(tempdir, nmName) - if err := os.WriteFile(path, []byte(nmContents), 0600); err != nil { - return nil, err - } - keyfileArgs = append(keyfileArgs, "--keyfile", path) - } - if len(keyfileArgs) > 0 { - - args := []string{"iso", "network", "embed", srcisopath} - args = append(args, keyfileArgs...) - cmd = exec.Command("coreos-installer", args...) - cmd.Stderr = os.Stderr - if err := cmd.Run(); err != nil { - return nil, errors.Wrapf(err, "running coreos-installer iso network embed") - } - - installerConfig.CopyNetwork = true - - // force networking on in the initrd to verify the keyfile was used - inst.kargs = append(inst.kargs, "rd.neednet=1") - } - - if len(inst.kargs) > 0 { - args := []string{"iso", "kargs", "modify", srcisopath} - for _, karg := range inst.kargs { - args = append(args, "--append", karg) - } - cmd = exec.Command("coreos-installer", args...) - cmd.Stderr = os.Stderr - if err := cmd.Run(); err != nil { - return nil, errors.Wrapf(err, "running coreos-installer iso kargs") - } - } - - if inst.Insecure { - installerConfig.Insecure = true - } - - installerConfigData, err := yaml.Marshal(installerConfig) - if err != nil { - return nil, err - } - mode := 0644 - - inst.liveIgnition.AddSystemdUnit("boot-started.service", bootStartedUnit, conf.Enable) - inst.liveIgnition.AddFile(installerConfig.IgnitionFile, serializedTargetConfig, mode) - inst.liveIgnition.AddFile("/etc/coreos/installer.d/mantle.yaml", string(installerConfigData), mode) - inst.liveIgnition.AddAutoLogin() - - if inst.MultiPathDisk { - inst.liveIgnition.AddSystemdUnit("coreos-installer-multipath.service", `[Unit] -Description=TestISO Enable Multipath -Before=multipathd.service -DefaultDependencies=no -[Service] -Type=oneshot -RemainAfterExit=yes -ExecStart=/usr/sbin/mpathconf --enable -[Install] -WantedBy=coreos-installer.target`, conf.Enable) - inst.liveIgnition.AddSystemdUnitDropin("coreos-installer.service", "wait-for-mpath-target.conf", `[Unit] -Requires=dev-mapper-mpatha.device -After=dev-mapper-mpatha.device`) - } - - qemubuilder := inst.Builder - bootStartedChan, err := qemubuilder.VirtioChannelRead("bootstarted") - if err != nil { - return nil, err - } - - qemubuilder.SetConfig(&inst.liveIgnition) - - // also save live config into the output dir for debugging - liveConfigPath := filepath.Join(outdir, "config-live.ign") - if err := inst.liveIgnition.WriteFile(liveConfigPath); err != nil { - return nil, err - } - - if err := qemubuilder.AddIso(srcisopath, "bootindex=3", false); err != nil { - return nil, err - } - - // With the recent change to use qemu -nodefaults (bc68d7c) we need to - // request network. Otherwise we get no network devices. - if !offline { - qemubuilder.UsermodeNetworking = true - } - - qinst, err := qemubuilder.Exec() - if err != nil { - return nil, err - } - cleanupTempdir = false // Transfer ownership - instmachine := machine{ - inst: qinst, - tempdir: tempdir, - } - switchBootOrderSignal(qinst, bootStartedChan, &instmachine.bootStartedErrorChannel) - return &instmachine, nil -} From 0b530b3a0b4501eeb609ce57bb085d6fb27783c6 Mon Sep 17 00:00:00 2001 From: Nikita Dubrovskii Date: Thu, 2 Apr 2026 17:06:19 +0200 Subject: [PATCH 21/31] mantle/platform: Add IgnitionPath() helper to QemuBuilder Extract the Ignition config path logic into a reusable IgnitionPath() method that returns the path where the Ignition config will be written. This allows callers to get the Ignition config path before rendering, which is useful for tests that need to create symlinks or perform other operations on the config file path before the machine is started. --- mantle/platform/qemu.go | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/mantle/platform/qemu.go b/mantle/platform/qemu.go index 91f519db1a..3cb172e045 100644 --- a/mantle/platform/qemu.go +++ b/mantle/platform/qemu.go @@ -612,6 +612,14 @@ func (builder *QemuBuilder) TempFile(pattern string) (*os.File, error) { return os.CreateTemp(builder.tempdir, pattern) } +// Returns the path where the Ignition config will be written. +func (builder *QemuBuilder) IgnitionPath() (string, error) { + if err := builder.ensureTempdir(); err != nil { + return "", err + } + return filepath.Join(builder.tempdir, "config.ign"), nil +} + // renderIgnition lazily renders a parsed config if one is set func (builder *QemuBuilder) renderIgnition() error { if !builder.ignitionSet || builder.ignitionRendered { @@ -621,10 +629,11 @@ func (builder *QemuBuilder) renderIgnition() error { panic("Both ConfigFile and ignition set") } - if err := builder.ensureTempdir(); err != nil { + var err error + builder.ConfigFile, err = builder.IgnitionPath() + if err != nil { return err } - builder.ConfigFile = filepath.Join(builder.tempdir, "config.ign") if err := builder.ignition.WriteFile(builder.ConfigFile); err != nil { return err } From a4f85b70d59c81076e76c1fe669732d86c40234e Mon Sep 17 00:00:00 2001 From: Nikita Dubrovskii Date: Thu, 2 Apr 2026 17:14:38 +0200 Subject: [PATCH 22/31] mantle/platform: Allow passing nil instead of Ignition config Allow callers to pass nil userdata/config when Ignition configuration is not needed, such as in PXE boot scenarios where configuration is provided through alternative means. --- mantle/platform/machine/qemu/cluster.go | 3 +++ mantle/platform/qemu.go | 3 +++ 2 files changed, 6 insertions(+) diff --git a/mantle/platform/machine/qemu/cluster.go b/mantle/platform/machine/qemu/cluster.go index 72d4b702e3..f25383739a 100644 --- a/mantle/platform/machine/qemu/cluster.go +++ b/mantle/platform/machine/qemu/cluster.go @@ -165,6 +165,9 @@ func (qc *Cluster) Destroy() { } func (qc *Cluster) RenderUserDataIfNeeded(userdata any) (*conf.Conf, error) { + if userdata == nil { + return nil, nil + } var config *conf.Conf var err error // Some callers provide the config directly rather than something diff --git a/mantle/platform/qemu.go b/mantle/platform/qemu.go index 3cb172e045..ed10240611 100644 --- a/mantle/platform/qemu.go +++ b/mantle/platform/qemu.go @@ -593,6 +593,9 @@ func (builder *QemuBuilder) ensureTempdir() error { // SetConfig injects Ignition; this can be used in place of ConfigFile. func (builder *QemuBuilder) SetConfig(config *conf.Conf) { + if config == nil { + return + } if builder.ignitionRendered { panic("SetConfig called after config rendered") } From eba33e3c78916f8457fcce1da41f28198d8339b4 Mon Sep 17 00:00:00 2001 From: Nikita Dubrovskii Date: Fri, 28 Nov 2025 17:18:59 +0100 Subject: [PATCH 23/31] kola/tests/iso: refactor *pxe* tests --- mantle/kola/tests/iso/common.go | 180 +------- mantle/kola/tests/iso/live-pxe.go | 429 +++++++++++++++++-- mantle/platform/machine/qemu/metal.go | 576 -------------------------- 3 files changed, 400 insertions(+), 785 deletions(-) delete mode 100644 mantle/platform/machine/qemu/metal.go diff --git a/mantle/kola/tests/iso/common.go b/mantle/kola/tests/iso/common.go index 3c1a082f87..2fb0168dbe 100644 --- a/mantle/kola/tests/iso/common.go +++ b/mantle/kola/tests/iso/common.go @@ -8,16 +8,10 @@ import ( "net/http" "os" "path/filepath" - "strconv" "strings" "time" "github.com/coreos/coreos-assembler/mantle/kola" - "github.com/coreos/coreos-assembler/mantle/kola/cluster" - "github.com/coreos/coreos-assembler/mantle/platform" - "github.com/coreos/coreos-assembler/mantle/platform/conf" - "github.com/coreos/coreos-assembler/mantle/platform/machine/qemu" - coreosarch "github.com/coreos/stream-metadata-go/arch" "github.com/pkg/errors" ) @@ -61,6 +55,7 @@ func absSymlink(src, dest string) error { return os.Symlink(src, dest) } +// setupMetalImage creates a symlink to the metal image. func setupMetalImage(builddir, metalimg, destdir string) (string, error) { if err := absSymlink(filepath.Join(builddir, metalimg), filepath.Join(destdir, metalimg)); err != nil { return "", err @@ -149,116 +144,6 @@ func IsDevBuild() bool { return false } -func newBaseQemuBuilder(opts IsoTestOpts, outdir string) (*platform.QemuBuilder, error) { - builder := qemu.NewMetalQemuBuilderDefault() - if opts.enableUefiSecure { - builder.Firmware = "uefi-secure" - } else if opts.enableUefi { - builder.Firmware = "uefi" - } - - if err := os.MkdirAll(outdir, 0755); err != nil { - return nil, err - } - - builder.InheritConsole = opts.console - if !opts.console { - builder.ConsoleFile = filepath.Join(outdir, "console.txt") - } - - if kola.QEMUOptions.Memory != "" { - parsedMem, err := strconv.ParseInt(kola.QEMUOptions.Memory, 10, 32) - if err != nil { - return nil, err - } - builder.MemoryMiB = int(parsedMem) - } - - // increase the memory for pxe tests with appended rootfs in the initrd - // we were bumping up into the 4GiB limit in RHCOS/c9s - // pxe-offline-install.rootfs-appended.bios tests - if opts.pxeAppendRootfs && builder.MemoryMiB < 5120 { - builder.MemoryMiB = 5120 - } - - return builder, nil -} - -func newQemuBuilder(opts IsoTestOpts, outdir string) (*platform.QemuBuilder, *conf.Conf, error) { - builder, err := newBaseQemuBuilder(opts, outdir) - if err != nil { - return nil, nil, err - } - - config, err := conf.EmptyIgnition().Render(conf.FailWarnings) - if err != nil { - return nil, nil, err - } - - err = forwardJournal(outdir, builder, config) - if err != nil { - return nil, nil, err - } - - return builder, config, nil -} - -func forwardJournal(outdir string, builder *platform.QemuBuilder, config *conf.Conf) error { - journalPipe, err := builder.VirtioJournal(config, "") - if err != nil { - return err - } - journalOut, err := os.OpenFile(filepath.Join(outdir, "journal.txt"), os.O_WRONLY|os.O_CREATE, 0644) - if err != nil { - return err - } - - go func() { - _, err := io.Copy(journalOut, journalPipe) - if err != nil && err != io.EOF { - panic(err) - } - }() - - return nil -} - -func newQemuBuilderWithDisk(opts IsoTestOpts, outdir string) (*platform.QemuBuilder, *conf.Conf, error) { - builder, config, err := newQemuBuilder(opts, outdir) - - if err != nil { - return nil, nil, err - } - - sectorSize := 0 - if opts.enable4k { - sectorSize = 4096 - } - - disk := platform.Disk{ - Size: "12G", // Arbitrary - SectorSize: sectorSize, - MultiPathDisk: opts.enableMultipath, - } - - //TBD: see if we can remove this and just use AddDisk and inject bootindex during startup - if coreosarch.CurrentRpmArch() == "s390x" || coreosarch.CurrentRpmArch() == "aarch64" { - // s390x and aarch64 need to use bootindex as they don't support boot once - if err := builder.AddDisk(&disk); err != nil { - return nil, nil, err - } - } else { - if err := builder.AddPrimaryDisk(&disk); err != nil { - return nil, nil, err - } - } - - return builder, config, nil -} - -// Reads from a virtio channel and validates that the expected -// strings are received in order. Returns an error if EOF is encountered, a read -// error occurs, or an unexpected string is received. func CheckTestOutput(output *os.File, expected []string) error { reader := bufio.NewReader(output) for _, exp := range expected { @@ -282,59 +167,24 @@ func CheckTestOutput(output *os.File, expected []string) error { return nil } -func awaitCompletion(c cluster.TestCluster, inst *platform.QemuInstance, console bool, outdir string, qchan *os.File, booterrchan chan error, expected []string) error { - ctx := c.Context() - - errchan := make(chan error) - go func() { - timeout := (time.Duration(installTimeoutMins*(100+kola.Options.ExtendTimeoutPercent)) * time.Minute) / 100 - time.Sleep(timeout) - errchan <- fmt.Errorf("timed out after %v", timeout) - }() - if !console { - go func() { - errBuf, err := inst.WaitIgnitionError(ctx) - if err == nil { - if errBuf != "" { - c.Logf("entered emergency.target in initramfs") - path := filepath.Join(outdir, "ignition-virtio-dump.txt") - if err := os.WriteFile(path, []byte(errBuf), 0644); err != nil { - c.Errorf("Failed to write journal: %v", err) - } - err = platform.ErrInitramfsEmergency - } - } - if err != nil { - errchan <- err - } - }() +func cat(outfile string, infiles ...string) error { + out, err := os.OpenFile(outfile, os.O_WRONLY|os.O_CREATE, 0644) + if err != nil { + return err } - go func() { - err := inst.Wait() - // only one Wait() gets process data, so also manually check for signal - //plog.Debugf("qemu exited err=%v", err) - if err == nil && inst.Signaled() { - err = errors.New("process killed") - } + defer out.Close() + for _, infile := range infiles { + in, err := os.Open(infile) if err != nil { - errchan <- errors.Wrapf(err, "QEMU unexpectedly exited while awaiting completion") + return err } - time.Sleep(1 * time.Minute) - errchan <- fmt.Errorf("QEMU exited; timed out waiting for completion") - }() - go func() { - errchan <- CheckTestOutput(qchan, expected) - }() - go func() { - //check for error when switching boot order - if booterrchan != nil { - if err := <-booterrchan; err != nil { - errchan <- err - } + defer in.Close() + _, err = io.Copy(out, in) + if err != nil { + return err } - }() - err := <-errchan - return err + } + return nil } var liveOKSignal = "live-test-OK" diff --git a/mantle/kola/tests/iso/live-pxe.go b/mantle/kola/tests/iso/live-pxe.go index 086e0df735..a5fdfcc354 100644 --- a/mantle/kola/tests/iso/live-pxe.go +++ b/mantle/kola/tests/iso/live-pxe.go @@ -2,17 +2,23 @@ package iso import ( "fmt" + "net" + "net/http" "os" + "os/exec" + "path/filepath" "strings" "time" "github.com/coreos/coreos-assembler/mantle/kola" "github.com/coreos/coreos-assembler/mantle/kola/cluster" "github.com/coreos/coreos-assembler/mantle/kola/register" + "github.com/coreos/coreos-assembler/mantle/platform" "github.com/coreos/coreos-assembler/mantle/platform/conf" "github.com/coreos/coreos-assembler/mantle/platform/machine/qemu" - "github.com/coreos/coreos-assembler/mantle/util" coreosarch "github.com/coreos/stream-metadata-go/arch" + "github.com/pkg/errors" + "gopkg.in/yaml.v2" ) var ( @@ -88,85 +94,420 @@ RequiredBy=coreos-installer.target ` func testPXE(c cluster.TestCluster, opts IsoTestOpts) { - var outdir string - var qc *qemu.Cluster - - switch pc := c.Cluster.(type) { - case *qemu.Cluster: - outdir = pc.RuntimeConf().OutputDir - qc = pc - default: + qc, ok := c.Cluster.(*qemu.Cluster) + if !ok { c.Fatalf("Unsupported cluster type") } if opts.addNmKeyfile { - c.Fatal("--add-nm-keyfile not yet supported for PXE") + c.Fatalf("--add-nm-keyfile not yet supported for PXE") } - inst := qemu.Install{ - CosaBuild: kola.CosaBuild, - NmKeyfiles: make(map[string]string), - Insecure: opts.instInsecure, - Native4k: opts.enable4k, - MultiPathDisk: opts.enableMultipath, - PxeAppendRootfs: opts.pxeAppendRootfs, + targetConfig, err := conf.EmptyIgnition().Render(conf.FailWarnings) + if err != nil { + c.Fatal(err) } - - tmpd, err := os.MkdirTemp("", "kola-iso.pxe") + keys, err := qc.Keys() if err != nil { c.Fatal(err) } - defer os.RemoveAll(tmpd) + targetConfig.CopyKeys(keys) + targetConfig.AddSystemdUnit("coreos-test-installer.service", signalCompletionUnit, conf.Enable) + targetConfig.AddSystemdUnit("coreos-test-entered-emergency-target.service", signalFailureUnit, conf.Enable) + targetConfig.AddSystemdUnit("coreos-test-installer-no-ignition.service", checkNoIgnition, conf.Enable) - sshPubKeyBuf, _, err := util.CreateSSHAuthorizedKey(tmpd) + tempdir, err := os.MkdirTemp("/var/tmp", "mantle-pxe") if err != nil { c.Fatal(err) } + defer os.RemoveAll(tempdir) - builder, virtioJournalConfig, err := newQemuBuilderWithDisk(opts, outdir) + pxe, server, err := createPXE(tempdir, opts) if err != nil { - c.Fatal(err) + c.Fatal(errors.Wrapf(err, "setting up install")) + } + defer server.Close() + + initBuilder := func(o platform.MachineOptions, builder *platform.QemuBuilder) error { + if err := qc.InitDefaultBuilder(o, builder); err != nil { + return err + } + // save PXE config + ignitionPath, err := builder.IgnitionPath() + if err != nil { + return err + } + liveConfig, err := getPXEConfig(opts.instInsecure, opts.isOffline) + if err != nil { + return err + } + if err := liveConfig.WriteFile(ignitionPath); err != nil { + return err + } + if err := absSymlink(ignitionPath, filepath.Join(pxe.tftpdir, "pxe-live.ign")); err != nil { + return err + } + // save target config + targetpath := filepath.Join(filepath.Dir(ignitionPath), "pxe-target.ign") + if err := targetConfig.WriteFile(targetpath); err != nil { + return err + } + if err := absSymlink(targetpath, filepath.Join(pxe.tftpdir, "pxe-target.ign")); err != nil { + return err + } + return nil } + setupNet := func(o platform.MachineOptions, builder *platform.QemuBuilder) error { + netdev := fmt.Sprintf("%s,netdev=mynet0,mac=52:54:00:12:34:56", pxe.networkdevice) + if pxe.bootindex == "" { + builder.Append("-boot", "once=n") + } else { + netdev += fmt.Sprintf(",bootindex=%s", pxe.bootindex) + } + builder.Append("-device", netdev) + usernetdev := fmt.Sprintf("user,id=mynet0,tftp=%s,bootfile=%s", pxe.tftpdir, pxe.bootfile) + if pxe.tftpipaddr != "10.0.2.2" { + usernetdev += ",net=192.168.76.0/24,dhcpstart=192.168.76.9" + } + builder.Append("-netdev", usernetdev) + // for SSH access + return qc.SetupDefaultNetwork(o, builder) + } + + errchan := make(chan error) + var bootStartedOutput *os.File + setupDisks := func(_ platform.MachineOptions, builder *platform.QemuBuilder) error { + sectorSize := 0 + if opts.enable4k { + sectorSize = 4096 + } + disk := platform.Disk{ + Size: "12G", // Arbitrary + SectorSize: sectorSize, + } + //TBD: see if we can remove this and just use AddDisk and inject bootindex during startup + if coreosarch.CurrentRpmArch() == "s390x" || coreosarch.CurrentRpmArch() == "aarch64" { + // s390x and aarch64 need to use bootindex as they don't support boot once + if err := builder.AddDisk(&disk); err != nil { + return err + } + } else { + if err := builder.AddPrimaryDisk(&disk); err != nil { + return err + } + } + isoCompletionOutput, err := builder.VirtioChannelRead("testisocompletion") + if err != nil { + return errors.Wrap(err, "setting up testisocompletion virtio-serial channel") + } + go func() { + errchan <- CheckTestOutput(isoCompletionOutput, []string{liveOKSignal, signalCompleteString}) + }() + + bootStartedOutput, err = builder.VirtioChannelRead("bootstarted") + if err != nil { + return errors.Wrap(err, "setting up bootstarted virtio-serial channel") + } + return nil + } + + options := platform.MachineOptions{ + MinMemory: 4096, + } + if opts.enableUefi { + options.Firmware = "uefi" + } // increase the memory for pxe tests with appended rootfs in the initrd // we were bumping up into the 4GiB limit in RHCOS/c9s - // pxe-offline-install.rootfs-appended.bios tests - if inst.PxeAppendRootfs && builder.MemoryMiB < 5120 { - builder.MemoryMiB = 5120 + if opts.pxeAppendRootfs { + options.MinMemory = 5120 } - inst.Builder = builder - completionChannel, err := inst.Builder.VirtioChannelRead("testisocompletion") + builder := &qemu.MachineBuilder{ + InitBuilder: initBuilder, + SetupDisks: setupDisks, + SetupNetwork: setupNet, + } + qm, err := qc.NewMachineWithBuilder(nil, options, builder) if err != nil { - c.Fatal(err) // , "setting up virtio-serial channel") + c.Fatal(errors.Wrap(err, "unable to create test machine")) + } + inst := qc.Instance(qm) + if inst == nil { + c.Fatalf("Failed to get QemuInstance from machine") + } + + //check for error when switching boot order + go func() { + if err := CheckTestOutput(bootStartedOutput, []string{bootStartedSignal}); err != nil { + errchan <- err + return + } + if err := inst.SwitchBootOrder(); err != nil { + errchan <- errors.Wrapf(err, "switching boot order failed") + return + } + }() + + if err := <-errchan; err != nil { + c.Fatal(err) } +} - var keys []string - keys = append(keys, strings.TrimSpace(string(sshPubKeyBuf))) - virtioJournalConfig.AddAuthorizedKeys("core", keys) +func getPXEConfig(insecure bool, offline bool) (*conf.Conf, error) { + installerConfig := coreosInstallerConfig{ + Console: []string{platform.ConsoleKernelArgument[coreosarch.CurrentRpmArch()]}, + AppendKargs: renderCosaTestIsoDebugKargs(), + Insecure: insecure, + } + installerConfigData, err := yaml.Marshal(installerConfig) + if err != nil { + return nil, err + } + mode := 0644 - liveConfig := *virtioJournalConfig + liveConfig, err := conf.EmptyIgnition().Render(conf.FailWarnings) + if err != nil { + return nil, err + } liveConfig.AddSystemdUnit("live-signal-ok.service", liveSignalOKUnit, conf.Enable) liveConfig.AddSystemdUnit("coreos-test-entered-emergency-target.service", signalFailureUnit, conf.Enable) - - if opts.isOffline { + if offline { contents := fmt.Sprintf(downloadCheck, kola.CosaBuild.Meta.OstreeVersion) liveConfig.AddSystemdUnit("coreos-installer-offline-check.service", contents, conf.Enable) } + // XXX: https://github.com/coreos/coreos-installer/issues/1171 + if coreosarch.CurrentRpmArch() != "s390x" { + liveConfig.AddFile("/etc/coreos/installer.d/mantle.yaml", string(installerConfigData), mode) + } + liveConfig.AddAutoLogin() + liveConfig.AddSystemdUnit("boot-started.service", bootStartedUnit, conf.Enable) + return liveConfig, nil +} - targetConfig := *virtioJournalConfig - targetConfig.AddSystemdUnit("coreos-test-installer.service", signalCompletionUnit, conf.Enable) - targetConfig.AddSystemdUnit("coreos-test-entered-emergency-target.service", signalFailureUnit, conf.Enable) - targetConfig.AddSystemdUnit("coreos-test-installer-no-ignition.service", checkNoIgnition, conf.Enable) +type PXE struct { + tftpdir string + tftpipaddr string + boottype string + networkdevice string + bootindex string + pxeimagepath string + bootfile string +} + +func createPXE(tempdir string, opts IsoTestOpts) (*PXE, *http.Server, error) { + kernel := kola.CosaBuild.Meta.BuildArtifacts.LiveKernel.Path + initramfs := kola.CosaBuild.Meta.BuildArtifacts.LiveInitramfs.Path + rootfs := kola.CosaBuild.Meta.BuildArtifacts.LiveRootfs.Path + builddir := kola.CosaBuild.Dir + + tftpdir := filepath.Join(tempdir, "tftp") + if err := os.Mkdir(tftpdir, 0777); err != nil { + return nil, nil, err + } + + for _, name := range []string{kernel, initramfs, rootfs} { + if err := absSymlink(filepath.Join(builddir, name), filepath.Join(tftpdir, name)); err != nil { + return nil, nil, err + } + } - mach, err := inst.PXE(kola.QEMUOptions.PxeKernelArgs, liveConfig, targetConfig, opts.isOffline) + if opts.pxeAppendRootfs { + // replace the initramfs symlink with a concatenation of + // the initramfs and rootfs + initrd := filepath.Join(tftpdir, initramfs) + if err := os.Remove(initrd); err != nil { + return nil, nil, err + } + if err := cat(initrd, filepath.Join(builddir, initramfs), filepath.Join(builddir, rootfs)); err != nil { + return nil, nil, err + } + } + + var metalimg string + if opts.enable4k { + metalimg = kola.CosaBuild.Meta.BuildArtifacts.Metal4KNative.Path + } else { + metalimg = kola.CosaBuild.Meta.BuildArtifacts.Metal.Path + } + metalname, err := setupMetalImage(builddir, metalimg, tftpdir) if err != nil { - c.Fatal(err) + return nil, nil, errors.Wrapf(err, "setting up metal image") } - qc.AddMach(mach) - err = awaitCompletion(c, mach.Instance(), opts.console, outdir, completionChannel, mach.BootStartedErrorChannel(), []string{liveOKSignal, signalCompleteString}) + pxe := &PXE{ + tftpdir: tftpdir, + } + if err := pxe.setupArchDefaults(opts); err != nil { + return nil, nil, err + } + + listener, err := net.Listen("tcp", ":0") if err != nil { - c.Fatal(err) + return nil, nil, err + } + port := listener.Addr().(*net.TCPAddr).Port + baseurl := fmt.Sprintf("http://%s:%d", pxe.tftpipaddr, port) + + kargs := renderCosaTestIsoDebugKargs() + kargs = append(kargs, renderBaseKargs()...) + kargs = append(kargs, kola.QEMUOptions.PxeKernelArgs...) + kargs = append(kargs, fmt.Sprintf("ignition.config.url=%s/pxe-live.ign", baseurl)) + kargs = append(kargs, renderInstallKargs(baseurl, metalname, opts)...) + if rootfs != "" && !opts.pxeAppendRootfs { + kargs = append(kargs, fmt.Sprintf("coreos.live.rootfs_url=%s/%s", baseurl, rootfs)) + } + kargsStr := strings.Join(kargs, " ") + + switch pxe.boottype { + case "pxe": + if err := pxe.configBootPxe(kargsStr); err != nil { + return nil, nil, err + } + case "grub": + if err := pxe.configBootGrub(kargsStr); err != nil { + return nil, nil, err + } + default: + return nil, nil, errors.Errorf("Unhandled boottype %s", pxe.boottype) + } + + server := startHTTPServer(listener, tftpdir) + return pxe, server, nil +} + +func (pxe *PXE) setupArchDefaults(opts IsoTestOpts) error { + pxe.tftpipaddr = "192.168.76.2" + switch coreosarch.CurrentRpmArch() { + case "x86_64": + pxe.networkdevice = "e1000" + if opts.enableUefi { + pxe.boottype = "grub" + pxe.bootfile = "/boot/grub2/grubx64.efi" + pxe.pxeimagepath = "/boot/efi/EFI/fedora/grubx64.efi" + // Choose bootindex=2. First boot the hard drive won't + // have an OS and will fall through to bootindex 2 (net) + pxe.bootindex = "2" + } else { + pxe.boottype = "pxe" + pxe.pxeimagepath = "/usr/share/syslinux/" + } + case "aarch64": + pxe.boottype = "grub" + pxe.networkdevice = "virtio-net-pci" + pxe.bootfile = "/boot/grub2/grubaa64.efi" + pxe.pxeimagepath = "/boot/efi/EFI/fedora/grubaa64.efi" + pxe.bootindex = "1" + case "ppc64le": + pxe.boottype = "grub" + pxe.networkdevice = "virtio-net-pci" + pxe.bootfile = "/boot/grub2/powerpc-ieee1275/core.elf" + case "s390x": + pxe.boottype = "pxe" + pxe.networkdevice = "virtio-net-ccw" + pxe.bootindex = "1" + pxe.tftpipaddr = "10.0.2.2" + default: + return fmt.Errorf("unsupported arch %s", coreosarch.CurrentRpmArch()) + } + return nil +} + +func (pxe *PXE) configBootPxe(kargs string) error { + kernel := kola.CosaBuild.Meta.BuildArtifacts.LiveKernel.Path + initramfs := kola.CosaBuild.Meta.BuildArtifacts.LiveInitramfs.Path + + pxeconfigdir := filepath.Join(pxe.tftpdir, "pxelinux.cfg") + if err := os.Mkdir(pxeconfigdir, 0777); err != nil { + return errors.Wrapf(err, "creating dir %s", pxeconfigdir) + } + pxeimages := []string{"pxelinux.0", "ldlinux.c32"} + pxeconfig := []byte(fmt.Sprintf(` +DEFAULT pxeboot +TIMEOUT 20 +PROMPT 0 +LABEL pxeboot + KERNEL %s + APPEND initrd=%s %s +`, kernel, initramfs, kargs)) + if coreosarch.CurrentRpmArch() == "s390x" { + pxeconfig = []byte(kargs) + } + pxeconfig_path := filepath.Join(pxeconfigdir, "default") + if err := os.WriteFile(pxeconfig_path, pxeconfig, 0777); err != nil { + return errors.Wrapf(err, "writing file %s", pxeconfig_path) + } + + // this is only for s390x where the pxe image has to be created; + // s390 doesn't seem to have a pre-created pxe image although have to check on this + if pxe.pxeimagepath == "" { + kernelpath := filepath.Join(pxe.tftpdir, kernel) + initrdpath := filepath.Join(pxe.tftpdir, initramfs) + err := exec.Command("/usr/bin/mk-s390image", kernelpath, "-r", initrdpath, + "-p", filepath.Join(pxeconfigdir, "default"), filepath.Join(pxe.tftpdir, pxeimages[0])).Run() + if err != nil { + return errors.Wrap(err, "running mk-s390image") + } + } else { + for _, img := range pxeimages { + srcpath := filepath.Join("/usr/share/syslinux", img) + cp_cmd := exec.Command("/usr/lib/coreos-assembler/cp-reflink", srcpath, pxe.tftpdir) + cp_cmd.Stderr = os.Stderr + if err := cp_cmd.Run(); err != nil { + return errors.Wrapf(err, "running cp-reflink %s %s", srcpath, pxe.tftpdir) + } + } + } + pxe.bootfile = "/" + pxeimages[0] + return nil +} + +func (pxe *PXE) configBootGrub(kargs string) error { + kernel := kola.CosaBuild.Meta.BuildArtifacts.LiveKernel.Path + initramfs := kola.CosaBuild.Meta.BuildArtifacts.LiveInitramfs.Path + + grub2_mknetdir_cmd := exec.Command("grub2-mknetdir", "--net-directory="+pxe.tftpdir) + grub2_mknetdir_cmd.Stderr = os.Stderr + if err := grub2_mknetdir_cmd.Run(); err != nil { + return errors.Wrap(err, "running grub2-mknetdir") + } + if pxe.pxeimagepath != "" { + dstpath := filepath.Join(pxe.tftpdir, "boot/grub2") + cp_cmd := exec.Command("/usr/lib/coreos-assembler/cp-reflink", pxe.pxeimagepath, dstpath) + cp_cmd.Stderr = os.Stderr + if err := cp_cmd.Run(); err != nil { + return errors.Wrapf(err, "running cp-reflink %s %s", pxe.pxeimagepath, dstpath) + } + } + if err := os.WriteFile(filepath.Join(pxe.tftpdir, "boot/grub2/grub.cfg"), []byte(fmt.Sprintf(` +default=0 +timeout=1 +menuentry "CoreOS (BIOS/UEFI)" { + echo "Loading kernel" + linux /%s %s + echo "Loading initrd" + initrd %s +}`, kernel, kargs, initramfs)), 0777); err != nil { + return errors.Wrap(err, "writing grub.cfg") + } + return nil +} + +func renderBaseKargs() []string { + baseKargs := []string{"rd.neednet=1", "ip=dhcp", "ignition.firstboot", "ignition.platform.id=metal"} + return append(baseKargs, fmt.Sprintf("console=%s", platform.ConsoleKernelArgument[coreosarch.CurrentRpmArch()])) +} + +func renderInstallKargs(baseurl string, metalname string, opts IsoTestOpts) []string { + args := []string{"coreos.inst.install_dev=/dev/vda", + fmt.Sprintf("coreos.inst.ignition_url=%s/pxe-target.ign", baseurl)} + if !opts.isOffline { + args = append(args, fmt.Sprintf("coreos.inst.image_url=%s/%s", baseurl, metalname)) + } + // FIXME - ship signatures by default too + if opts.instInsecure { + args = append(args, "coreos.inst.insecure") } + return args } diff --git a/mantle/platform/machine/qemu/metal.go b/mantle/platform/machine/qemu/metal.go deleted file mode 100644 index d4730bfe87..0000000000 --- a/mantle/platform/machine/qemu/metal.go +++ /dev/null @@ -1,576 +0,0 @@ -// Copyright 2020 Red Hat -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package qemu - -import ( - "bufio" - "fmt" - "io" - "net" - "net/http" - "os" - "path/filepath" - "strings" - "time" - - "github.com/coreos/coreos-assembler/mantle/platform" - coreosarch "github.com/coreos/stream-metadata-go/arch" - "github.com/pkg/errors" - "gopkg.in/yaml.v2" - - "github.com/coreos/coreos-assembler/mantle/platform/conf" - "github.com/coreos/coreos-assembler/mantle/system/exec" - "github.com/coreos/coreos-assembler/mantle/util" -) - -const ( - // defaultQemuHostIPv4 is documented in `man qemu-kvm`, under the `-netdev` option - defaultQemuHostIPv4 = "10.0.2.2" - - bootStartedSignal = "boot-started-OK" -) - -// TODO derive this from docs, or perhaps include kargs in cosa metadata? -var baseKargs = []string{"rd.neednet=1", "ip=dhcp", "ignition.firstboot", "ignition.platform.id=metal"} - -var ( - bootStartedUnit = fmt.Sprintf(`[Unit] - Description=TestISO Boot Started - Requires=dev-virtio\\x2dports-bootstarted.device - OnFailure=emergency.target - OnFailureJobMode=isolate - [Service] - Type=oneshot - RemainAfterExit=yes - ExecStart=/bin/sh -c '/usr/bin/echo %s >/dev/virtio-ports/bootstarted' - [Install] - RequiredBy=coreos-installer.target - `, bootStartedSignal) -) - -// NewMetalQemuBuilderDefault returns a QEMU builder instance with some -// defaults set up for bare metal. -func NewMetalQemuBuilderDefault() *platform.QemuBuilder { - builder := platform.NewQemuBuilder() - // https://github.com/coreos/fedora-coreos-tracker/issues/388 - // https://github.com/coreos/fedora-coreos-docs/pull/46 - builder.MemoryMiB = 4096 - return builder -} - -type Install struct { - CosaBuild *util.LocalBuild - Builder *platform.QemuBuilder - Insecure bool - Native4k bool - MultiPathDisk bool - PxeAppendRootfs bool - NmKeyfiles map[string]string - - // These are set by the install path - kargs []string - ignition conf.Conf - liveIgnition conf.Conf -} - -// Check that artifact has been built and locally exists -func (inst *Install) checkArtifactsExist(artifacts []string) error { - version := inst.CosaBuild.Meta.OstreeVersion - for _, name := range artifacts { - artifact, err := inst.CosaBuild.Meta.GetArtifact(name) - if err != nil { - return fmt.Errorf("Missing artifact %s for %s build: %s", name, version, err) - } - path := filepath.Join(inst.CosaBuild.Dir, artifact.Path) - if _, err := os.Stat(path); err != nil { - if os.IsNotExist(err) { - return fmt.Errorf("Missing local file for artifact %s for build %s", name, version) - } - } - } - return nil -} - -func (inst *Install) PXE(kargs []string, liveIgnition, ignition conf.Conf, offline bool) (*machine, error) { - artifacts := []string{"live-kernel", "live-rootfs"} - if err := inst.checkArtifactsExist(artifacts); err != nil { - return nil, err - } - - installerConfig := installerConfig{ - Console: []string{platform.ConsoleKernelArgument[coreosarch.CurrentRpmArch()]}, - AppendKargs: renderCosaTestIsoDebugKargs(), - } - installerConfigData, err := yaml.Marshal(installerConfig) - if err != nil { - return nil, err - } - mode := 0644 - - // XXX: https://github.com/coreos/coreos-installer/issues/1171 - if coreosarch.CurrentRpmArch() != "s390x" { - liveIgnition.AddFile("/etc/coreos/installer.d/mantle.yaml", string(installerConfigData), mode) - } - - inst.kargs = append(renderCosaTestIsoDebugKargs(), kargs...) - inst.ignition = ignition - inst.liveIgnition = liveIgnition - - mach, err := inst.runPXE(&kernelSetup{ - kernel: inst.CosaBuild.Meta.BuildArtifacts.LiveKernel.Path, - initramfs: inst.CosaBuild.Meta.BuildArtifacts.LiveInitramfs.Path, - rootfs: inst.CosaBuild.Meta.BuildArtifacts.LiveRootfs.Path, - }, offline) - if err != nil { - return nil, errors.Wrapf(err, "testing live installer") - } - - return mach, nil -} - -type kernelSetup struct { - kernel, initramfs, rootfs string -} - -type pxeSetup struct { - tftpipaddr string - boottype string - networkdevice string - bootindex string - pxeimagepath string - - // bootfile is initialized later - bootfile string -} - -type installerRun struct { - inst *Install - builder *platform.QemuBuilder - - builddir string - tempdir string - tftpdir string - - metalimg string - metalname string - - baseurl string - - kern kernelSetup - pxe pxeSetup -} - -func absSymlink(src, dest string) error { - src, err := filepath.Abs(src) - if err != nil { - return err - } - return os.Symlink(src, dest) -} - -// setupMetalImage creates a symlink to the metal image. -func setupMetalImage(builddir, metalimg, destdir string) (string, error) { - if err := absSymlink(filepath.Join(builddir, metalimg), filepath.Join(destdir, metalimg)); err != nil { - return "", err - } - return metalimg, nil -} - -func (inst *Install) setup(kern *kernelSetup) (*installerRun, error) { - var artifacts []string - if inst.Native4k { - artifacts = append(artifacts, "metal4k") - } else { - artifacts = append(artifacts, "metal") - } - if err := inst.checkArtifactsExist(artifacts); err != nil { - return nil, err - } - - builder := inst.Builder - - tempdir, err := os.MkdirTemp("/var/tmp", "mantle-pxe") - if err != nil { - return nil, err - } - cleanupTempdir := true - defer func() { - if cleanupTempdir { - os.RemoveAll(tempdir) - } - }() - - tftpdir := filepath.Join(tempdir, "tftp") - if err := os.Mkdir(tftpdir, 0777); err != nil { - return nil, err - } - - builddir := inst.CosaBuild.Dir - if err := inst.ignition.WriteFile(filepath.Join(tftpdir, "config.ign")); err != nil { - return nil, err - } - // This code will ensure to add an SSH key to `pxe-live.ign` config. - inst.liveIgnition.AddAutoLogin() - inst.liveIgnition.AddSystemdUnit("boot-started.service", bootStartedUnit, conf.Enable) - if err := inst.liveIgnition.WriteFile(filepath.Join(tftpdir, "pxe-live.ign")); err != nil { - return nil, err - } - - for _, name := range []string{kern.kernel, kern.initramfs, kern.rootfs} { - if err := absSymlink(filepath.Join(builddir, name), filepath.Join(tftpdir, name)); err != nil { - return nil, err - } - } - if inst.PxeAppendRootfs { - // replace the initramfs symlink with a concatenation of - // the initramfs and rootfs - initrd := filepath.Join(tftpdir, kern.initramfs) - if err := os.Remove(initrd); err != nil { - return nil, err - } - if err := cat(initrd, filepath.Join(builddir, kern.initramfs), filepath.Join(builddir, kern.rootfs)); err != nil { - return nil, err - } - } - - var metalimg string - if inst.Native4k { - metalimg = inst.CosaBuild.Meta.BuildArtifacts.Metal4KNative.Path - } else { - metalimg = inst.CosaBuild.Meta.BuildArtifacts.Metal.Path - } - metalname, err := setupMetalImage(builddir, metalimg, tftpdir) - if err != nil { - return nil, errors.Wrapf(err, "setting up metal image") - } - - pxe := pxeSetup{} - pxe.tftpipaddr = "192.168.76.2" - switch coreosarch.CurrentRpmArch() { - case "x86_64": - pxe.networkdevice = "e1000" - if builder.Firmware == "uefi" { - pxe.boottype = "grub" - pxe.bootfile = "/boot/grub2/grubx64.efi" - pxe.pxeimagepath = "/boot/efi/EFI/fedora/grubx64.efi" - // Choose bootindex=2. First boot the hard drive won't - // have an OS and will fall through to bootindex 2 (net) - pxe.bootindex = "2" - } else { - pxe.boottype = "pxe" - pxe.pxeimagepath = "/usr/share/syslinux/" - } - case "aarch64": - pxe.boottype = "grub" - pxe.networkdevice = "virtio-net-pci" - pxe.bootfile = "/boot/grub2/grubaa64.efi" - pxe.pxeimagepath = "/boot/efi/EFI/fedora/grubaa64.efi" - pxe.bootindex = "1" - case "ppc64le": - pxe.boottype = "grub" - pxe.networkdevice = "virtio-net-pci" - pxe.bootfile = "/boot/grub2/powerpc-ieee1275/core.elf" - case "s390x": - pxe.boottype = "pxe" - pxe.networkdevice = "virtio-net-ccw" - pxe.tftpipaddr = "10.0.2.2" - pxe.bootindex = "1" - default: - return nil, fmt.Errorf("Unsupported arch %s", coreosarch.CurrentRpmArch()) - } - - mux := http.NewServeMux() - mux.Handle("/", http.FileServer(http.Dir(tftpdir))) - listener, err := net.Listen("tcp", ":0") - if err != nil { - return nil, err - } - port := listener.Addr().(*net.TCPAddr).Port - //nolint // Yeah this leaks - go func() { - http.Serve(listener, mux) - }() - baseurl := fmt.Sprintf("http://%s:%d", pxe.tftpipaddr, port) - - cleanupTempdir = false // Transfer ownership - return &installerRun{ - inst: inst, - - builder: builder, - tempdir: tempdir, - tftpdir: tftpdir, - builddir: builddir, - - metalimg: metalimg, - metalname: metalname, - - baseurl: baseurl, - - pxe: pxe, - kern: *kern, - }, nil -} - -func renderBaseKargs() []string { - return append(baseKargs, fmt.Sprintf("console=%s", platform.ConsoleKernelArgument[coreosarch.CurrentRpmArch()])) -} - -func renderInstallKargs(t *installerRun, offline bool) []string { - args := []string{"coreos.inst.install_dev=/dev/vda", - fmt.Sprintf("coreos.inst.ignition_url=%s/config.ign", t.baseurl)} - if !offline { - args = append(args, fmt.Sprintf("coreos.inst.image_url=%s/%s", t.baseurl, t.metalname)) - } - // FIXME - ship signatures by default too - if t.inst.Insecure { - args = append(args, "coreos.inst.insecure") - } - return args -} - -// Sometimes the logs that stream from various virtio streams can be -// incomplete because they depend on services inside the guest. -// When you are debugging earlyboot/initramfs issues this can be -// problematic. Let's add a hook here to enable more debugging. -func renderCosaTestIsoDebugKargs() []string { - if _, ok := os.LookupEnv("COSA_TESTISO_DEBUG"); ok { - return []string{"systemd.log_color=0", "systemd.log_level=debug", - "systemd.journald.forward_to_console=1", - "systemd.journald.max_level_console=debug"} - } else { - return []string{} - } -} - -func (t *installerRun) destroy() error { - t.builder.Close() - if t.tempdir != "" { - return os.RemoveAll(t.tempdir) - } - return nil -} - -func (t *installerRun) completePxeSetup(kargs []string) error { - if t.kern.rootfs != "" && !t.inst.PxeAppendRootfs { - kargs = append(kargs, fmt.Sprintf("coreos.live.rootfs_url=%s/%s", t.baseurl, t.kern.rootfs)) - } - kargsStr := strings.Join(kargs, " ") - - switch t.pxe.boottype { - case "pxe": - pxeconfigdir := filepath.Join(t.tftpdir, "pxelinux.cfg") - if err := os.Mkdir(pxeconfigdir, 0777); err != nil { - return errors.Wrapf(err, "creating dir %s", pxeconfigdir) - } - pxeimages := []string{"pxelinux.0", "ldlinux.c32"} - pxeconfig := []byte(fmt.Sprintf(` - DEFAULT pxeboot - TIMEOUT 20 - PROMPT 0 - LABEL pxeboot - KERNEL %s - APPEND initrd=%s %s - `, t.kern.kernel, t.kern.initramfs, kargsStr)) - if coreosarch.CurrentRpmArch() == "s390x" { - pxeconfig = []byte(kargsStr) - } - pxeconfig_path := filepath.Join(pxeconfigdir, "default") - if err := os.WriteFile(pxeconfig_path, pxeconfig, 0777); err != nil { - return errors.Wrapf(err, "writing file %s", pxeconfig_path) - } - - // this is only for s390x where the pxe image has to be created; - // s390 doesn't seem to have a pre-created pxe image although have to check on this - if t.pxe.pxeimagepath == "" { - kernelpath := filepath.Join(t.tftpdir, t.kern.kernel) - initrdpath := filepath.Join(t.tftpdir, t.kern.initramfs) - err := exec.Command("/usr/bin/mk-s390image", kernelpath, "-r", initrdpath, - "-p", filepath.Join(pxeconfigdir, "default"), filepath.Join(t.tftpdir, pxeimages[0])).Run() - if err != nil { - return errors.Wrap(err, "running mk-s390image") - } - } else { - for _, img := range pxeimages { - srcpath := filepath.Join("/usr/share/syslinux", img) - cp_cmd := exec.Command("/usr/lib/coreos-assembler/cp-reflink", srcpath, t.tftpdir) - cp_cmd.Stderr = os.Stderr - if err := cp_cmd.Run(); err != nil { - return errors.Wrapf(err, "running cp-reflink %s %s", srcpath, t.tftpdir) - } - } - } - t.pxe.bootfile = "/" + pxeimages[0] - case "grub": - grub2_mknetdir_cmd := exec.Command("grub2-mknetdir", "--net-directory="+t.tftpdir) - grub2_mknetdir_cmd.Stderr = os.Stderr - if err := grub2_mknetdir_cmd.Run(); err != nil { - return errors.Wrap(err, "running grub2-mknetdir") - } - if t.pxe.pxeimagepath != "" { - dstpath := filepath.Join(t.tftpdir, "boot/grub2") - cp_cmd := exec.Command("/usr/lib/coreos-assembler/cp-reflink", t.pxe.pxeimagepath, dstpath) - cp_cmd.Stderr = os.Stderr - if err := cp_cmd.Run(); err != nil { - return errors.Wrapf(err, "running cp-reflink %s %s", t.pxe.pxeimagepath, dstpath) - } - } - if err := os.WriteFile(filepath.Join(t.tftpdir, "boot/grub2/grub.cfg"), []byte(fmt.Sprintf(` - default=0 - timeout=1 - menuentry "CoreOS (BIOS/UEFI)" { - echo "Loading kernel" - linux /%s %s - echo "Loading initrd" - initrd %s - } - `, t.kern.kernel, kargsStr, t.kern.initramfs)), 0777); err != nil { - return errors.Wrap(err, "writing grub.cfg") - } - default: - panic("Unhandled boottype " + t.pxe.boottype) - } - - return nil -} - -func switchBootOrderSignal(qinst *platform.QemuInstance, bootstartedchan *os.File, booterrchan *chan error) { - *booterrchan = make(chan error) - go func() { - err := qinst.Wait() - // only one Wait() gets process data, so also manually check for signal - if err == nil && qinst.Signaled() { - err = errors.New("process killed") - } - if err != nil { - *booterrchan <- errors.Wrapf(err, "QEMU unexpectedly exited while waiting for %s", bootStartedSignal) - } - }() - go func() { - r := bufio.NewReader(bootstartedchan) - l, err := r.ReadString('\n') - if err != nil { - if err == io.EOF { - // this may be from QEMU getting killed or exiting; wait a bit - // to give a chance for .Wait() above to feed the channel with a - // better error - time.Sleep(1 * time.Second) - *booterrchan <- fmt.Errorf("Got EOF from boot started channel, %s expected", bootStartedSignal) - } else { - *booterrchan <- errors.Wrapf(err, "reading from boot started channel") - } - return - } - line := strings.TrimSpace(l) - // switch the boot order here, we are well into the installation process - only for aarch64 and s390x - if line == bootStartedSignal { - if err := qinst.SwitchBootOrder(); err != nil { - *booterrchan <- errors.Wrapf(err, "switching boot order failed") - return - } - } - // OK! - *booterrchan <- nil - }() -} - -func cat(outfile string, infiles ...string) error { - out, err := os.OpenFile(outfile, os.O_WRONLY|os.O_CREATE, 0644) - if err != nil { - return err - } - defer out.Close() - for _, infile := range infiles { - in, err := os.Open(infile) - if err != nil { - return err - } - defer in.Close() - _, err = io.Copy(out, in) - if err != nil { - return err - } - } - return nil -} - -func (t *installerRun) run() (*platform.QemuInstance, error) { - builder := t.builder - netdev := fmt.Sprintf("%s,netdev=mynet0,mac=52:54:00:12:34:56", t.pxe.networkdevice) - if t.pxe.bootindex == "" { - builder.Append("-boot", "once=n") - } else { - netdev += fmt.Sprintf(",bootindex=%s", t.pxe.bootindex) - } - builder.Append("-device", netdev) - usernetdev := fmt.Sprintf("user,id=mynet0,tftp=%s,bootfile=%s", t.tftpdir, t.pxe.bootfile) - if t.pxe.tftpipaddr != "10.0.2.2" { - usernetdev += ",net=192.168.76.0/24,dhcpstart=192.168.76.9" - } - builder.Append("-netdev", usernetdev) - - inst, err := builder.Exec() - if err != nil { - return nil, err - } - return inst, nil -} - -func (inst *Install) runPXE(kern *kernelSetup, offline bool) (*machine, error) { - t, err := inst.setup(kern) - if err != nil { - return nil, errors.Wrapf(err, "setting up install") - } - defer func() { - err = t.destroy() - }() - - bootStartedChan, err := inst.Builder.VirtioChannelRead("bootstarted") - if err != nil { - return nil, errors.Wrapf(err, "setting up bootstarted virtio-serial channel") - } - - kargs := renderBaseKargs() - kargs = append(kargs, inst.kargs...) - kargs = append(kargs, fmt.Sprintf("ignition.config.url=%s/pxe-live.ign", t.baseurl)) - - kargs = append(kargs, renderInstallKargs(t, offline)...) - if err := t.completePxeSetup(kargs); err != nil { - return nil, errors.Wrapf(err, "completing PXE setup") - } - qinst, err := t.run() - if err != nil { - return nil, errors.Wrapf(err, "running PXE install") - } - tempdir := t.tempdir - t.tempdir = "" // Transfer ownership - instmachine := machine{ - inst: qinst, - tempdir: tempdir, - } - switchBootOrderSignal(qinst, bootStartedChan, &instmachine.bootStartedErrorChannel) - return &instmachine, nil -} - -// This object gets serialized to YAML and fed to coreos-installer: -// https://coreos.github.io/coreos-installer/customizing-install/#config-file-format -type installerConfig struct { - ImageURL string `yaml:"image-url,omitempty"` - IgnitionFile string `yaml:"ignition-file,omitempty"` - Insecure bool `yaml:",omitempty"` - AppendKargs []string `yaml:"append-karg,omitempty"` - CopyNetwork bool `yaml:"copy-network,omitempty"` - DestDevice string `yaml:"dest-device,omitempty"` - Console []string `yaml:"console,omitempty"` -} From 8e60899c450d858ed528f59f29b20fc4402f8cf6 Mon Sep 17 00:00:00 2001 From: Nikita Dubrovskii Date: Fri, 28 Nov 2025 17:48:58 +0100 Subject: [PATCH 24/31] mantle/platform/machine/qemu: drop no longer used code Remove deprecated machine struct fields and methods that are no longer needed after the migration all ISO tests under kola. --- mantle/platform/machine/qemu/machine.go | 46 ++++--------------------- 1 file changed, 7 insertions(+), 39 deletions(-) diff --git a/mantle/platform/machine/qemu/machine.go b/mantle/platform/machine/qemu/machine.go index 9416bca277..aaba71b882 100644 --- a/mantle/platform/machine/qemu/machine.go +++ b/mantle/platform/machine/qemu/machine.go @@ -26,35 +26,19 @@ import ( ) type machine struct { - qc *Cluster - id string - inst *platform.QemuInstance - journal *platform.Journal - consolePath string - console string - ip string - tempdir string - bootStartedErrorChannel chan error + qc *Cluster + id string + inst *platform.QemuInstance + journal *platform.Journal + consolePath string + console string + ip string } func (m *machine) ID() string { return m.id } -// Instance returns the underlying QemuInstance for this machine. -// This is primarily used for ISO installation tests that need direct access -// to the QEMU instance. -func (m *machine) Instance() *platform.QemuInstance { - return m.inst -} - -// BootStartedErrorChannel returns the channel used to signal boot completion -// or errors during the boot process. This is used by ISO installation tests -// to coordinate boot order changes. -func (m *machine) BootStartedErrorChannel() chan error { - return m.bootStartedErrorChannel -} - func (m *machine) IP() string { return m.ip } @@ -107,18 +91,6 @@ func (m *machine) WaitForSoftReboot(timeout time.Duration, oldSoftRebootsCount s return platform.WaitForMachineSoftReboot(m, m.journal, timeout, oldSoftRebootsCount) } -// DeleteTempdir removes the temporary directory associated with this machine. -// It's safe to call multiple times. This is automatically called by Destroy(), -// but can be called earlier if needed to free disk space. -func (m *machine) DeleteTempdir() error { - if m.tempdir == "" { - return nil - } - err := os.RemoveAll(m.tempdir) - m.tempdir = "" - return err -} - func (m *machine) Destroy() { if m == nil { return @@ -146,10 +118,6 @@ func (m *machine) Destroy() { m.qc.DelMach(m) m.qc = nil } - - if err := m.DeleteTempdir(); err != nil { - plog.Errorf("Error removing tempdir for instance %v: %v", m.ID(), err) - } } func (m *machine) ConsoleOutput() string { From 95a660f5d343b71536fd482d5d776664a7302d92 Mon Sep 17 00:00:00 2001 From: Nikita Dubrovskii Date: Thu, 2 Apr 2026 17:57:10 +0200 Subject: [PATCH 25/31] mantle/kola/tests/iso: Use firmware string instead of booleans Replace enableUefi and enableUefiSecure boolean flags with a single firmware string field in IsoTestOpts for simpler and more consistent firmware configuration across all ISO tests. --- mantle/kola/tests/iso/common.go | 27 +++++++++++++-------------- mantle/kola/tests/iso/live-as-disk.go | 9 +-------- mantle/kola/tests/iso/live-fips.go | 6 +++--- mantle/kola/tests/iso/live-iscsi.go | 6 +++--- mantle/kola/tests/iso/live-iso.go | 4 +--- mantle/kola/tests/iso/live-pxe.go | 7 +++---- 6 files changed, 24 insertions(+), 35 deletions(-) diff --git a/mantle/kola/tests/iso/common.go b/mantle/kola/tests/iso/common.go index 2fb0168dbe..fd51f321b8 100644 --- a/mantle/kola/tests/iso/common.go +++ b/mantle/kola/tests/iso/common.go @@ -79,18 +79,17 @@ type IsoTestOpts struct { // Flags().BoolVarP(&instInsecure, "inst-insecure", "S", false, "Do not verify signature on metal image") instInsecure bool // Flags().BoolVar(&console, "console", false, "Connect qemu console to terminal, turn off automatic initramfs failure checking") - console bool - addNmKeyfile bool - enable4k bool - enableMultipath bool - isOffline bool - isISOFromRAM bool - isMiniso bool - enableUefi bool - enableUefiSecure bool - enableIbft bool - manual bool - pxeAppendRootfs bool + console bool + addNmKeyfile bool + enable4k bool + enableMultipath bool + isOffline bool + isISOFromRAM bool + isMiniso bool + enableIbft bool + manual bool + pxeAppendRootfs bool + firmware string } func getIsoTestOpts(testName string) IsoTestOpts { @@ -101,9 +100,9 @@ func getIsoTestOpts(testName string) IsoTestOpts { opts.enable4k = true } if strings.Contains(testName, "uefi-secure") { - opts.enableUefiSecure = true + opts.firmware = "uefi-secure" } else if strings.Contains(testName, "uefi") { - opts.enableUefi = true + opts.firmware = "uefi" } if strings.Contains(testName, "mpath") { opts.enableMultipath = true diff --git a/mantle/kola/tests/iso/live-as-disk.go b/mantle/kola/tests/iso/live-as-disk.go index 700c24a92c..aec4084bfa 100644 --- a/mantle/kola/tests/iso/live-as-disk.go +++ b/mantle/kola/tests/iso/live-as-disk.go @@ -77,14 +77,7 @@ func isoTestAsDisk(c cluster.TestCluster, opts IsoTestOpts) { return nil } - options := platform.MachineOptions{} - switch { - case opts.enableUefiSecure: - options.Firmware = "uefi-secure" - case opts.enableUefi: - options.Firmware = "uefi" - } - + options := platform.MachineOptions{Firmware: opts.firmware} machineBuilder := &qemu.MachineBuilder{ SetupDisks: setupDisks, } diff --git a/mantle/kola/tests/iso/live-fips.go b/mantle/kola/tests/iso/live-fips.go index fac149656e..b81ae524b8 100644 --- a/mantle/kola/tests/iso/live-fips.go +++ b/mantle/kola/tests/iso/live-fips.go @@ -85,9 +85,9 @@ func testLiveFIPS(c cluster.TestCluster, opts IsoTestOpts) { return nil } - options := platform.MachineOptions{AppendKernelArgs: "fips=1"} - if opts.enableUefi { - options.Firmware = "uefi" + options := platform.MachineOptions{ + AppendKernelArgs: "fips=1", + Firmware: opts.firmware, } machineBuilder := &qemu.MachineBuilder{ diff --git a/mantle/kola/tests/iso/live-iscsi.go b/mantle/kola/tests/iso/live-iscsi.go index c8a285ba0b..d8920e2cfd 100644 --- a/mantle/kola/tests/iso/live-iscsi.go +++ b/mantle/kola/tests/iso/live-iscsi.go @@ -179,9 +179,9 @@ func isoInstalliScsi(c cluster.TestCluster, opts IsoTestOpts) { } // We need more memory to start another VM within ! - options := platform.MachineOptions{MinMemory: 2048} - if opts.enableUefi { - options.Firmware = "uefi" + options := platform.MachineOptions{ + MinMemory: 2048, + Firmware: opts.firmware, } machineBuilder := &qemu.MachineBuilder{ SetupDisks: setupDisks, diff --git a/mantle/kola/tests/iso/live-iso.go b/mantle/kola/tests/iso/live-iso.go index a3b05f8559..9014f9baa9 100644 --- a/mantle/kola/tests/iso/live-iso.go +++ b/mantle/kola/tests/iso/live-iso.go @@ -328,9 +328,7 @@ func runIsoTest(qc *qemu.Cluster, opts IsoTestOpts, tempdir string) error { MinMemory: 4096, MultiPathDisk: opts.enableMultipath, AppendKernelArgs: strings.Join(kargs, " "), - } - if opts.enableUefi { - options.Firmware = "uefi" + Firmware: opts.firmware, } machineBuilder := &qemu.MachineBuilder{ diff --git a/mantle/kola/tests/iso/live-pxe.go b/mantle/kola/tests/iso/live-pxe.go index a5fdfcc354..90c6fab4bf 100644 --- a/mantle/kola/tests/iso/live-pxe.go +++ b/mantle/kola/tests/iso/live-pxe.go @@ -214,10 +214,9 @@ func testPXE(c cluster.TestCluster, opts IsoTestOpts) { options := platform.MachineOptions{ MinMemory: 4096, + Firmware: opts.firmware, } - if opts.enableUefi { - options.Firmware = "uefi" - } + // increase the memory for pxe tests with appended rootfs in the initrd // we were bumping up into the 4GiB limit in RHCOS/c9s if opts.pxeAppendRootfs { @@ -382,7 +381,7 @@ func (pxe *PXE) setupArchDefaults(opts IsoTestOpts) error { switch coreosarch.CurrentRpmArch() { case "x86_64": pxe.networkdevice = "e1000" - if opts.enableUefi { + if opts.firmware == "uefi" { pxe.boottype = "grub" pxe.bootfile = "/boot/grub2/grubx64.efi" pxe.pxeimagepath = "/boot/efi/EFI/fedora/grubx64.efi" From 101c90872b2a40e46ee819445ba58b42be976d59 Mon Sep 17 00:00:00 2001 From: Nikita Dubrovskii Date: Thu, 2 Apr 2026 17:59:51 +0200 Subject: [PATCH 26/31] mantle/kola/tests/iso: Remove obsolete console field from IsoTestOpts --- mantle/kola/tests/iso/common.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/mantle/kola/tests/iso/common.go b/mantle/kola/tests/iso/common.go index fd51f321b8..f0c6d3781c 100644 --- a/mantle/kola/tests/iso/common.go +++ b/mantle/kola/tests/iso/common.go @@ -76,10 +76,7 @@ func startHTTPServer(listener net.Listener, dir string) *http.Server { } type IsoTestOpts struct { - // Flags().BoolVarP(&instInsecure, "inst-insecure", "S", false, "Do not verify signature on metal image") - instInsecure bool - // Flags().BoolVar(&console, "console", false, "Connect qemu console to terminal, turn off automatic initramfs failure checking") - console bool + instInsecure bool addNmKeyfile bool enable4k bool enableMultipath bool From 86a059561d6394525ade98c516ebe14741cd408a Mon Sep 17 00:00:00 2001 From: Nikita Dubrovskii Date: Tue, 14 Apr 2026 12:13:05 +0200 Subject: [PATCH 27/31] mantle/platform/qemu: add setter for netdev's bootindex property Add SetNetbootIndex() method to QemuBuilder to allow configuring the bootindex property for network devices. This enables proper boot order control when using PXE/network boot, as an alternative to the legacy -boot order=n option. The bootindex property is mutually exclusive with -boot order, as most firmware implementations support one or the other but not both. When bootindex is set, the -boot order option is automatically omitted. --- mantle/platform/qemu.go | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/mantle/platform/qemu.go b/mantle/platform/qemu.go index ed10240611..431d37dc5c 100644 --- a/mantle/platform/qemu.go +++ b/mantle/platform/qemu.go @@ -542,6 +542,7 @@ type QemuBuilder struct { additionalNics int netbootP string netbootDir string + netbootIndex string finalized bool diskID uint @@ -682,6 +683,10 @@ func (builder *QemuBuilder) SetNetbootP(filename, dir string) { builder.netbootDir = dir } +func (builder *QemuBuilder) SetNetbootIndex(index string) { + builder.netbootIndex = index +} + func (builder *QemuBuilder) AddAdditionalNics(additionalNics int) { builder.additionalNics = additionalNics } @@ -735,10 +740,17 @@ func (builder *QemuBuilder) setupNetworking() error { relpath = builder.netbootP } netdev += fmt.Sprintf(",tftp=%s,bootfile=/%s", tftpDir, relpath) - builder.Append("-boot", "order=n") + // It does not make sense to use the bootindex together with the -boot order=... (or -boot once=...). + // The guest firmware implementations normally either support the one or the other. + if builder.netbootIndex == "" { + builder.Append("-boot", "order=n") + } } - - builder.Append("-netdev", netdev, "-device", virtio(builder.architecture, "net", "netdev=eth0")) + args := "netdev=eth0" + if builder.netbootIndex != "" { + args += fmt.Sprintf(",bootindex=%s", builder.netbootIndex) + } + builder.Append("-netdev", netdev, "-device", virtio(builder.architecture, "net", args)) return nil } From 85b71baf31cad6f313c1d0fe98ec88680fec77d5 Mon Sep 17 00:00:00 2001 From: Nikita Dubrovskii Date: Tue, 14 Apr 2026 12:15:45 +0200 Subject: [PATCH 28/31] kola/tests/iso: simplify iso.pxe* networking initialization Simplify PXE test network setup by using QemuBuilder's built-in methods (SetNetbootP, SetNetbootIndex, EnableUsermodeNetworking) instead of manually constructing QEMU network device arguments. --- mantle/kola/tests/iso/live-pxe.go | 38 ++++++++++++------------------- 1 file changed, 15 insertions(+), 23 deletions(-) diff --git a/mantle/kola/tests/iso/live-pxe.go b/mantle/kola/tests/iso/live-pxe.go index 90c6fab4bf..ede1da53cd 100644 --- a/mantle/kola/tests/iso/live-pxe.go +++ b/mantle/kola/tests/iso/live-pxe.go @@ -159,20 +159,17 @@ func testPXE(c cluster.TestCluster, opts IsoTestOpts) { } setupNet := func(o platform.MachineOptions, builder *platform.QemuBuilder) error { - netdev := fmt.Sprintf("%s,netdev=mynet0,mac=52:54:00:12:34:56", pxe.networkdevice) - if pxe.bootindex == "" { - builder.Append("-boot", "once=n") - } else { - netdev += fmt.Sprintf(",bootindex=%s", pxe.bootindex) - } - builder.Append("-device", netdev) - usernetdev := fmt.Sprintf("user,id=mynet0,tftp=%s,bootfile=%s", pxe.tftpdir, pxe.bootfile) + usernetdev := "" if pxe.tftpipaddr != "10.0.2.2" { - usernetdev += ",net=192.168.76.0/24,dhcpstart=192.168.76.9" + usernetdev = "192.168.76.0/24,dhcpstart=192.168.76.9" } - builder.Append("-netdev", usernetdev) - // for SSH access - return qc.SetupDefaultNetwork(o, builder) + h := []platform.HostForwardPort{ + {Service: "ssh", HostPort: 0, GuestPort: 22}, + } + builder.SetNetbootP(pxe.bootfile, pxe.tftpdir) + builder.SetNetbootIndex(pxe.bootindex) + builder.EnableUsermodeNetworking(h, usernetdev) + return nil } errchan := make(chan error) @@ -286,13 +283,12 @@ func getPXEConfig(insecure bool, offline bool) (*conf.Conf, error) { } type PXE struct { - tftpdir string - tftpipaddr string - boottype string - networkdevice string - bootindex string - pxeimagepath string - bootfile string + tftpdir string + tftpipaddr string + boottype string + bootindex string + pxeimagepath string + bootfile string } func createPXE(tempdir string, opts IsoTestOpts) (*PXE, *http.Server, error) { @@ -380,7 +376,6 @@ func (pxe *PXE) setupArchDefaults(opts IsoTestOpts) error { pxe.tftpipaddr = "192.168.76.2" switch coreosarch.CurrentRpmArch() { case "x86_64": - pxe.networkdevice = "e1000" if opts.firmware == "uefi" { pxe.boottype = "grub" pxe.bootfile = "/boot/grub2/grubx64.efi" @@ -394,17 +389,14 @@ func (pxe *PXE) setupArchDefaults(opts IsoTestOpts) error { } case "aarch64": pxe.boottype = "grub" - pxe.networkdevice = "virtio-net-pci" pxe.bootfile = "/boot/grub2/grubaa64.efi" pxe.pxeimagepath = "/boot/efi/EFI/fedora/grubaa64.efi" pxe.bootindex = "1" case "ppc64le": pxe.boottype = "grub" - pxe.networkdevice = "virtio-net-pci" pxe.bootfile = "/boot/grub2/powerpc-ieee1275/core.elf" case "s390x": pxe.boottype = "pxe" - pxe.networkdevice = "virtio-net-ccw" pxe.bootindex = "1" pxe.tftpipaddr = "10.0.2.2" default: From 779c267ac7c613280086fc5560e37f721f15a37f Mon Sep 17 00:00:00 2001 From: Nikita Dubrovskii Date: Tue, 14 Apr 2026 12:55:18 +0200 Subject: [PATCH 29/31] kola/tests/iso: add CheckLiveArtifactsExist helper function for ISO tests Add helper function in common.go to validate presence of live artifacts. --- mantle/kola/tests/iso/common.go | 7 +++++++ mantle/kola/tests/iso/live-as-disk.go | 3 +++ mantle/kola/tests/iso/live-fips.go | 3 +++ mantle/kola/tests/iso/live-iscsi.go | 3 +++ mantle/kola/tests/iso/live-iso.go | 3 +++ mantle/kola/tests/iso/live-login.go | 5 ++--- mantle/kola/tests/iso/live-pxe.go | 3 +++ 7 files changed, 24 insertions(+), 3 deletions(-) diff --git a/mantle/kola/tests/iso/common.go b/mantle/kola/tests/iso/common.go index f0c6d3781c..721e156722 100644 --- a/mantle/kola/tests/iso/common.go +++ b/mantle/kola/tests/iso/common.go @@ -130,6 +130,13 @@ func getIsoTestOpts(testName string) IsoTestOpts { return opts } +func CheckLiveArtifactsExist() error { + if kola.CosaBuild.Meta.BuildArtifacts.LiveIso == nil || kola.CosaBuild.Meta.BuildArtifacts.LiveKernel == nil || kola.CosaBuild.Meta.BuildArtifacts.LiveInitramfs == nil || kola.CosaBuild.Meta.BuildArtifacts.LiveRootfs == nil { + return fmt.Errorf("Build %s is missing live artifacts\n", kola.CosaBuild.Meta.Name) + } + return nil +} + func IsDevBuild() bool { // Ignore signing verification by default when running with development build // https://github.com/coreos/fedora-coreos-tracker/issues/908 diff --git a/mantle/kola/tests/iso/live-as-disk.go b/mantle/kola/tests/iso/live-as-disk.go index aec4084bfa..75c727129d 100644 --- a/mantle/kola/tests/iso/live-as-disk.go +++ b/mantle/kola/tests/iso/live-as-disk.go @@ -44,6 +44,9 @@ func init() { } func isoTestAsDisk(c cluster.TestCluster, opts IsoTestOpts) { + if err := CheckLiveArtifactsExist(); err != nil { + c.Fatal(err) + } qc, ok := c.Cluster.(*qemu.Cluster) if !ok { c.Fatalf("Unsupported cluster type") diff --git a/mantle/kola/tests/iso/live-fips.go b/mantle/kola/tests/iso/live-fips.go index b81ae524b8..41e3144ece 100644 --- a/mantle/kola/tests/iso/live-fips.go +++ b/mantle/kola/tests/iso/live-fips.go @@ -51,6 +51,9 @@ ExecStart=grep FIPS etc/crypto-policies/config RequiredBy=fips-signal-ok.service` func testLiveFIPS(c cluster.TestCluster, opts IsoTestOpts) { + if err := CheckLiveArtifactsExist(); err != nil { + c.Fatal(err) + } qc, ok := c.Cluster.(*qemu.Cluster) if !ok { c.Fatalf("Unsupported cluster type") diff --git a/mantle/kola/tests/iso/live-iscsi.go b/mantle/kola/tests/iso/live-iscsi.go index d8920e2cfd..16af70a56d 100644 --- a/mantle/kola/tests/iso/live-iscsi.go +++ b/mantle/kola/tests/iso/live-iscsi.go @@ -106,6 +106,9 @@ var iscsi_butane_config string // - when the system is booted, write a success string to /dev/virtio-ports/testisocompletion // - as this serial device is mapped to the host serial device, the test concludes func isoInstalliScsi(c cluster.TestCluster, opts IsoTestOpts) { + if err := CheckLiveArtifactsExist(); err != nil { + c.Fatal(err) + } qc, ok := c.Cluster.(*qemu.Cluster) if !ok { c.Fatalf("Unsupported cluster type") diff --git a/mantle/kola/tests/iso/live-iso.go b/mantle/kola/tests/iso/live-iso.go index 9014f9baa9..1a47c12aad 100644 --- a/mantle/kola/tests/iso/live-iso.go +++ b/mantle/kola/tests/iso/live-iso.go @@ -100,6 +100,9 @@ func init() { } func runLiveIsoInstallTest(c cluster.TestCluster, opts IsoTestOpts) { + if err := CheckLiveArtifactsExist(); err != nil { + c.Fatal(err) + } if opts.isMiniso && opts.isOffline { // ideally this'd be one enum parameter c.Fatal("Can't run minimal install offline") } diff --git a/mantle/kola/tests/iso/live-login.go b/mantle/kola/tests/iso/live-login.go index 8335f11c86..6bde0f843b 100644 --- a/mantle/kola/tests/iso/live-login.go +++ b/mantle/kola/tests/iso/live-login.go @@ -71,10 +71,9 @@ func init() { } func testLiveLogin(c cluster.TestCluster, firmware string) { - if kola.CosaBuild.Meta.BuildArtifacts.LiveIso == nil || kola.CosaBuild.Meta.BuildArtifacts.LiveKernel == nil { - c.Fatalf("Build %s is missing live artifacts\n", kola.CosaBuild.Meta.Name) + if err := CheckLiveArtifactsExist(); err != nil { + c.Fatal(err) } - butane := conf.Butane(` variant: fcos version: 1.1.0`) diff --git a/mantle/kola/tests/iso/live-pxe.go b/mantle/kola/tests/iso/live-pxe.go index ede1da53cd..c67aa201c9 100644 --- a/mantle/kola/tests/iso/live-pxe.go +++ b/mantle/kola/tests/iso/live-pxe.go @@ -94,6 +94,9 @@ RequiredBy=coreos-installer.target ` func testPXE(c cluster.TestCluster, opts IsoTestOpts) { + if err := CheckLiveArtifactsExist(); err != nil { + c.Fatal(err) + } qc, ok := c.Cluster.(*qemu.Cluster) if !ok { c.Fatalf("Unsupported cluster type") From 36269c0495d024f8ffef9a99547f3f8d6ce36d5a Mon Sep 17 00:00:00 2001 From: Nikita Dubrovskii Date: Mon, 20 Apr 2026 08:47:26 +0200 Subject: [PATCH 30/31] kola/tests/iso: align test function names to testLive* pattern --- mantle/kola/tests/iso/live-as-disk.go | 4 ++-- mantle/kola/tests/iso/live-iscsi.go | 4 ++-- mantle/kola/tests/iso/live-iso.go | 4 ++-- mantle/kola/tests/iso/live-pxe.go | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/mantle/kola/tests/iso/live-as-disk.go b/mantle/kola/tests/iso/live-as-disk.go index 75c727129d..321a9db966 100644 --- a/mantle/kola/tests/iso/live-as-disk.go +++ b/mantle/kola/tests/iso/live-as-disk.go @@ -31,7 +31,7 @@ func init() { register.RegisterTest(®ister.Test{ Run: func(c cluster.TestCluster) { opts := getIsoTestOpts(testName) - isoTestAsDisk(c, opts) + testLiveAsDisk(c, opts) }, ClusterSize: 0, Name: "iso." + testName, @@ -43,7 +43,7 @@ func init() { } } -func isoTestAsDisk(c cluster.TestCluster, opts IsoTestOpts) { +func testLiveAsDisk(c cluster.TestCluster, opts IsoTestOpts) { if err := CheckLiveArtifactsExist(); err != nil { c.Fatal(err) } diff --git a/mantle/kola/tests/iso/live-iscsi.go b/mantle/kola/tests/iso/live-iscsi.go index 16af70a56d..93cbb27a0a 100644 --- a/mantle/kola/tests/iso/live-iscsi.go +++ b/mantle/kola/tests/iso/live-iscsi.go @@ -65,7 +65,7 @@ func init() { register.RegisterTest(®ister.Test{ Run: func(c cluster.TestCluster) { opts := getIsoTestOpts(testName) - isoInstalliScsi(c, opts) + testLiveSCSI(c, opts) }, ClusterSize: 0, Name: "iso." + testName, @@ -105,7 +105,7 @@ var iscsi_butane_config string // 6 - /var/nested-ign.json contains an ignition config: // - when the system is booted, write a success string to /dev/virtio-ports/testisocompletion // - as this serial device is mapped to the host serial device, the test concludes -func isoInstalliScsi(c cluster.TestCluster, opts IsoTestOpts) { +func testLiveSCSI(c cluster.TestCluster, opts IsoTestOpts) { if err := CheckLiveArtifactsExist(); err != nil { c.Fatal(err) } diff --git a/mantle/kola/tests/iso/live-iso.go b/mantle/kola/tests/iso/live-iso.go index 1a47c12aad..6f9df9deb4 100644 --- a/mantle/kola/tests/iso/live-iso.go +++ b/mantle/kola/tests/iso/live-iso.go @@ -86,7 +86,7 @@ func init() { register.RegisterTest(®ister.Test{ Run: func(c cluster.TestCluster) { opts := getIsoTestOpts(testName) - runLiveIsoInstallTest(c, opts) + testLiveIso(c, opts) }, ClusterSize: 0, Name: "iso." + testName, @@ -99,7 +99,7 @@ func init() { } } -func runLiveIsoInstallTest(c cluster.TestCluster, opts IsoTestOpts) { +func testLiveIso(c cluster.TestCluster, opts IsoTestOpts) { if err := CheckLiveArtifactsExist(); err != nil { c.Fatal(err) } diff --git a/mantle/kola/tests/iso/live-pxe.go b/mantle/kola/tests/iso/live-pxe.go index c67aa201c9..f2ea8cbe83 100644 --- a/mantle/kola/tests/iso/live-pxe.go +++ b/mantle/kola/tests/iso/live-pxe.go @@ -65,7 +65,7 @@ func init() { register.RegisterTest(®ister.Test{ Run: func(c cluster.TestCluster) { opts := getIsoTestOpts(testName) - testPXE(c, opts) + testLivePXE(c, opts) }, ClusterSize: 0, Name: "iso." + testName, @@ -93,7 +93,7 @@ ExecStart=/bin/sh -c "/usr/bin/jq -er '.[\"build\"]? + .[\"version\"]? == \"%s\" RequiredBy=coreos-installer.target ` -func testPXE(c cluster.TestCluster, opts IsoTestOpts) { +func testLivePXE(c cluster.TestCluster, opts IsoTestOpts) { if err := CheckLiveArtifactsExist(); err != nil { c.Fatal(err) } From e5b1db503f63b831587ef19ac5db7babe06a9ee9 Mon Sep 17 00:00:00 2001 From: Nikita Dubrovskii Date: Mon, 20 Apr 2026 09:46:47 +0200 Subject: [PATCH 31/31] cmd-kola: add testiso to run iso.* translation wrapper Add translate_testiso_args() function to convert legacy 'kola testiso' command to new 'kola run iso.*' format. --- src/cmd-kola | 47 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/src/cmd-kola b/src/cmd-kola index 0b6d1c150c..2a17e96a91 100755 --- a/src/cmd-kola +++ b/src/cmd-kola @@ -19,6 +19,49 @@ from cosalib import cmdlib basearch = cmdlib.get_basearch() + +def translate_testiso_args(kolaargs, subargs, unknown_args) -> bool: + """ + Translate legacy 'kola testiso' command to 'kola run iso.*' format. + Modifies kolaargs in place. + """ + if not subargs or subargs[0] != 'testiso': + return False + + # Find and replace 'testiso' with 'run' + testiso_idx = kolaargs.index('testiso') + kolaargs[testiso_idx] = 'run' + + # Collect test patterns from unknown_args (non-flag arguments) + # Need to skip flag values by tracking when we see a flag that takes a value + test_patterns = [] + skip_next = False + flags_with_values = {'--denylist-test', '--tag', '--platform', '-p', '--parallel', '-j'} + + for arg in unknown_args: + if skip_next: + skip_next = False + continue + if arg.startswith('-'): + # Check if this flag takes a value + if any(arg.startswith(f) for f in flags_with_values): + skip_next = True + continue + # This is a test pattern + test_patterns.append(arg) + + if not test_patterns: + # No test pattern specified, default to all iso tests + kolaargs.append('iso.*') + else: + # Prepend 'iso.' to test patterns if not already present + for i in range(testiso_idx + 1, len(kolaargs)): + arg = kolaargs[i] + if arg in test_patterns and not arg.startswith('iso.'): + kolaargs[i] = f'iso.{arg}' + return True + + # Parse args and dispatch parser = argparse.ArgumentParser() parser.add_argument("--build", help="Build ID") @@ -54,6 +97,10 @@ subargs = args.subargs or [default_cmd] kolaargs.extend(subargs) kolaargs.extend(unknown_args) +# ISO tests were not part of kola before, so denylist by default to match old behavior. +# Allow via legacy "kola testiso" command or explicit "iso.*" patterns. +if not translate_testiso_args(kolaargs, subargs, unknown_args) and not any(arg.startswith('iso.') for arg in kolaargs): + kolaargs.extend(['--denylist-test', 'iso.*']) if args.upgrades: kolaargs.extend(['--output-dir', outputdir])