From 16f585a1315b77b4789fcf805406540351e702cf Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 5 Jun 2025 16:56:35 -0400 Subject: [PATCH 1/3] kolet: Capture error messages from mkfifo The Go `exec.Command` when used naively like this captures stderr and then it gets lost. It turned out with soft reboot it was this `mkfifo` that was failing but all we got is "Child process exited with code 1" but there are like 5 different processes on two different hosts that could be talking about, and we *really* thought it was talking about the code-under-test, not the framework setting it up. --- mantle/cmd/kolet/kolet.go | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/mantle/cmd/kolet/kolet.go b/mantle/cmd/kolet/kolet.go index 0093a225d7..999698795f 100644 --- a/mantle/cmd/kolet/kolet.go +++ b/mantle/cmd/kolet/kolet.go @@ -259,6 +259,16 @@ func initiateReboot(mark string) error { return nil } +func mkfifo(path string) error { + c := exec.Command("mkfifo", path) + c.Stderr = os.Stderr + err := c.Run() + if err != nil { + return fmt.Errorf("creating fifo %s: %w", path, err) + } + return nil +} + func runExtUnit(cmd *cobra.Command, args []string) error { rebootOff, _ := cmd.Flags().GetBool("deny-reboots") // Write the autopkgtest wrappers @@ -276,7 +286,7 @@ func runExtUnit(cmd *cobra.Command, args []string) error { // We want to prevent certain tests (like non-exclusive tests) from rebooting if !rebootOff { - err := exec.Command("mkfifo", rebootRequestFifo).Run() + err := mkfifo(rebootRequestFifo) if err != nil { return err } @@ -366,7 +376,7 @@ func runReboot(cmd *cobra.Command, args []string) error { mark := args[0] systemdjournal.Print(systemdjournal.PriInfo, "Requesting reboot with mark: %s", mark) - err := exec.Command("mkfifo", kola.KoletRebootAckFifo).Run() + err := mkfifo(kola.KoletRebootAckFifo) if err != nil { return err } From 40ac8638495ba358d93c744f5a3df150e23c94c9 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 5 Jun 2025 16:58:35 -0400 Subject: [PATCH 2/3] kola: Don't drop stderr on the floor from unit starting function Amazingly we had *two* places that dropped stderr, this was the second. The default `c.SSH` captures logs from test failures but this was an *infra* failure so we need to drop to the raw ssh tool. (This could use a big cleanup but that's...a bigger project) --- mantle/kola/harness.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/mantle/kola/harness.go b/mantle/kola/harness.go index 2b74f9f729..e51023c9c2 100644 --- a/mantle/kola/harness.go +++ b/mantle/kola/harness.go @@ -1100,7 +1100,6 @@ func metadataFromTestBinary(executable string) (*externalTestMeta, error) { // handling. func runExternalTest(c cluster.TestCluster, mach platform.Machine, testNum int) error { var previousRebootState string - var stdout []byte for { bootID, err := platform.GetMachineBootId(mach) if err != nil { @@ -1126,11 +1125,11 @@ func runExternalTest(c cluster.TestCluster, mach platform.Machine, testNum int) unit := fmt.Sprintf("%s.service", KoletExtTestUnit) cmd = fmt.Sprintf("sudo /usr/local/bin/kolet run-test-unit %s", shellquote.Join(unit)) } - stdout, err = c.SSH(mach, cmd) - + stdout, stderr, err := mach.SSH(cmd) if err != nil { - return errors.Wrapf(err, "kolet run-test-unit failed") + return errors.Wrapf(err, "kolet run-test-unit failed: %s %s", string(stdout), string(stderr)) } + koletRes := KoletResult{} if len(stdout) > 0 { err = json.Unmarshal(stdout, &koletRes) From 67278c14b7f04fd77e8a76451ae95b3e729782da Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 5 Jun 2025 16:59:56 -0400 Subject: [PATCH 3/3] mantle: Log commands executed over ssh and return code This definitely adds some chatter at debug level but I think it's really worth it. --- mantle/platform/cluster.go | 1 + 1 file changed, 1 insertion(+) diff --git a/mantle/platform/cluster.go b/mantle/platform/cluster.go index e03f0f931a..a574850694 100644 --- a/mantle/platform/cluster.go +++ b/mantle/platform/cluster.go @@ -109,6 +109,7 @@ func (bc *BaseCluster) SSH(m Machine, cmd string) ([]byte, []byte, error) { session.Stdout = &stdout session.Stderr = &stderr err = session.Run(cmd) + plog.Debugf("Running cmd=%v res=%v", cmd, err) outBytes := bytes.TrimSpace(stdout.Bytes()) errBytes := bytes.TrimSpace(stderr.Bytes()) return outBytes, errBytes, err