tests: run package tests in LLB graph

Move spec/package test execution out of the frontend's manual container-run path and into the normal LLB graph.

The previous test runner solved the package build, then used the BuildKit API to run test containers and inspect filesystem state from outside the graph. That made the build less lazy, introduced extra solve behavior, and broke clients such as `docker buildx dap build` that expect the frontend solve to remain debuggable as a normal graph.

This change represents test execution as BuildKit operations instead. The frontend binary is mounted into test containers and invoked through internal subcommands for filesystem checks. Each assertion is emitted as a separate exec op so failures can retain useful source mapping.

Running tests this way keeps the test work attached to the build graph, lets BuildKit schedule and cache it normally, and avoids forcing package outputs to be materialized just so the frontend can run tests manually.

Signed-off-by: Brian Goff <cpuguy83@gmail.com>

Author: cpuguy83

cmd/frontend/git_credential_helper.go

@@ -3,6 +3,7 @@ package main
 import (
 	"bufio"
 	"bytes"
+	"context"
 	"encoding/base64"
 	"flag"
 	"fmt"
@@ -11,9 +12,18 @@ import (
 	"path/filepath"
 	"slices"
 	"strings"
+
+	"github.com/project-dalec/dalec/internal/commands"
+	"github.com/project-dalec/dalec/internal/plugins"
 )
 
+func init() {
+	commands.RegisterPlugin(credHelperSubcmd, plugins.CmdHandlerFunc(credentialHelperCmd))
+}
+
 const (
+	credHelperSubcmd = "credential-helper"
+
 	keyProtocol          = "protocol"
 	keyHost              = "host"
 	keyPath              = "path"
@@ -72,7 +82,7 @@ type credConfig struct {
 	kind string
 }
 
-func gomodMain(args []string) {
+func credentialHelperCmd(ctx context.Context, args []string) {
 	var cfg credConfig
 	fs := flag.NewFlagSet(credHelperSubcmd, flag.ExitOnError)
 	fs.Func("kind", "the kind of secret to retrieve (token or header)", readKind(&cfg.kind))

cmd/frontend/main.go

@@ -1,25 +1,28 @@
 package main
 
 import (
+	"context"
 	_ "embed"
 	"flag"
 	"fmt"
 	"os"
+	"path/filepath"
 
+	"github.com/containerd/plugin"
 	"github.com/moby/buildkit/frontend/gateway/grpcclient"
 	"github.com/moby/buildkit/util/appcontext"
 	"github.com/moby/buildkit/util/bklog"
 	"github.com/project-dalec/dalec/frontend"
 	"github.com/project-dalec/dalec/internal/frontendapi"
-	"github.com/project-dalec/dalec/internal/testrunner"
+	"github.com/project-dalec/dalec/internal/plugins"
 	"github.com/sirupsen/logrus"
 	"google.golang.org/grpc/grpclog"
+
+	_ "github.com/project-dalec/dalec/internal/commands"
 )
 
 const (
 	Package = "github.com/project-dalec/dalec/cmd/frontend"
-
-	credHelperSubcmd = "credential-helper"
 )
 
 func init() {
@@ -28,43 +31,62 @@ func init() {
 }
 
 func main() {
-	fs := flag.CommandLine
-	fs.Usage = func() {
-		fmt.Fprintf(os.Stderr, `usage: %s [subcommand [args...]]`, os.Args[0])
-	}
+	flags := flag.NewFlagSet(filepath.Base(os.Args[0]), flag.ExitOnError)
 
-	if err := fs.Parse(os.Args); err != nil {
+	if err := flags.Parse(os.Args[1:]); err != nil {
 		bklog.L.WithError(err).Fatal("error parsing frontend args")
 		os.Exit(70) // 70 is EX_SOFTWARE, meaning internal software error occurred
+	}
 
+	ctx := appcontext.Context()
+	if flags.NArg() == 0 {
+		dalecMain(ctx)
+		return
 	}
 
-	subCmd := fs.Arg(1)
-
-	// NOTE: for subcommands we take args[2:]
-	// skip args[0] (the executable) and args[1] (the subcommand)
-
-	// each "sub-main" function handles its own exit
-	switch subCmd {
-	case credHelperSubcmd:
-		args := flag.Args()[2:]
-		gomodMain(args)
-	case testrunner.StepRunnerCmdName:
-		args := flag.Args()[2:]
-		testrunner.StepCmd(args)
-	case testrunner.CheckFilesCmdName:
-		args := flag.Args()[2:]
-		testrunner.CheckFilesCmd(args)
-	case frontend.TestErrorCmdName:
-		args := flag.Args()[2:]
-		frontend.TestErrorCmd(args)
-	default:
-		dalecMain()
+	h, err := lookupCmd(ctx, flags.Arg(0))
+	if err != nil {
+		bklog.L.WithError(err).Fatal("error handling command")
+		os.Exit(70) // 70 is EX_SOFTWARE, meaning internal software error occurred
+	}
+
+	if h == nil {
+		fmt.Fprintln(os.Stderr, "unknown subcommand:", flags.Arg(1))
+		fmt.Fprintln(os.Stderr, "full args:", flag.Args())
+		fmt.Fprintln(os.Stderr, "If you see this message this is probably a bug in dalec.")
+		os.Exit(64) // 64 is EX_USAGE, meaning command line usage error
 	}
+
+	h.HandleCmd(ctx, flags.Args()[1:])
 }
 
-func dalecMain() {
-	ctx := appcontext.Context()
+func lookupCmd(ctx context.Context, cmd string) (plugins.CmdHandler, error) {
+	set := plugin.NewPluginSet()
+	filter := func(r *plugins.Registration) bool {
+		return r.Type != plugins.TypeCmd || r.ID != cmd
+	}
+
+	for _, r := range plugins.Graph(filter) {
+		cfg := plugin.NewContext(ctx, set, nil)
+
+		p := r.Init(cfg)
+
+		v, err := p.Instance()
+		if plugin.IsSkipPlugin(err) {
+			continue
+		}
+
+		if err != nil {
+			return nil, err
+		}
+
+		return v.(plugins.CmdHandler), nil
+	}
+
+	return nil, nil
+}
+
+func dalecMain(ctx context.Context) {
 	mux, err := frontendapi.NewBuildRouter(ctx)
 	if err != nil {
 		bklog.L.WithError(err).Fatal("error creating frontend router")

frontend/build.go

@@ -54,7 +54,7 @@ func LoadSpec(ctx context.Context, client *dockerui.Client, platform *ocispecs.P
 		o(&cfg)
 	}
 
-	src, err := client.ReadEntrypoint(ctx, "Dockerfile")
+	src, err := client.ReadEntrypoint(ctx, "dalec")
 	if err != nil {
 		return nil, fmt.Errorf("could not read spec file: %w", err)
 	}
@@ -218,26 +218,26 @@ func (c *clientWithPlatform) BuildOpts() gwclient.BuildOpts {
 	return opts
 }
 
-func GetCurrentFrontend(client gwclient.Client) (llb.State, error) {
+func GetCurrentFrontend(client gwclient.Client) llb.State {
+	return *getCurrentFrontend(client)
+}
+
+func getCurrentFrontend(client gwclient.Client) *llb.State {
 	f, err := client.(frontendClient).CurrentFrontend()
 	if err != nil {
-		return llb.Scratch(), err
+		panic(err)
 	}
 
 	if f == nil {
-		return llb.Scratch(), fmt.Errorf("nil frontend state returned")
+		panic("nil frontend state returned -- this should never happen")
 	}
 
-	return *f, nil
+	return f
 }
 
 func withCredHelper(c gwclient.Client) func() (llb.RunOption, error) {
 	return func() (llb.RunOption, error) {
-		f, err := GetCurrentFrontend(c)
-		if err != nil {
-			return nil, err
-		}
-
+		f := GetCurrentFrontend(c)
 		return dalec.RunOptFunc(func(ei *llb.ExecInfo) {
 			llb.AddMount("/usr/local/bin/frontend", f, llb.SourcePath("/frontend")).SetRunOption(ei)
 		}), nil

frontend/request.go

@@ -203,7 +203,7 @@ func MaybeSign(ctx context.Context, client gwclient.Client, st llb.State, spec *
 		Warnf(ctx, client, st, "Spec signing config overwritten by config at path %q in build-context %q", cfgPath, configCtxName)
 	}
 
-	cfg, err := getSigningConfigFromContext(ctx, client, cfgPath, configCtxName, sOpt)
+	cfg, err := getSigningConfigFromContext(ctx, client, cfgPath, configCtxName, sOpt, opts...)
 	if err != nil {
 		return dalec.ErrorState(llb.Scratch(), err)
 	}