diff --git a/determinism_test.go b/determinism_test.go index 2af9d5772..3292a2571 100644 --- a/determinism_test.go +++ b/determinism_test.go @@ -222,7 +222,12 @@ func TestGomodPatchEnvProducesDeterministicLLB(t *testing.T) { worker := llb.Scratch() build := func() llb.State { - st, err := spec.generateGomodPatchStateForSource("src", gen, base, worker, nil) + st, err := spec.generateGomodPatchStateForSource(gomodGeneratorOpts{ + sourceName: "src", + gen: gen, + sourceState: base, + worker: worker, + }) assert.NilError(t, err) assert.Assert(t, st != nil) return *st diff --git a/frontend/gateway.go b/frontend/gateway.go index 61ba78e7d..d387523ce 100644 --- a/frontend/gateway.go +++ b/frontend/gateway.go @@ -134,6 +134,13 @@ func SourceOptFromUIClient(ctx context.Context, c gwclient.Client, dc *dockerui. return loadSourceFilterConfig(ctx, c, sOpt.GetContext) }) + if gomodProxy, ok := GetBuildArg(c, dalec.BuildArgDalecGomodProxy); ok { + if sOpt.ExtraEnvs == nil { + sOpt.ExtraEnvs = map[string]string{} + } + sOpt.ExtraEnvs["GOPROXY"] = gomodProxy + } + return sOpt } diff --git a/frontend/gateway_test.go b/frontend/gateway_test.go new file mode 100644 index 000000000..732687deb --- /dev/null +++ b/frontend/gateway_test.go @@ -0,0 +1,20 @@ +package frontend + +import ( + "testing" + + "github.com/project-dalec/dalec" +) + +func TestSourceOptFromUIClientReadsGomodProxyBuildArgIntoExtraEnvs(t *testing.T) { + t.Parallel() + + const proxy = "http://proxy.example:5000,direct" + client := newStubClient() + client.opts["build-arg:"+dalec.BuildArgDalecGomodProxy] = proxy + + sOpt := SourceOptFromUIClient(t.Context(), client, nil, nil) + if sOpt.ExtraEnvs["GOPROXY"] != proxy { + t.Fatalf("expected GOPROXY extra env %q, got %q", proxy, sOpt.ExtraEnvs["GOPROXY"]) + } +} diff --git a/generator_gomod.go b/generator_gomod.go index 6153b9b48..dc4c5eb9d 100644 --- a/generator_gomod.go +++ b/generator_gomod.go @@ -19,6 +19,9 @@ const ( // GoModCacheKey is the key used to identify the go module cache in the buildkit cache. // It is exported only for testing purposes. GomodCacheKey = "dalec-gomod-proxy-cache" + // BuildArgDalecGomodProxy is the frontend build arg used to override GOPROXY + // for gomod dependency generation. + BuildArgDalecGomodProxy = "DALEC_GOMOD_PROXY" ) func (g *GeneratorGomod) processBuildArgs(args map[string]string, allowArg func(key string) bool) error { @@ -63,7 +66,24 @@ func (s *Spec) HasGomods() bool { return false } -func withGomod(g *SourceGenerator, srcSt, worker llb.State, subPath string, credHelper llb.RunOption, opts ...llb.ConstraintsOpt) func(llb.State) llb.State { +type gomodGeneratorOpts struct { + sourceName string + gen *SourceGenerator + sourceState llb.State + worker llb.State + credHelper llb.RunOption + extraEnvs map[string]string + constraints []llb.ConstraintsOpt +} + +func (opts gomodGeneratorOpts) gomodProxy() string { + if opts.extraEnvs == nil { + return "" + } + return opts.extraEnvs["GOPROXY"] +} + +func withGomod(gomodOpts gomodGeneratorOpts) func(llb.State) llb.State { return func(in llb.State) llb.State { const ( workDir = "/work/src" @@ -71,8 +91,10 @@ func withGomod(g *SourceGenerator, srcSt, worker llb.State, subPath string, cred gomodDownloadWrapperBasename = "go_mod_download.sh" ) - joinedWorkDir := filepath.Join(workDir, subPath, g.Subpath) - srcMount := llb.AddMount(workDir, srcSt) + g := gomodOpts.gen + opts := gomodOpts.constraints + joinedWorkDir := filepath.Join(workDir, gomodOpts.sourceName, g.Subpath) + srcMount := llb.AddMount(workDir, gomodOpts.sourceState) paths := g.Gomod.Paths if g.Gomod.Paths == nil { @@ -86,16 +108,16 @@ func withGomod(g *SourceGenerator, srcSt, worker llb.State, subPath string, cred scriptPath := filepath.Join(scriptMountpoint, gomodDownloadWrapperBasename) for _, path := range paths { - in = worker.Run( + runOpts := []llb.RunOption{ // First download the go module deps to our persistent cache // Then set the GOPROXY to the cache dir so that we can extract just the deps we need // This allows us to persist the module cache across builds and avoid downloading // the same deps over and over again. - ShArgs(`set -e; GOMODCACHE="${TMP_GOMODCACHE}" `+scriptPath+`; GOPROXY="file://${TMP_GOMODCACHE}/cache/download" `+scriptPath), + ShArgs(`set -e; GOMODCACHE="${TMP_GOMODCACHE}" ` + scriptPath + `; GOPROXY="file://${TMP_GOMODCACHE}/cache/download" ` + scriptPath), g.withGomodSecretsAndSockets(), llb.AddMount(scriptMountpoint, script), llb.AddEnv("GOPATH", "/go"), - credHelper, + gomodOpts.credHelper, llb.AddEnv("TMP_GOMODCACHE", proxyPath), llb.AddEnv("GIT_SSH_COMMAND", "ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no"), llb.Dir(filepath.Join(joinedWorkDir, path)), @@ -103,6 +125,13 @@ func withGomod(g *SourceGenerator, srcSt, worker llb.State, subPath string, cred llb.AddMount(proxyPath, llb.Scratch(), llb.AsPersistentCacheDir(GomodCacheKey, llb.CacheMountShared)), WithConstraints(opts...), g.Gomod._sourceMap.GetLocation(in), + } + if gomodProxy := gomodOpts.gomodProxy(); gomodProxy != "" { + runOpts = append(runOpts, llb.AddEnv("GOPROXY", gomodProxy)) + } + + in = gomodOpts.worker.Run( + runOpts..., ).AddMount(gomodCacheDir, in) } @@ -238,7 +267,15 @@ func (s *Spec) GomodDeps(sOpt SourceOpts, worker llb.State, opts ...llb.Constrai deps = deps.With(func(in llb.State) llb.State { for _, gen := range src.Generate { if gen.Gomod != nil { - in = in.With(withGomod(gen, patched[key], worker, key, credHelperRunOpt, opts...)) + in = in.With(withGomod(gomodGeneratorOpts{ + sourceName: key, + gen: gen, + sourceState: patched[key], + worker: worker, + credHelper: credHelperRunOpt, + extraEnvs: sOpt.ExtraEnvs, + constraints: opts, + })) } } return in diff --git a/generator_gomod_test.go b/generator_gomod_test.go index 36a50ad16..039c3c302 100644 --- a/generator_gomod_test.go +++ b/generator_gomod_test.go @@ -1,10 +1,14 @@ package dalec import ( + "context" "encoding/json" + "slices" + "strings" "testing" "github.com/goccy/go-yaml" + "github.com/moby/buildkit/client/llb" ) func TestGomodReplaceUnmarshal(t *testing.T) { @@ -296,3 +300,158 @@ func TestGomodEditArgs_DropRequire(t *testing.T) { }) } } + +func TestGomodDepsUsesGomodProxy(t *testing.T) { + t.Parallel() + + const proxy = "http://proxy.example:5000,direct" + spec := testGomodProxySpec() + st := spec.GomodDeps(testGomodProxySourceOpts(proxy), llb.Scratch()) + if st == nil { + t.Fatal("gomod generator succeeded but returned nil state") + } + + env := gomodDownloadExecEnv(context.Background(), t, *st) + if !slices.Contains(env, "GOPROXY="+proxy) { + t.Fatalf("expected gomod exec env to include GOPROXY=%q, got %v", proxy, env) + } +} + +func TestGomodDepsSkipsEmptyGomodProxy(t *testing.T) { + t.Parallel() + + spec := testGomodProxySpec() + st := spec.GomodDeps(testGomodProxySourceOpts(""), llb.Scratch()) + if st == nil { + t.Fatal("gomod generator succeeded but returned nil state") + } + + env := gomodDownloadExecEnv(context.Background(), t, *st) + for _, item := range env { + if strings.HasPrefix(item, "GOPROXY=") { + t.Fatalf("expected empty gomod proxy to omit GOPROXY, got %v", env) + } + } +} + +func TestGomodProxyBuildArgIsKnown(t *testing.T) { + t.Parallel() + + spec := &Spec{} + err := spec.SubstituteArgs(map[string]string{ + BuildArgDalecGomodProxy: "http://proxy.example:5000", + }) + if err != nil { + t.Fatal(err) + } +} + +func TestGomodPatchUsesGomodProxy(t *testing.T) { + t.Parallel() + + const proxy = "http://proxy.example:5000,direct" + gen := &SourceGenerator{ + Gomod: &GeneratorGomod{ + Edits: &GomodEdits{ + Replace: []GomodReplace{ + {Original: "example.com/old", Update: "example.com/new v1.2.3"}, + }, + }, + }, + } + + st, err := (&Spec{}).generateGomodPatchStateForSource(gomodGeneratorOpts{ + sourceName: "src", + gen: gen, + sourceState: llb.Scratch(), + worker: llb.Scratch(), + extraEnvs: map[string]string{ + "GOPROXY": proxy, + }, + }) + if err != nil { + t.Fatal(err) + } + if st == nil { + t.Fatal("gomod patch generation succeeded but returned nil state") + } + + env := gomodPatchExecEnv(context.Background(), t, *st) + if !slices.Contains(env, "GOPROXY="+proxy) { + t.Fatalf("expected gomod patch exec env to include GOPROXY=%q, got %v", proxy, env) + } +} + +func testGomodProxySpec() *Spec { + return &Spec{ + Sources: map[string]Source{ + "src": { + Git: &SourceGit{ + URL: "https://example.com/repo.git", + Commit: "0123456789abcdef", + }, + Generate: []*SourceGenerator{ + {Gomod: &GeneratorGomod{}}, + }, + }, + }, + } +} + +func testGomodProxySourceOpts(proxy string) SourceOpts { + sOpt := SourceOpts{ + GetContext: func(name string, opts ...llb.LocalOption) (*llb.State, error) { + st := llb.Local(name, opts...) + return &st, nil + }, + GitCredHelperOpt: func() (llb.RunOption, error) { + st := llb.Scratch().File(llb.Mkfile("/frontend", 0o755, []byte("#!/usr/bin/env bash\nexit 0\n"))) + return RunOptFunc(func(ei *llb.ExecInfo) { + llb.AddMount("/usr/local/bin/frontend", st, llb.SourcePath("/frontend")).SetRunOption(ei) + }), nil + }, + } + if proxy != "" { + sOpt.ExtraEnvs = map[string]string{ + "GOPROXY": proxy, + } + } + return sOpt +} + +func gomodDownloadExecEnv(ctx context.Context, t *testing.T, st llb.State) []string { + t.Helper() + + for _, op := range sourceOpsFromState(ctx, t, st) { + exec := op.GetExec() + if exec == nil { + continue + } + + env := exec.Meta.Env + if slices.Contains(env, "GOPATH=/go") && slices.Contains(env, "TMP_GOMODCACHE=/tmp/dalec/gomod-proxy-cache") { + return env + } + } + + t.Fatal("expected gomod download exec") + return nil +} + +func gomodPatchExecEnv(ctx context.Context, t *testing.T, st llb.State) []string { + t.Helper() + + for _, op := range sourceOpsFromState(ctx, t, st) { + exec := op.GetExec() + if exec == nil { + continue + } + + if slices.Contains(exec.Meta.Args, "/gomod-patch.sh") { + return exec.Meta.Env + } + } + + t.Fatal("expected gomod patch exec") + return nil +} diff --git a/load.go b/load.go index c5c2ca6ea..88b19e896 100644 --- a/load.go +++ b/load.go @@ -53,6 +53,8 @@ func knownArg(key string) bool { return true case "DALEC_SOURCE_FILTER_CONFIG_CONTEXT_NAME": return true + case BuildArgDalecGomodProxy: + return true case KeyDalecTarget: return true } diff --git a/preprocess.go b/preprocess.go index a953180f7..a2bb8b22e 100644 --- a/preprocess.go +++ b/preprocess.go @@ -70,7 +70,15 @@ func (s *Spec) preprocessGomodEdits(sOpt SourceOpts, worker llb.State, opts ...l } // Generate patch state (LLB state, not solved bytes) - patchSt, err := s.generateGomodPatchStateForSource(sourceName, gen, baseState, worker, credHelper, opts...) + patchSt, err := s.generateGomodPatchStateForSource(gomodGeneratorOpts{ + sourceName: sourceName, + gen: gen, + sourceState: baseState, + worker: worker, + credHelper: credHelper, + extraEnvs: sOpt.ExtraEnvs, + constraints: opts, + }) if err != nil { return errors.Wrapf(err, "failed to generate gomod patch state for source %s", sourceName) } @@ -234,7 +242,11 @@ func buildGomodPatchEnv(editArgs string, paths []string, gen *SourceGenerator, s // generateGomodPatchStateForSource generates a single merged patch LLB state for all paths // in a gomod generator by running go mod edit + tidy and capturing the diff. // Returns the LLB state containing the patch file, or nil if no changes are needed. -func (s *Spec) generateGomodPatchStateForSource(sourceName string, gen *SourceGenerator, baseState llb.State, worker llb.State, credHelper llb.RunOption, opts ...llb.ConstraintsOpt) (*llb.State, error) { +func (s *Spec) generateGomodPatchStateForSource(gomodOpts gomodGeneratorOpts) (*llb.State, error) { + sourceName := gomodOpts.sourceName + gen := gomodOpts.gen + opts := gomodOpts.constraints + editArgs, err := gomodEditArgs(gen.Gomod) if err != nil { return nil, err @@ -278,8 +290,8 @@ func (s *Spec) generateGomodPatchStateForSource(sourceName string, gen *SourceGe runOpts := []llb.RunOption{ llb.Args([]string{"/gomod-patch.sh"}), llb.AddMount("/gomod-patch.sh", scriptState, llb.SourcePath("/gomod-patch.sh")), - llb.AddMount(workDir, baseState), - llb.AddMount(origWorkDir, baseState, llb.Readonly), // Read-only mount for diffing + llb.AddMount(workDir, gomodOpts.sourceState), + llb.AddMount(origWorkDir, gomodOpts.sourceState, llb.Readonly), // Read-only mount for diffing llb.AddMount(proxyPath, llb.Scratch(), llb.AsPersistentCacheDir(GomodCacheKey, llb.CacheMountShared)), llb.AddMount(patchOutputDir, patchOutput), // Mount scratch state to capture patch file llb.AddEnv("GOPATH", "/go"), @@ -288,13 +300,17 @@ func (s *Spec) generateGomodPatchStateForSource(sourceName string, gen *SourceGe WithConstraints(opts...), } + if gomodProxy := gomodOpts.gomodProxy(); gomodProxy != "" { + runOpts = append(runOpts, llb.AddEnv("GOPROXY", gomodProxy)) + } + // Add environment variables from the script for key, value := range SortedMapIter(envVars) { runOpts = append(runOpts, llb.AddEnv(key, value)) } - if credHelper != nil { - runOpts = append(runOpts, credHelper) + if gomodOpts.credHelper != nil { + runOpts = append(runOpts, gomodOpts.credHelper) } if secretOpt := gen.withGomodSecretsAndSockets(); secretOpt != nil { runOpts = append(runOpts, secretOpt) @@ -304,7 +320,7 @@ func (s *Spec) generateGomodPatchStateForSource(sourceName string, gen *SourceGe // The AddMount call returns the state of the patchOutput scratch. // Since we mounted at patchOutputDir and wrote to patchPath, // the file in the mount will be at gomodPatchFilename (path relative to mount point) - patchMount := worker.Run(runOpts...).AddMount(patchOutputDir, patchOutput) + patchMount := gomodOpts.worker.Run(runOpts...).AddMount(patchOutputDir, patchOutput) // Create a scratch state with the patch file at a generic location // The sourceFilters will handle renaming it to the final source name diff --git a/source.go b/source.go index b370f5676..8220ded44 100644 --- a/source.go +++ b/source.go @@ -155,6 +155,9 @@ type SourceOpts struct { TargetPlatform *ocispecs.Platform GitCredHelperOpt func() (llb.RunOption, error) SourceFilter func() (SourceFilterConfig, error) + // ExtraEnvs contains environment variables that source generators may use + // while preparing generated dependency sources. + ExtraEnvs map[string]string } var errInvalidMountConfig = errors.New("invalid mount config")