From efcebc917dc50feede2d5ff9d18209015611e81f Mon Sep 17 00:00:00 2001 From: Brian Goff Date: Thu, 14 May 2026 14:06:26 -0700 Subject: [PATCH] frontend: add build-time source filters Add a source-options context that can provide a YAML source filter config at build time. The config currently supports global_excludes which can be used to filter out paths from all sources. Signed-off-by: Brian Goff --- frontend/debug/handle_sources.go | 4 +- frontend/gateway.go | 8 +- frontend/request.go | 70 +++++++++--- generator_cargohome.go | 1 + generator_gomod.go | 1 + generator_nodemodules.go | 2 +- generator_pip.go | 1 + load.go | 4 + packaging/linux/rpm/buildroot.go | 12 +- packaging/linux/rpm/template.go | 52 ++++++++- packaging/linux/rpm/template_test.go | 60 +++++++++- source.go | 46 +++++++- source_filter.go | 96 ++++++++++++++++ source_inline.go | 20 +++- source_test.go | 161 +++++++++++++++++++++++++++ spec_test.go | 20 ++++ test/source_test.go | 109 ++++++++++++++++++ website/docs/sources.md | 33 ++++++ 18 files changed, 667 insertions(+), 33 deletions(-) create mode 100644 source_filter.go diff --git a/frontend/debug/handle_sources.go b/frontend/debug/handle_sources.go index 9db18a67a..f09a42fe8 100644 --- a/frontend/debug/handle_sources.go +++ b/frontend/debug/handle_sources.go @@ -18,10 +18,10 @@ func Sources(ctx context.Context, client gwclient.Client) (*gwclient.Result, err return nil, nil, err } - sources := dalec.Sources(spec, sOpt) - pg := dalec.ProgressGroup("Sources for " + targetKey + " rpm build: " + spec.Name) + sources := dalec.Sources(spec, sOpt, pg) + def, err := dalec.MergeAtPath(llb.Scratch(), dalec.SortedMapValues(sources), "/", pg).Marshal(ctx) if err != nil { return nil, nil, err diff --git a/frontend/gateway.go b/frontend/gateway.go index c2ab35a30..6037550cd 100644 --- a/frontend/gateway.go +++ b/frontend/gateway.go @@ -103,7 +103,7 @@ func GetBuildArg(client gwclient.Client, k string) (string, bool) { } func SourceOptFromUIClient(ctx context.Context, c gwclient.Client, dc *dockerui.Client, platform *ocispecs.Platform) dalec.SourceOpts { - return dalec.SourceOpts{ + sOpt := dalec.SourceOpts{ TargetPlatform: platform, Resolver: c, Forward: ForwarderFromClient(ctx, c), @@ -129,6 +129,12 @@ func SourceOptFromUIClient(ctx context.Context, c gwclient.Client, dc *dockerui. }, GitCredHelperOpt: withCredHelper(c), } + + sOpt.SourceFilter = sync.OnceValues(func() (dalec.SourceFilterConfig, error) { + return loadSourceFilterConfig(ctx, c, sOpt.GetContext) + }) + + return sOpt } func SourceOptFromClient(ctx context.Context, c gwclient.Client, platform *ocispecs.Platform) (dalec.SourceOpts, error) { diff --git a/frontend/request.go b/frontend/request.go index ea25a5731..6b799281e 100644 --- a/frontend/request.go +++ b/frontend/request.go @@ -142,10 +142,43 @@ func marshalDockerfile(ctx context.Context, dt []byte, opts ...llb.ConstraintsOp } func getSigningConfigFromContext(ctx context.Context, client gwclient.Client, cfgPath string, configCtxName string, sOpt dalec.SourceOpts, opts ...llb.ConstraintsOpt) (*dalec.PackageSigner, error) { + dt, err := readConfigFromContext(ctx, client, cfgPath, configCtxName, sOpt, opts...) + if err != nil { + return nil, err + } + + var pc dalec.PackageConfig + if err := yaml.Unmarshal(dt, &pc); err != nil { + return nil, err + } + + return pc.Signer, nil +} + +func getSourceFilterConfigFromContext(ctx context.Context, client gwclient.Client, cfgPath string, configCtxName string, getContext func(string, ...llb.LocalOption) (*llb.State, error), opts ...llb.ConstraintsOpt) (dalec.SourceFilterConfig, error) { + dt, err := readConfigFromContext(ctx, client, cfgPath, configCtxName, dalec.SourceOpts{ + GetContext: getContext, + }, opts...) + if err != nil { + return dalec.SourceFilterConfig{}, err + } + + return decodeSourceFilterConfig(ctx, dt) +} + +func decodeSourceFilterConfig(ctx context.Context, dt []byte) (dalec.SourceFilterConfig, error) { + var cfg dalec.SourceFilterConfig + if err := yaml.UnmarshalContext(ctx, dt, &cfg, yaml.Strict()); err != nil { + return dalec.SourceFilterConfig{}, err + } + return cfg, nil +} + +func readConfigFromContext(ctx context.Context, client gwclient.Client, cfgPath string, configCtxName string, sOpt dalec.SourceOpts, opts ...llb.ConstraintsOpt) ([]byte, error) { src := dalec.Source{Path: cfgPath, Context: &dalec.SourceContext{Name: configCtxName}} - signConfigState := src.ToState("", sOpt, opts...) + configState := src.ToState("", dalec.SourceOpts{GetContext: sOpt.GetContext}, opts...) - scDef, err := signConfigState.Marshal(ctx) + scDef, err := configState.Marshal(ctx) if err != nil { return nil, err } @@ -162,19 +195,9 @@ func getSigningConfigFromContext(ctx context.Context, client gwclient.Client, cf return nil, err } - dt, err := ref.ReadFile(ctx, gwclient.ReadRequest{ + return ref.ReadFile(ctx, gwclient.ReadRequest{ Filename: cfgPath, }) - if err != nil { - return nil, err - } - - var pc dalec.PackageConfig - if err := yaml.Unmarshal(dt, &pc); err != nil { - return nil, err - } - - return pc.Signer, nil } func MaybeSign(ctx context.Context, client gwclient.Client, st llb.State, spec *dalec.Spec, targetKey string, sOpt dalec.SourceOpts, opts ...llb.ConstraintsOpt) llb.State { @@ -242,6 +265,27 @@ func getSignConfigCtxName(client gwclient.Client) string { return client.BuildOpts().Opts["build-arg:"+buildArgDalecSigningConfigContextName] } +func getSourceFilterConfigPath(client gwclient.Client) string { + return client.BuildOpts().Opts["build-arg:"+dalec.BuildArgDalecSourceFilterConfigPath] +} + +func getSourceFilterContextNameWithDefault(client gwclient.Client) string { + configCtxName := dalec.DefaultSourceOptionsContextName + if cn := client.BuildOpts().Opts["build-arg:"+dalec.BuildArgDalecSourceFilterContextName]; cn != "" { + configCtxName = cn + } + return configCtxName +} + +func loadSourceFilterConfig(ctx context.Context, client gwclient.Client, getContext func(string, ...llb.LocalOption) (*llb.State, error)) (dalec.SourceFilterConfig, error) { + cfgPath := getSourceFilterConfigPath(client) + if cfgPath == "" { + return dalec.SourceFilterConfig{}, nil + } + + return getSourceFilterConfigFromContext(ctx, client, cfgPath, getSourceFilterContextNameWithDefault(client), getContext) +} + func forwardToSigner(ctx context.Context, client gwclient.Client, cfg *dalec.PackageSigner, s llb.State, opts ...llb.ConstraintsOpt) (llb.State, error) { const ( // See https://github.com/moby/buildkit/blob/d8d946b85c52095d34a52ce210960832f4e06775/frontend/dockerui/context.go#L29 diff --git a/generator_cargohome.go b/generator_cargohome.go index 90cae7830..fa92b55c1 100644 --- a/generator_cargohome.go +++ b/generator_cargohome.go @@ -101,6 +101,7 @@ func (s *Spec) CargohomeDeps(sOpt SourceOpts, worker llb.State, opts ...llb.Cons }) } + deps = deps.With(sourceFilter(sOpt, opts...)) return &deps } diff --git a/generator_gomod.go b/generator_gomod.go index b84a24933..182f67530 100644 --- a/generator_gomod.go +++ b/generator_gomod.go @@ -245,6 +245,7 @@ func (s *Spec) GomodDeps(sOpt SourceOpts, worker llb.State, opts ...llb.Constrai }) } + deps = deps.With(sourceFilter(sOpt, opts...)) return &deps } diff --git a/generator_nodemodules.go b/generator_nodemodules.go index a2fd099af..6cf82d704 100644 --- a/generator_nodemodules.go +++ b/generator_nodemodules.go @@ -104,7 +104,7 @@ func (s *Spec) NodeModDeps(sOpt SourceOpts, worker llb.State, opts ...llb.Constr } merged = merged.With(withNodeMod(gen, worker, key, opts...)) } - result[key] = merged + result[key] = merged.With(sourceFilterAtPath(sOpt, key, opts...)) } return result } diff --git a/generator_pip.go b/generator_pip.go index 133a0d8e0..7b4321ec9 100644 --- a/generator_pip.go +++ b/generator_pip.go @@ -133,6 +133,7 @@ func (s *Spec) PipDeps(sOpt SourceOpts, worker llb.State, opts ...llb.Constraint // Merge all cache states into a single state merged := MergeAtPath(llb.Scratch(), cacheStates, "/", opts...) + merged = merged.With(sourceFilter(sOpt, opts...)) return &merged } diff --git a/load.go b/load.go index d0646a4aa..95de63351 100644 --- a/load.go +++ b/load.go @@ -39,6 +39,10 @@ func knownArg(key string) bool { return true case "DALEC_SKIP_TESTS": return true + case "DALEC_SOURCE_FILTER_CONFIG_PATH": + return true + case "DALEC_SOURCE_FILTER_CONFIG_CONTEXT_NAME": + return true case KeyDalecTarget: return true } diff --git a/packaging/linux/rpm/buildroot.go b/packaging/linux/rpm/buildroot.go index 10ab366c4..01da1b229 100644 --- a/packaging/linux/rpm/buildroot.go +++ b/packaging/linux/rpm/buildroot.go @@ -11,6 +11,10 @@ import ( ) func RPMSpec(spec *dalec.Spec, in llb.State, targetKey, dir string, opts ...llb.ConstraintsOpt) llb.State { + return RPMSpecWithSourceFilter(spec, in, targetKey, dir, dalec.SourceFilterConfig{}, opts...) +} + +func RPMSpecWithSourceFilter(spec *dalec.Spec, in llb.State, targetKey, dir string, filter dalec.SourceFilterConfig, opts ...llb.ConstraintsOpt) llb.State { if err := ValidateSpec(spec); err != nil { return dalec.ErrorState(llb.Scratch(), fmt.Errorf("invalid spec: %w", err)) } @@ -20,7 +24,7 @@ func RPMSpec(spec *dalec.Spec, in llb.State, targetKey, dir string, opts ...llb. buf.WriteString("# Automatically generated by " + info.Main.Path + "\n") buf.WriteString("\n") - if err := WriteSpec(spec, targetKey, buf); err != nil { + if err := WriteSpecWithSourceFilter(spec, targetKey, filter, buf); err != nil { return dalec.ErrorState(llb.Scratch(), err) } @@ -36,5 +40,9 @@ func RPMSpec(spec *dalec.Spec, in llb.State, targetKey, dir string, opts ...llb. func BuildRoot(worker llb.State, spec *dalec.Spec, sOpt dalec.SourceOpts, targetKey string, opts ...llb.ConstraintsOpt) llb.State { opts = append(opts, dalec.ProgressGroup("Create RPM buildroot")) sources := Sources(worker, spec, sOpt, opts...) - return RPMSpec(spec, dalec.MergeAtPath(llb.Scratch(), sources, "SOURCES", opts...), targetKey, "", opts...) + filter, err := sOpt.GetSourceFilter() + if err != nil { + return dalec.ErrorState(llb.Scratch(), err) + } + return RPMSpecWithSourceFilter(spec, dalec.MergeAtPath(llb.Scratch(), sources, "SOURCES", opts...), targetKey, "", filter, opts...) } diff --git a/packaging/linux/rpm/template.go b/packaging/linux/rpm/template.go index 478fd00d1..0e0b3896a 100644 --- a/packaging/linux/rpm/template.go +++ b/packaging/linux/rpm/template.go @@ -67,7 +67,8 @@ var tmplFuncs = map[string]any{ type specWrapper struct { *dalec.Spec - Target string + Target string + SourceFilter dalec.SourceFilterConfig } func (w *specWrapper) Changelog() (fmt.Stringer, error) { @@ -337,21 +338,46 @@ func (w *specWrapper) Sources() (fmt.Stringer, error) { if scanner.Err() != nil { return nil, scanner.Err() } + if !w.SourceFilter.IsEmpty() && isDir { + if err := docSourceFilter(b, "Exclusions", w.SourceFilter.GlobalExcludes); err != nil { + return nil, err + } + } fmt.Fprintf(b, "Source%d: %s\n", idx, ref) } sourceIdx := len(keys) if w.Spec.HasGomods() { + if !w.SourceFilter.IsEmpty() { + if err := docSourceFilter(b, "Exclusions", w.SourceFilter.GlobalExcludes); err != nil { + return nil, err + } + } fmt.Fprintf(b, "Source%d: %s.tar.gz\n", sourceIdx, gomodsName) sourceIdx += 1 } if w.Spec.HasCargohomes() { + if !w.SourceFilter.IsEmpty() { + if err := docSourceFilter(b, "Exclusions", w.SourceFilter.GlobalExcludes); err != nil { + return nil, err + } + } fmt.Fprintf(b, "Source%d: %s.tar.gz\n", sourceIdx, cargohomeName) sourceIdx += 1 } + if w.Spec.HasPips() { + if !w.SourceFilter.IsEmpty() { + if err := docSourceFilter(b, "Exclusions", w.SourceFilter.GlobalExcludes); err != nil { + return nil, err + } + } + fmt.Fprintf(b, "Source%d: %s.tar.gz\n", sourceIdx, pipDepsName) + sourceIdx += 1 + } + if len(w.Spec.Build.Steps) > 0 { fmt.Fprintf(b, "Source%d: %s\n", sourceIdx, buildScriptName) } @@ -362,6 +388,24 @@ func (w *specWrapper) Sources() (fmt.Stringer, error) { return b, nil } +func docSourceFilter(w io.Writer, name string, values []string) error { + if _, err := fmt.Fprintf(w, "# %s:\n", name); err != nil { + return err + } + for _, value := range values { + scanner := bufio.NewScanner(strings.NewReader(value)) + for scanner.Scan() { + if _, err := fmt.Fprintf(w, "# \t%s\n", scanner.Text()); err != nil { + return err + } + } + if err := scanner.Err(); err != nil { + return err + } + } + return nil +} + func (w *specWrapper) Release() string { if w.Spec.Revision == "" { return "1" @@ -1133,7 +1177,11 @@ func (w *specWrapper) DisableAutoReq() string { // WriteSpec generates an rpm spec from the provided [dalec.Spec] and distro target and writes it to the passed in writer func WriteSpec(spec *dalec.Spec, target string, w io.Writer) error { - s := &specWrapper{spec, target} + return WriteSpecWithSourceFilter(spec, target, dalec.SourceFilterConfig{}, w) +} + +func WriteSpecWithSourceFilter(spec *dalec.Spec, target string, filter dalec.SourceFilterConfig, w io.Writer) error { + s := &specWrapper{Spec: spec, Target: target, SourceFilter: filter} err := specTmpl.Execute(w, s) if err != nil { diff --git a/packaging/linux/rpm/template_test.go b/packaging/linux/rpm/template_test.go index b89d7a59a..5c78e6bc7 100644 --- a/packaging/linux/rpm/template_test.go +++ b/packaging/linux/rpm/template_test.go @@ -161,8 +161,16 @@ func TestTemplateSources(t *testing.T) { t.Fatalf("unexpected error: %v", err) } s2 := out2.String() - if s2 != s { - t.Fatalf("expected no additional sources for pip, got: %q", s2) + // trim last newline from the first output since that has shifted + s3 := s[:len(s)-1] + if !strings.HasPrefix(s2, s3) { + t.Fatalf("expected output to start with %q, got %q", s, out2.String()) + } + + s2 = strings.TrimPrefix(out2.String(), s3) + expected := "Source1: " + pipDepsName + ".tar.gz\n\n" + if s2 != expected { + t.Fatalf("unexpected sources: expected %q, got: %q", expected, s2) } }) @@ -253,11 +261,10 @@ func TestTemplateSources(t *testing.T) { s = s[len(expected):] } - // Now we should have entries for gomods and cargohome. + // Now we should have entries for gomods, cargohome, and pip deps. // Note there are 2 gomod sources but they should be combined into one entry. - // Pip no longer creates a separate cache source. - expected := "Source7: " + gomodsName + ".tar.gz\nSource8: " + cargohomeName + ".tar.gz\n\n" + expected := "Source7: " + gomodsName + ".tar.gz\nSource8: " + cargohomeName + ".tar.gz\nSource9: " + pipDepsName + ".tar.gz\n\n" if s != expected { t.Fatalf("generators: unexpected sources: expected %q, got: %q", expected, s) } @@ -266,6 +273,49 @@ func TestTemplateSources(t *testing.T) { t.Fatalf("unexpected trailing sources: %q", s) } }) + + t.Run("source filter docs", func(t *testing.T) { + w := &specWrapper{ + Spec: &dalec.Spec{ + Sources: map[string]dalec.Source{ + "src1": { + Includes: []string{"cmd/**"}, + Excludes: []string{"testdata/**"}, + Inline: &dalec.SourceInline{ + Dir: &dalec.SourceInlineDir{}, + }, + }, + }, + }, + SourceFilter: dalec.SourceFilterConfig{GlobalExcludes: []string{"vendor/**"}}, + } + + out, err := w.Sources() + assert.NilError(t, err) + s := out.String() + assert.Check(t, strings.Contains(s, "# \tIncludes:\n# \t\t cmd/**\n")) + assert.Check(t, strings.Contains(s, "# \tExcludes:\n# \t\t testdata/**\n")) + assert.Check(t, strings.Contains(s, "# Exclusions:\n# \tvendor/**\n")) + }) + + t.Run("source filter docs multiline exclusion", func(t *testing.T) { + w := &specWrapper{ + Spec: &dalec.Spec{ + Sources: map[string]dalec.Source{ + "src1": { + Inline: &dalec.SourceInline{Dir: &dalec.SourceInlineDir{}}, + }, + }, + }, + SourceFilter: dalec.SourceFilterConfig{GlobalExcludes: []string{"vendor/**\nmalicious: value"}}, + } + + out, err := w.Sources() + assert.NilError(t, err) + s := out.String() + assert.Check(t, strings.Contains(s, "# Exclusions:\n# \tvendor/**\n# \tmalicious: value\n")) + assert.Check(t, !strings.Contains(s, "\nmalicious: value\n")) + }) } func TestTemplate_Artifacts(t *testing.T) { diff --git a/source.go b/source.go index 703274318..b370f5676 100644 --- a/source.go +++ b/source.go @@ -10,6 +10,7 @@ import ( "io" "os" "path/filepath" + "slices" "strings" "github.com/goccy/go-yaml/ast" @@ -153,6 +154,7 @@ type SourceOpts struct { GetContext func(string, ...llb.LocalOption) (*llb.State, error) TargetPlatform *ocispecs.Platform GitCredHelperOpt func() (llb.RunOption, error) + SourceFilter func() (SourceFilterConfig, error) } var errInvalidMountConfig = errors.New("invalid mount config") @@ -198,9 +200,21 @@ func (s Source) Doc(name string) io.Reader { if s.Path != "" { fmt.Fprintln(buf, " Extracted path:", s.Path) } + writeSourceDocList(buf, "Includes", s.Includes) + writeSourceDocList(buf, "Excludes", s.Excludes) return buf } +func writeSourceDocList(w io.Writer, name string, values []string) { + if len(values) == 0 { + return + } + printDocLn(w, "\t"+name+":") + for _, value := range values { + printDocLn(w, "\t\t", value) + } +} + func patchSource(worker, sourceState llb.State, sourceToState map[string]llb.State, patchNames []PatchSpec, subPath string, sources map[string]Source, sourceName string, opts ...llb.ConstraintsOpt) llb.State { for _, p := range patchNames { patchState := sourceToState[p.Source] @@ -441,7 +455,10 @@ func (s *Source) validate() error { if !invalid { // Only validate the source if it is a valid source variant so as to avoid panics. - if err := s.toInterface().validate(s.fetchOptions(SourceOpts{})); err != nil { + fo, err := s.fetchOptions(SourceOpts{}) + if err != nil { + errs = append(errs, err) + } else if err := s.toInterface().validate(fo); err != nil { errs = append(errs, err) } } @@ -474,17 +491,31 @@ type fetchOptions struct { SourceOpt SourceOpts } -func (s *Source) fetchOptions(sOpt SourceOpts) fetchOptions { +func (s *Source) fetchOptions(sOpt SourceOpts) (fetchOptions, error) { + excludes := s.Excludes + if s.IsDir() { + globalExcludes, err := sOpt.sourceFilterExcludes() + if err != nil { + return fetchOptions{}, err + } + if len(globalExcludes) > 0 { + excludes = append(slices.Clone(excludes), globalExcludes...) + } + } + return fetchOptions{ Includes: s.Includes, - Excludes: s.Excludes, + Excludes: excludes, Path: s.Path, SourceOpt: sOpt, - } + }, nil } func (s *Source) ToState(name string, sOpt SourceOpts, opts ...llb.ConstraintsOpt) llb.State { - fo := s.fetchOptions(sOpt) + fo, err := s.fetchOptions(sOpt) + if err != nil { + return ErrorState(llb.Scratch(), err) + } fo.Constraints = opts fo.Rename = name st := s.toInterface().toState(fo) @@ -492,7 +523,10 @@ func (s *Source) ToState(name string, sOpt SourceOpts, opts ...llb.ConstraintsOp } func (s *Source) ToMount(sOpt SourceOpts, constraints ...llb.ConstraintsOpt) (llb.State, []llb.MountOption) { - fo := s.fetchOptions(sOpt) + fo, err := s.fetchOptions(sOpt) + if err != nil { + return ErrorState(llb.Scratch(), err), nil + } fo.Constraints = append(fo.Constraints, constraints...) st, mountOpts := s.toInterface().toMount(fo) diff --git a/source_filter.go b/source_filter.go new file mode 100644 index 000000000..2e9e891fa --- /dev/null +++ b/source_filter.go @@ -0,0 +1,96 @@ +package dalec + +import ( + "path/filepath" + + "github.com/moby/buildkit/client/llb" +) + +const ( + BuildArgDalecSourceFilterConfigPath = "DALEC_SOURCE_FILTER_CONFIG_PATH" + BuildArgDalecSourceFilterContextName = "DALEC_SOURCE_FILTER_CONFIG_CONTEXT_NAME" + DefaultSourceOptionsContextName = "dalec-source-options" +) + +// SourceFilterConfig configures build-time filtering for source package inputs. +// It is intentionally global; future versions may add more specific filter +// scopes alongside GlobalExcludes. +type SourceFilterConfig struct { + GlobalExcludes []string `yaml:"global_excludes,omitempty" json:"global_excludes,omitempty"` +} + +func (sOpt SourceOpts) GetSourceFilter() (SourceFilterConfig, error) { + if sOpt.SourceFilter == nil { + return SourceFilterConfig{}, nil + } + return sOpt.SourceFilter() +} + +func (cfg SourceFilterConfig) IsEmpty() bool { + return len(cfg.GlobalExcludes) == 0 +} + +func (sOpt SourceOpts) sourceFilterExcludes() ([]string, error) { + cfg, err := sOpt.GetSourceFilter() + if err != nil { + return nil, err + } + return cfg.GlobalExcludes, nil +} + +func sourceFilter(sOpt SourceOpts, opts ...llb.ConstraintsOpt) llb.StateOption { + return func(in llb.State) llb.State { + excludes, err := sOpt.sourceFilterExcludes() + if err != nil { + return ErrorState(in, err) + } + if len(excludes) == 0 { + return in + } + return in.With(SourceFilter(SourceFilterConfig{GlobalExcludes: excludes}, opts...)) + } +} + +func sourceFilterAtPath(sOpt SourceOpts, base string, opts ...llb.ConstraintsOpt) llb.StateOption { + return func(in llb.State) llb.State { + excludes, err := sOpt.sourceFilterExcludes() + if err != nil { + return ErrorState(in, err) + } + if len(excludes) == 0 { + return in + } + return in.With(SourceFilterAtPath(base, SourceFilterConfig{GlobalExcludes: excludes}, opts...)) + } +} + +// SourceFilter filters source package content from the root of the input state. +func SourceFilter(cfg SourceFilterConfig, opts ...llb.ConstraintsOpt) llb.StateOption { + return func(in llb.State) llb.State { + if cfg.IsEmpty() { + return in + } + return llb.Scratch().File(llb.Copy(in, "/", "/", WithDirContentsOnly(), WithExcludes(cfg.GlobalExcludes)), opts...) + } +} + +// SourceFilterAtPath applies a global source filter to content nested under base. +// The external config remains global while named source states keep their source +// name as a top-level directory in package source assembly. +func SourceFilterAtPath(base string, cfg SourceFilterConfig, opts ...llb.ConstraintsOpt) llb.StateOption { + return func(in llb.State) llb.State { + if cfg.IsEmpty() { + return in + } + if isRoot(base) { + return in.With(SourceFilter(cfg, opts...)) + } + + excludes := make([]string, 0, len(cfg.GlobalExcludes)) + for _, exclude := range cfg.GlobalExcludes { + excludes = append(excludes, filepath.ToSlash(filepath.Join(base, exclude))) + } + + return SourceFilter(SourceFilterConfig{GlobalExcludes: excludes}, opts...)(in) + } +} diff --git a/source_inline.go b/source_inline.go index 7b4ea67ec..7f3a8cd8b 100644 --- a/source_inline.go +++ b/source_inline.go @@ -327,7 +327,25 @@ func (s *SourceInlineFile) toMount(opts fetchOptions) (llb.State, []llb.MountOpt func (s *SourceInlineDir) toState(opts fetchOptions) llb.State { base := s.baseState(opts) // inline dir handles dir names and subpaths itself - // Do not pass rename to sourceFilters + // Include/exclude patterns are relative to the requested source root, but + // inline dirs create files under Rename before sourceFilters runs. + if opts.Rename != "" { + if len(opts.Includes) > 0 { + includes := make([]string, len(opts.Includes)) + for i, include := range opts.Includes { + includes[i] = filepath.ToSlash(filepath.Join(opts.Rename, include)) + } + opts.Includes = includes + } + if len(opts.Excludes) > 0 { + excludes := make([]string, len(opts.Excludes)) + for i, exclude := range opts.Excludes { + excludes[i] = filepath.ToSlash(filepath.Join(opts.Rename, exclude)) + } + opts.Excludes = excludes + } + } + // Do not pass rename to sourceFilters as a destination rename. opts.Rename = "" return base.With(sourceFilters(opts)) } diff --git a/source_test.go b/source_test.go index faa1d8a22..bfe8102ed 100644 --- a/source_test.go +++ b/source_test.go @@ -315,6 +315,126 @@ func TestSourceHTTP(t *testing.T) { }) } +func TestSourceFilterAtPath(t *testing.T) { + t.Parallel() + + ctx := context.Background() + filter := SourceFilterConfig{GlobalExcludes: []string{"nested/bad.txt", "*.tmp"}} + st := llb.Scratch().File(llb.Mkfile("src/nested/bad.txt", 0o644, nil)).With(SourceFilterAtPath("src", filter)) + + def, err := st.Marshal(ctx) + assert.NilError(t, err) + + var fileOp *pb.FileOp + for _, dt := range def.Def { + var op pb.Op + assert.NilError(t, op.Unmarshal(dt)) + if op.GetFile() != nil { + fileOp = op.GetFile() + } + } + if fileOp == nil { + t.Fatal("expected file op") + } + + cp := fileOp.Actions[0].GetCopy() + if cp == nil { + t.Fatal("expected copy action") + } + assert.Check(t, cmp.DeepEqual(cp.ExcludePatterns, []string{"src/nested/bad.txt", "src/*.tmp"})) +} + +func TestSourceFilter(t *testing.T) { + t.Parallel() + + ctx := context.Background() + filter := SourceFilterConfig{GlobalExcludes: []string{"bad.txt"}} + st := llb.Scratch().File(llb.Mkfile("bad.txt", 0o644, nil)).With(SourceFilter(filter)) + + def, err := st.Marshal(ctx) + assert.NilError(t, err) + + var fileOp *pb.FileOp + for _, dt := range def.Def { + var op pb.Op + assert.NilError(t, op.Unmarshal(dt)) + if op.GetFile() != nil { + fileOp = op.GetFile() + } + } + if fileOp == nil { + t.Fatal("expected file op") + } + + cp := fileOp.Actions[0].GetCopy() + if cp == nil { + t.Fatal("expected copy action") + } + assert.Check(t, cmp.DeepEqual(cp.ExcludePatterns, filter.GlobalExcludes)) +} + +func TestSourceFilterEmptyNoop(t *testing.T) { + t.Parallel() + + ctx := context.Background() + base := llb.Scratch().File(llb.Mkfile("keep.txt", 0o644, nil)) + filtered := base.With(SourceFilter(SourceFilterConfig{})) + + baseDef, err := base.Marshal(ctx) + assert.NilError(t, err) + filteredDef, err := filtered.Marshal(ctx) + assert.NilError(t, err) + + assert.Check(t, cmp.DeepEqual(filteredDef.Def, baseDef.Def)) +} + +func TestNodeModDepsSourceFilter(t *testing.T) { + t.Parallel() + + ctx := context.Background() + spec := &Spec{Sources: map[string]Source{ + "src": { + Inline: &SourceInline{Dir: &SourceInlineDir{Files: map[string]*SourceInlineFile{ + "package.json": {Contents: "{}"}, + }}}, + Generate: []*SourceGenerator{{NodeMod: &GeneratorNodeMod{}}}, + }, + }} + spec.FillDefaults() + + result := spec.NodeModDeps(SourceOpts{SourceFilter: func() (SourceFilterConfig, error) { + return SourceFilterConfig{GlobalExcludes: []string{"node_modules/**"}}, nil + }}, llb.Scratch()) + + st, ok := result["src"] + if !ok { + t.Fatal("expected generated node module source") + } + + def, err := st.Marshal(ctx) + assert.NilError(t, err) + + for _, dt := range def.Def { + var op pb.Op + assert.NilError(t, op.Unmarshal(dt)) + fileOp := op.GetFile() + if fileOp == nil { + continue + } + for _, action := range fileOp.Actions { + cp := action.GetCopy() + if cp == nil { + continue + } + if slices.Equal(cp.ExcludePatterns, []string{"src/node_modules/**"}) { + return + } + } + } + + t.Fatal("expected node module dependency output to be filtered with source-prefixed excludes") +} + func toImageRef(ref string) string { return "docker-image://" + ref } @@ -685,6 +805,16 @@ func TestSourceContext(t *testing.T) { checkContext(t, ops[0].GetSource(), &src) testWithFilters(t, src) }) + + t.Run("with source filter", func(t *testing.T) { + src := Source{Context: &SourceContext{}} + sOpt := prepareGetSourceOp(ctx, t, &src) + sOpt.SourceFilter = func() (SourceFilterConfig, error) { + return SourceFilterConfig{GlobalExcludes: []string{"drop.txt"}}, nil + } + ops := getSourceOpWithOpts(ctx, t, src, sOpt) + checkContext(t, ops[0].GetSource(), &Source{Context: &SourceContext{}, Excludes: []string{"drop.txt"}}) + }) } func TestSourceInlineFile(t *testing.T) { @@ -805,6 +935,26 @@ func TestSourceInlineDir(t *testing.T) { }) }) } + + t.Run("with source filter", func(t *testing.T) { + src := Source{Inline: &SourceInline{Dir: &SourceInlineDir{Files: map[string]*SourceInlineFile{ + "keep.txt": {Contents: "keep"}, + "drop.txt": {Contents: "drop"}, + }}}} + sOpt := SourceOpts{SourceFilter: func() (SourceFilterConfig, error) { + return SourceFilterConfig{GlobalExcludes: []string{"drop.txt"}}, nil + }} + ops := getSourceOpWithOpts(ctx, t, src, sOpt) + if len(ops) != 4 { + t.Fatalf("expected mkdir, two mkfile ops, and filter copy op, got %d", len(ops)) + } + + cp := ops[3].GetFile().Actions[0].GetCopy() + if cp == nil { + t.Fatal("expected copy action") + } + assert.Check(t, cmp.DeepEqual(cp.ExcludePatterns, []string{"drop.txt"})) + }) } func checkMkdir(t *testing.T, op *pb.FileOp, src *SourceInlineDir) { @@ -987,6 +1137,17 @@ func getSourceOp(ctx context.Context, t *testing.T, src Source) []*pb.Op { return sourceOpsFromState(ctx, t, st) } +func getSourceOpWithOpts(ctx context.Context, t *testing.T, src Source, sOpt SourceOpts) []*pb.Op { + t.Helper() + src.fillDefaults() + name := "" + if !src.IsDir() { + name = "test" + } + st := src.ToState(name, sOpt) + return sourceOpsFromState(ctx, t, st) +} + func getMountOp(ctx context.Context, t *testing.T, src Source, target string) []*pb.Op { t.Helper() diff --git a/spec_test.go b/spec_test.go index 95ac87203..ad6428357 100644 --- a/spec_test.go +++ b/spec_test.go @@ -35,6 +35,26 @@ func TestDate(t *testing.T) { assert.Check(t, cmp.Equal(d3.Format(time.DateOnly), expect)) } +func TestSourceFilterConfig(t *testing.T) { + t.Parallel() + + var cfg SourceFilterConfig + err := yaml.Unmarshal([]byte(` +global_excludes: + - github.com/klauspost/compress@*/zip/corpus/14.zip + - cache/download/github.com/klauspost/compress/@v/*.zip +`), &cfg) + assert.NilError(t, err) + assert.Check(t, cmp.DeepEqual(cfg.GlobalExcludes, []string{ + "github.com/klauspost/compress@*/zip/corpus/14.zip", + "cache/download/github.com/klauspost/compress/@v/*.zip", + })) + assert.Check(t, !cfg.IsEmpty()) + + var empty SourceFilterConfig + assert.Check(t, empty.IsEmpty()) +} + func TestSourceGeneratorValidateGomodEdits(t *testing.T) { t.Parallel() diff --git a/test/source_test.go b/test/source_test.go index fa9e0a827..bccc04671 100644 --- a/test/source_test.go +++ b/test/source_test.go @@ -1032,6 +1032,115 @@ func TestSourceWithCargohome(t *testing.T) { }) } +func TestDebugGomodSourceFilterConfig(t *testing.T) { + t.Parallel() + + spec := &dalec.Spec{ + Name: "test-source-filter-gomod", + Version: "0.0.1", + Revision: "1", + License: "MIT", + Website: "https://github.com/project-dalec/dalec", + Vendor: "Dalec", + Packager: "Dalec", + Description: "Testing source filter config with gomod", + Sources: map[string]dalec.Source{ + "src": { + Generate: []*dalec.SourceGenerator{{Gomod: &dalec.GeneratorGomod{}}}, + Inline: &dalec.SourceInline{ + Dir: &dalec.SourceInlineDir{ + Files: map[string]*dalec.SourceInlineFile{ + "main.go": {Contents: gomodFixtureMain}, + "go.mod": {Contents: gomodFixtureMod}, + "go.sum": {Contents: gomodFixtureSum}, + }, + }, + }, + }, + }, + Dependencies: &dalec.PackageDependencies{ + Build: map[string]dalec.PackageConstraints{ + "golang": {}, + }, + }, + } + + filterConfig := llb.Scratch().File(llb.Mkfile("/source-filter.yml", 0o644, []byte(` +global_excludes: + - github.com/cpuguy83/tar2go@v0.3.1 +`))) + + runTest(t, func(ctx context.Context, gwc gwclient.Client) { + req := newSolveRequest( + withBuildTarget("debug/gomods"), + withSpec(ctx, t, spec), + withBuildContext(ctx, t, dalec.DefaultSourceOptionsContextName, filterConfig), + withBuildArg(dalec.BuildArgDalecSourceFilterConfigPath, "/source-filter.yml"), + ) + + res := solveT(ctx, t, gwc, req) + ref, err := res.SingleRef() + assert.NilError(t, err) + + _, err = ref.StatFile(ctx, gwclient.StatRequest{Path: "github.com/cpuguy83/tar2go@v0.3.1"}) + assert.Assert(t, err != nil, "expected filtered gomod directory to be absent") + + _, err = ref.StatFile(ctx, gwclient.StatRequest{Path: "cache"}) + assert.NilError(t, err) + }) +} + +func TestDebugSourcesSourceFilterConfig(t *testing.T) { + t.Parallel() + + spec := &dalec.Spec{ + Name: "test-source-filter-debug-sources", + Version: "0.0.1", + Revision: "1", + License: "MIT", + Website: "https://github.com/project-dalec/dalec", + Vendor: "Dalec", + Packager: "Dalec", + Description: "Testing source filter config with debug sources", + Sources: map[string]dalec.Source{ + "src": { + Inline: &dalec.SourceInline{ + Dir: &dalec.SourceInlineDir{ + Files: map[string]*dalec.SourceInlineFile{ + "keep.txt": {Contents: "keep"}, + "drop.txt": {Contents: "drop"}, + }, + }, + }, + }, + }, + } + + filterConfig := llb.Scratch().File(llb.Mkfile("/source-filter.yml", 0o644, []byte(` +global_excludes: + - drop.txt +`))) + + runTest(t, func(ctx context.Context, gwc gwclient.Client) { + req := newSolveRequest( + withBuildTarget("debug/sources"), + withSpec(ctx, t, spec), + withBuildContext(ctx, t, dalec.DefaultSourceOptionsContextName, filterConfig), + withBuildArg(dalec.BuildArgDalecSourceFilterConfigPath, "/source-filter.yml"), + ) + + res := solveT(ctx, t, gwc, req) + ref, err := res.SingleRef() + assert.NilError(t, err) + + _, err = ref.StatFile(ctx, gwclient.StatRequest{Path: "src/keep.txt"}) + assert.NilError(t, err) + + _, err = ref.StatFile(ctx, gwclient.StatRequest{Path: "src/drop.txt"}) + assert.Assert(t, err != nil, "expected filtered source file to be absent") + }) +} + func TestSourceContext(t *testing.T) { t.Parallel() diff --git a/website/docs/sources.md b/website/docs/sources.md index 31708efd7..50e9053bc 100644 --- a/website/docs/sources.md +++ b/website/docs/sources.md @@ -354,6 +354,39 @@ Build sources are considered to be "directory" sources. Generators are used to generate a source from another source. Currently the generators supported are `gomod`, `cargohome`, `pip`, and `nodemod`. +## Build-time source filters + +Dalec can load a build-time source filter configuration from a build context. +This is useful when a packaging or signing environment must reject specific +source files, but the package spec should not be changed. + +The filter config is a YAML file with a `global_excludes` list: + +```yaml +global_excludes: + - "testdata/large-fixture.zip" + - "vendor/**/examples/**" +``` + +Pass the config path with `DALEC_SOURCE_FILTER_CONFIG_PATH`. The file is read +from the `dalec-source-options` build context unless +`DALEC_SOURCE_FILTER_CONFIG_CONTEXT_NAME` names a different build context. + +Example with Docker Buildx: + +```console +$ docker buildx build \ + --build-arg DALEC_SOURCE_FILTER_CONFIG_PATH=source-filter.yml \ + --build-context dalec-source-options=./ci/dalec \ + ... +``` + +`global_excludes` patterns are evaluated relative to each source root. For +normal directory sources this uses the same filtering path as `sources.*.excludes`; +local context filters are pushed into BuildKit context loading. For generated +aggregate sources such as `gomod`, patterns are relative to the generated cache +root and are applied after generation. + ### Gomod The `gomod` generator manages a single go module cache for all sources that