frontend: add build-time source filters

Add a source-options context that can provide a YAML source filter
config at build time. The config currently supports global_excludes
which can be used to filter out paths from all sources.

Signed-off-by: Brian Goff <cpuguy83@gmail.com>

Author: cpuguy83

frontend/debug/handle_sources.go

@@ -18,10 +18,10 @@ func Sources(ctx context.Context, client gwclient.Client) (*gwclient.Result, err
 			return nil, nil, err
 		}
 
-		sources := dalec.Sources(spec, sOpt)
-
 		pg := dalec.ProgressGroup("Sources for " + targetKey + " rpm build: " + spec.Name)
 
+		sources := dalec.Sources(spec, sOpt, pg)
+
 		def, err := dalec.MergeAtPath(llb.Scratch(), dalec.SortedMapValues(sources), "/", pg).Marshal(ctx)
 		if err != nil {
 			return nil, nil, err

frontend/gateway.go

@@ -103,7 +103,7 @@ func GetBuildArg(client gwclient.Client, k string) (string, bool) {
 }
 
 func SourceOptFromUIClient(ctx context.Context, c gwclient.Client, dc *dockerui.Client, platform *ocispecs.Platform) dalec.SourceOpts {
-	return dalec.SourceOpts{
+	sOpt := dalec.SourceOpts{
 		TargetPlatform: platform,
 		Resolver:       c,
 		Forward:        ForwarderFromClient(ctx, c),
@@ -129,6 +129,12 @@ func SourceOptFromUIClient(ctx context.Context, c gwclient.Client, dc *dockerui.
 		},
 		GitCredHelperOpt: withCredHelper(c),
 	}
+
+	sOpt.SourceFilter = sync.OnceValues(func() (dalec.SourceFilterConfig, error) {
+		return loadSourceFilterConfig(ctx, c, sOpt.GetContext)
+	})
+
+	return sOpt
 }
 
 func SourceOptFromClient(ctx context.Context, c gwclient.Client, platform *ocispecs.Platform) (dalec.SourceOpts, error) {

frontend/request.go

@@ -142,39 +142,76 @@ func marshalDockerfile(ctx context.Context, dt []byte, opts ...llb.ConstraintsOp
 }
 
 func getSigningConfigFromContext(ctx context.Context, client gwclient.Client, cfgPath string, configCtxName string, sOpt dalec.SourceOpts, opts ...llb.ConstraintsOpt) (*dalec.PackageSigner, error) {
-	src := dalec.Source{Path: cfgPath, Context: &dalec.SourceContext{Name: configCtxName}}
-	signConfigState := src.ToState("", sOpt, opts...)
-
-	scDef, err := signConfigState.Marshal(ctx)
+	dt, err := readConfigFromContext(ctx, client, cfgPath, configCtxName, sOpt, opts...)
 	if err != nil {
 		return nil, err
 	}
 
+	var pc dalec.PackageConfig
+	if err := yaml.Unmarshal(dt, &pc); err != nil {
+		return nil, err
+	}
+
+	return pc.Signer, nil
+}
+
+func getSourceFilterConfigFromContext(ctx context.Context, client gwclient.Client, cfgPath string, configState llb.State) (dalec.SourceFilterConfig, error) {
+	scDef, err := configState.Marshal(ctx)
+	if err != nil {
+		return dalec.SourceFilterConfig{}, err
+	}
+
 	res, err := client.Solve(ctx, gwclient.SolveRequest{
 		Definition: scDef.ToPB(),
 	})
 	if err != nil {
-		return nil, err
+		return dalec.SourceFilterConfig{}, err
 	}
 
 	ref, err := res.SingleRef()
 	if err != nil {
-		return nil, err
+		return dalec.SourceFilterConfig{}, err
 	}
 
 	dt, err := ref.ReadFile(ctx, gwclient.ReadRequest{
 		Filename: cfgPath,
 	})
+	if err != nil {
+		return dalec.SourceFilterConfig{}, err
+	}
+
+	var cfg dalec.SourceFilterConfig
+	if err := yaml.Unmarshal(dt, &cfg); err != nil {
+		return dalec.SourceFilterConfig{}, err
+	}
+
+	return cfg, nil
+}
+
+func readConfigFromContext(ctx context.Context, client gwclient.Client, cfgPath string, configCtxName string, sOpt dalec.SourceOpts, opts ...llb.ConstraintsOpt) ([]byte, error) {
+	src := dalec.Source{Path: cfgPath, Context: &dalec.SourceContext{Name: configCtxName}}
+	configState := src.ToState("", sOpt.WithoutSourceFilter(), opts...)
+
+	scDef, err := configState.Marshal(ctx)
 	if err != nil {
 		return nil, err
 	}
 
-	var pc dalec.PackageConfig
-	if err := yaml.Unmarshal(dt, &pc); err != nil {
+	res, err := client.Solve(ctx, gwclient.SolveRequest{
+		Definition: scDef.ToPB(),
+	})
+	if err != nil {
 		return nil, err
 	}
 
-	return pc.Signer, nil
+	ref, err := res.SingleRef()
+	if err != nil {
+		return nil, err
+	}
+
+	return ref.ReadFile(ctx, gwclient.ReadRequest{
+		Filename: cfgPath,
+	})
 }
 
 func MaybeSign(ctx context.Context, client gwclient.Client, st llb.State, spec *dalec.Spec, targetKey string, sOpt dalec.SourceOpts, opts ...llb.ConstraintsOpt) llb.State {
@@ -242,6 +279,36 @@ func getSignConfigCtxName(client gwclient.Client) string {
 	return client.BuildOpts().Opts["build-arg:"+buildArgDalecSigningConfigContextName]
 }
 
+func getSourceFilterConfigPath(client gwclient.Client) string {
+	return client.BuildOpts().Opts["build-arg:"+dalec.BuildArgDalecSourceFilterConfigPath]
+}
+
+func getSourceFilterContextNameWithDefault(client gwclient.Client) string {
+	configCtxName := dalec.DefaultSourceOptionsContextName
+	if cn := client.BuildOpts().Opts["build-arg:"+dalec.BuildArgDalecSourceFilterContextName]; cn != "" {
+		configCtxName = cn
+	}
+	return configCtxName
+}
+
+func loadSourceFilterConfig(ctx context.Context, client gwclient.Client, getContext func(string, ...llb.LocalOption) (*llb.State, error), opts ...llb.LocalOption) (dalec.SourceFilterConfig, error) {
+	cfgPath := getSourceFilterConfigPath(client)
+	if cfgPath == "" {
+		return dalec.SourceFilterConfig{}, nil
+	}
+
+	configCtxName := getSourceFilterContextNameWithDefault(client)
+	configState, err := getContext(configCtxName, opts...)
+	if err != nil {
+		return dalec.SourceFilterConfig{}, err
+	}
+	if configState == nil {
+		return dalec.SourceFilterConfig{}, errors.Errorf("context %q not found", configCtxName)
+	}
+
+	return getSourceFilterConfigFromContext(ctx, client, cfgPath, *configState)
+}
+
 func forwardToSigner(ctx context.Context, client gwclient.Client, cfg *dalec.PackageSigner, s llb.State, opts ...llb.ConstraintsOpt) (llb.State, error) {
 	const (
 		// See https://github.com/moby/buildkit/blob/d8d946b85c52095d34a52ce210960832f4e06775/frontend/dockerui/context.go#L29

generator_cargohome.go

@@ -101,6 +101,7 @@ func (s *Spec) CargohomeDeps(sOpt SourceOpts, worker llb.State, opts ...llb.Cons
 		})
 	}
 
+	deps = deps.With(sourceFilter(sOpt, opts...))
 	return &deps
 }
 

generator_gomod.go

@@ -245,6 +245,7 @@ func (s *Spec) GomodDeps(sOpt SourceOpts, worker llb.State, opts ...llb.Constrai
 		})
 	}
 
+	deps = deps.With(sourceFilter(sOpt, opts...))
 	return &deps
 }
 

generator_pip.go

@@ -133,6 +133,7 @@ func (s *Spec) PipDeps(sOpt SourceOpts, worker llb.State, opts ...llb.Constraint
 
 	// Merge all cache states into a single state
 	merged := MergeAtPath(llb.Scratch(), cacheStates, "/", opts...)
+	merged = merged.With(sourceFilter(sOpt, opts...))
 	return &merged
 }
 

load.go

@@ -39,6 +39,10 @@ func knownArg(key string) bool {
 		return true
 	case "DALEC_SKIP_TESTS":
 		return true
+	case "DALEC_SOURCE_FILTER_CONFIG_PATH":
+		return true
+	case "DALEC_SOURCE_FILTER_CONFIG_CONTEXT_NAME":
+		return true
 	case KeyDalecTarget:
 		return true
 	}

packaging/linux/rpm/buildroot.go

@@ -11,6 +11,10 @@ import (
 )
 
 func RPMSpec(spec *dalec.Spec, in llb.State, targetKey, dir string, opts ...llb.ConstraintsOpt) llb.State {
+	return RPMSpecWithSourceFilter(spec, in, targetKey, dir, dalec.SourceFilterConfig{}, opts...)
+}
+
+func RPMSpecWithSourceFilter(spec *dalec.Spec, in llb.State, targetKey, dir string, filter dalec.SourceFilterConfig, opts ...llb.ConstraintsOpt) llb.State {
 	if err := ValidateSpec(spec); err != nil {
 		return dalec.ErrorState(llb.Scratch(), fmt.Errorf("invalid spec: %w", err))
 	}
@@ -20,7 +24,7 @@ func RPMSpec(spec *dalec.Spec, in llb.State, targetKey, dir string, opts ...llb.
 	buf.WriteString("# Automatically generated by " + info.Main.Path + "\n")
 	buf.WriteString("\n")
 
-	if err := WriteSpec(spec, targetKey, buf); err != nil {
+	if err := WriteSpecWithSourceFilter(spec, targetKey, filter, buf); err != nil {
 		return dalec.ErrorState(llb.Scratch(), err)
 	}
 
@@ -36,5 +40,9 @@ func RPMSpec(spec *dalec.Spec, in llb.State, targetKey, dir string, opts ...llb.
 func BuildRoot(worker llb.State, spec *dalec.Spec, sOpt dalec.SourceOpts, targetKey string, opts ...llb.ConstraintsOpt) llb.State {
 	opts = append(opts, dalec.ProgressGroup("Create RPM buildroot"))
 	sources := Sources(worker, spec, sOpt, opts...)
-	return RPMSpec(spec, dalec.MergeAtPath(llb.Scratch(), sources, "SOURCES", opts...), targetKey, "", opts...)
+	filter, err := sOpt.GetSourceFilter()
+	if err != nil {
+		return dalec.ErrorState(llb.Scratch(), err)
+	}
+	return RPMSpecWithSourceFilter(spec, dalec.MergeAtPath(llb.Scratch(), sources, "SOURCES", opts...), targetKey, "", filter, opts...)
 }

packaging/linux/rpm/template.go

@@ -67,7 +67,8 @@ var tmplFuncs = map[string]any{
 
 type specWrapper struct {
 	*dalec.Spec
-	Target string
+	Target       string
+	SourceFilter dalec.SourceFilterConfig
 }
 
 func (w *specWrapper) Changelog() (fmt.Stringer, error) {
@@ -337,21 +338,38 @@ func (w *specWrapper) Sources() (fmt.Stringer, error) {
 		if scanner.Err() != nil {
 			return nil, scanner.Err()
 		}
+		if !w.SourceFilter.IsEmpty() && isDir {
+			docSourceFilter(b, "Exclusions", w.SourceFilter.GlobalExcludes)
+		}
 		fmt.Fprintf(b, "Source%d: %s\n", idx, ref)
 	}
 
 	sourceIdx := len(keys)
 
 	if w.Spec.HasGomods() {
+		if !w.SourceFilter.IsEmpty() {
+			docSourceFilter(b, "Exclusions", w.SourceFilter.GlobalExcludes)
+		}
 		fmt.Fprintf(b, "Source%d: %s.tar.gz\n", sourceIdx, gomodsName)
 		sourceIdx += 1
 	}
 
 	if w.Spec.HasCargohomes() {
+		if !w.SourceFilter.IsEmpty() {
+			docSourceFilter(b, "Exclusions", w.SourceFilter.GlobalExcludes)
+		}
 		fmt.Fprintf(b, "Source%d: %s.tar.gz\n", sourceIdx, cargohomeName)
 		sourceIdx += 1
 	}
 
+	if w.Spec.HasPips() {
+		if !w.SourceFilter.IsEmpty() {
+			docSourceFilter(b, "Exclusions", w.SourceFilter.GlobalExcludes)
+		}
+		fmt.Fprintf(b, "Source%d: %s.tar.gz\n", sourceIdx, pipDepsName)
+		sourceIdx += 1
+	}
+
 	if len(w.Spec.Build.Steps) > 0 {
 		fmt.Fprintf(b, "Source%d: %s\n", sourceIdx, buildScriptName)
 	}
@@ -362,6 +380,13 @@ func (w *specWrapper) Sources() (fmt.Stringer, error) {
 	return b, nil
 }
 
+func docSourceFilter(w io.Writer, name string, values []string) {
+	fmt.Fprintf(w, "# %s:\n", name)
+	for _, value := range values {
+		fmt.Fprintf(w, "# \t%s\n", value)
+	}
+}
+
 func (w *specWrapper) Release() string {
 	if w.Spec.Revision == "" {
 		return "1"
@@ -1133,7 +1158,11 @@ func (w *specWrapper) DisableAutoReq() string {
 
 // WriteSpec generates an rpm spec from the provided [dalec.Spec] and distro target and writes it to the passed in writer
 func WriteSpec(spec *dalec.Spec, target string, w io.Writer) error {
-	s := &specWrapper{spec, target}
+	return WriteSpecWithSourceFilter(spec, target, dalec.SourceFilterConfig{}, w)
+}
+
+func WriteSpecWithSourceFilter(spec *dalec.Spec, target string, filter dalec.SourceFilterConfig, w io.Writer) error {
+	s := &specWrapper{Spec: spec, Target: target, SourceFilter: filter}
 
 	err := specTmpl.Execute(w, s)
 	if err != nil {

packaging/linux/rpm/template_test.go

@@ -161,8 +161,16 @@ func TestTemplateSources(t *testing.T) {
 				t.Fatalf("unexpected error: %v", err)
 			}
 			s2 := out2.String()
-			if s2 != s {
-				t.Fatalf("expected no additional sources for pip, got: %q", s2)
+			// trim last newline from the first output since that has shifted
+			s3 := s[:len(s)-1]
+			if !strings.HasPrefix(s2, s3) {
+				t.Fatalf("expected output to start with %q, got %q", s, out2.String())
+			}
+
+			s2 = strings.TrimPrefix(out2.String(), s3)
+			expected := "Source1: " + pipDepsName + ".tar.gz\n\n"
+			if s2 != expected {
+				t.Fatalf("unexpected sources: expected %q, got: %q", expected, s2)
 			}
 		})
 
@@ -253,11 +261,10 @@ func TestTemplateSources(t *testing.T) {
 			s = s[len(expected):]
 		}
 
-		// Now we should have entries for gomods and cargohome.
+		// Now we should have entries for gomods, cargohome, and pip deps.
 		// Note there are 2 gomod sources but they should be combined into one entry.
-		// Pip no longer creates a separate cache source.
 
-		expected := "Source7: " + gomodsName + ".tar.gz\nSource8: " + cargohomeName + ".tar.gz\n\n"
+		expected := "Source7: " + gomodsName + ".tar.gz\nSource8: " + cargohomeName + ".tar.gz\nSource9: " + pipDepsName + ".tar.gz\n\n"
 		if s != expected {
 			t.Fatalf("generators: unexpected sources: expected %q, got: %q", expected, s)
 		}
@@ -266,6 +273,30 @@ func TestTemplateSources(t *testing.T) {
 			t.Fatalf("unexpected trailing sources: %q", s)
 		}
 	})
+
+	t.Run("source filter docs", func(t *testing.T) {
+		w := &specWrapper{
+			Spec: &dalec.Spec{
+				Sources: map[string]dalec.Source{
+					"src1": {
+						Includes: []string{"cmd/**"},
+						Excludes: []string{"testdata/**"},
+						Inline: &dalec.SourceInline{
+							Dir: &dalec.SourceInlineDir{},
+						},
+					},
+				},
+			},
+			SourceFilter: dalec.SourceFilterConfig{GlobalExcludes: []string{"vendor/**"}},
+		}
+
+		out, err := w.Sources()
+		assert.NilError(t, err)
+		s := out.String()
+		assert.Check(t, strings.Contains(s, "# \tIncludes:\n# \t\t cmd/**\n"))
+		assert.Check(t, strings.Contains(s, "# \tExcludes:\n# \t\t testdata/**\n"))
+		assert.Check(t, strings.Contains(s, "# Exclusions:\n# \tvendor/**\n"))
+	})
 }
 
 func TestTemplate_Artifacts(t *testing.T) {