sources: verify git refs with checksum

Add an optional git source checksum that is passed to BuildKit so tags and branches can be fetched by ref while ensuring they resolve to the expected commit.

This preserves .git tag metadata when keepGitDir is enabled, which lets tools derive version information from tags without giving up source pinning. If a tag or branch moves away from the expected commit, checksum verification fails the build.

Signed-off-by: Brian Goff <cpuguy83@gmail.com>

Author: cpuguy83

docs/spec.schema.json

@@ -1820,22 +1820,33 @@
 			],
 			"properties": {
 				"auth": {
-					"$ref": "#/$defs/GitAuth"
+					"$ref": "#/$defs/GitAuth",
+					"description": "Auth can be used to add instructions on how to authenticate with the git repository."
+				},
+				"checksum": {
+					"type": [
+						"string",
+						"null"
+					],
+					"description": "Checksum is the expected commit hash for the resolved ref.\nIt is useful when \"Commit\" refers to a mutable ref, such as a tag."
 				},
 				"commit": {
 					"type": [
 						"string"
-					]
+					],
+					"description": "Commit is the ref, which may be a commit, a tag, or even a full ref.\nNOTE: When using a commit ref, tag info is *not* fetched."
 				},
 				"keepGitDir": {
 					"type": [
 						"boolean"
-					]
+					],
+					"description": "KeepGitDir includes the .git directory in the source.\nNOTE: Not all git metadata may be available. The available metadata depends\non the ref specified by \"Commit\"."
 				},
 				"url": {
 					"type": [
 						"string"
-					]
+					],
+					"description": "URL is the URL of the git repository."
 				}
 			},
 			"additionalProperties": {

load_test.go

@@ -41,6 +41,27 @@ func TestSourceValidation(t *testing.T) {
 			},
 			expectErr: true,
 		},
+		{
+			title: "git source checksum must be hex",
+			src: Source{
+				Git: &SourceGit{
+					URL:      "https://example.com/repo.git",
+					Commit:   "v1.2.3",
+					Checksum: "not-a-commit",
+				},
+			},
+			expectErr: true,
+		},
+		{
+			title: "git source checksum accepts hex",
+			src: Source{
+				Git: &SourceGit{
+					URL:      "https://example.com/repo.git",
+					Commit:   "v1.2.3",
+					Checksum: "0123456789abcdef",
+				},
+			},
+		},
 		{
 			title:     "has multiple source types in docker-image command mount",
 			expectErr: true,
@@ -701,6 +722,13 @@ func TestSpec_SubstituteBuildArgs(t *testing.T) {
 		Excludes: []string{"foo/${BAR}"},
 		Inline:   &SourceInline{},
 	}
+	spec.Sources["git"] = Source{
+		Git: &SourceGit{
+			URL:      "https://example.com/foo/${BAR}.git",
+			Commit:   "$FOO",
+			Checksum: "$BAR",
+		},
+	}
 
 	spec.Patches = map[string][]PatchSpec{
 		"src": {
@@ -802,6 +830,9 @@ func TestSpec_SubstituteBuildArgs(t *testing.T) {
 	assert.Check(t, cmp.Equal(spec.Sources["patch"].Path, "foo/"+bar))
 	assert.Check(t, cmp.Equal(spec.Sources["patch"].Includes[0], "foo/"+bar))
 	assert.Check(t, cmp.Equal(spec.Sources["patch"].Excludes[0], "foo/"+bar))
+	assert.Check(t, cmp.Equal(spec.Sources["git"].Git.URL, "https://example.com/foo/"+bar+".git"))
+	assert.Check(t, cmp.Equal(spec.Sources["git"].Git.Commit, foo))
+	assert.Check(t, cmp.Equal(spec.Sources["git"].Git.Checksum, bar))
 	assert.Check(t, cmp.Equal(spec.Patches["src"][0].Path, foo))
 
 	// Base package config

source_git.go

@@ -13,10 +13,20 @@ import (
 )
 
 type SourceGit struct {
-	URL        string  `yaml:"url" json:"url"`
-	Commit     string  `yaml:"commit" json:"commit"`
-	KeepGitDir bool    `yaml:"keepGitDir,omitempty" json:"keepGitDir,omitempty"`
-	Auth       GitAuth `yaml:"auth,omitempty" json:"auth,omitempty"`
+	// URL is the URL of the git repository.
+	URL string `yaml:"url" json:"url"`
+	// Commit is the ref, which may be a commit, a tag, or even a full ref.
+	// NOTE: When using a commit ref, tag info is *not* fetched.
+	Commit string `yaml:"commit" json:"commit"`
+	// Checksum is the expected commit hash for the resolved ref.
+	// It is useful when "Commit" refers to a mutable ref, such as a tag.
+	Checksum string `yaml:"checksum,omitempty" json:"checksum,omitempty"`
+	// KeepGitDir includes the .git directory in the source.
+	// NOTE: Not all git metadata may be available. The available metadata depends
+	// on the ref specified by "Commit".
+	KeepGitDir bool `yaml:"keepGitDir,omitempty" json:"keepGitDir,omitempty"`
+	// Auth can be used to add instructions on how to authenticate with the git repository.
+	Auth GitAuth `yaml:"auth,omitempty" json:"auth,omitempty"`
 
 	_sourceMap *sourceMap `yaml:"-" json:"-"`
 }
@@ -99,6 +109,15 @@ func (src *SourceGit) validate(opts fetchOptions) error {
 	if src.Commit == "" {
 		errs = append(errs, fmt.Errorf("git source must have a commit"))
 	}
+	if src.Checksum != "" {
+		for _, c := range src.Checksum {
+			if c >= '0' && c <= '9' || c >= 'a' && c <= 'f' || c >= 'A' && c <= 'F' {
+				continue
+			}
+			errs = append(errs, fmt.Errorf("git source checksum %q must be a hex commit hash", src.Checksum))
+			break
+		}
+	}
 
 	if len(errs) > 0 {
 		err := fmt.Errorf("invalid git source: %w", stderrors.Join(errs...))
@@ -118,6 +137,9 @@ func (src *SourceGit) baseState(opts fetchOptions) llb.State {
 	if src.KeepGitDir {
 		gOpts = append(gOpts, llb.KeepGitDir())
 	}
+	if src.Checksum != "" {
+		gOpts = append(gOpts, llb.GitChecksum(src.Checksum))
+	}
 	gOpts = append(gOpts, WithConstraints(opts.Constraints...))
 	gOpts = append(gOpts, &src.Auth)
 	gOpts = append(gOpts, src._sourceMap.GetRootLocation())
@@ -162,6 +184,12 @@ func (src *SourceGit) processBuildArgs(lex *shell.Lex, args map[string]string, a
 	if err != nil {
 		errs = append(errs, err)
 	}
+
+	updated, err = expandArgs(lex, src.Checksum, args, allowArg)
+	src.Checksum = updated
+	if err != nil {
+		errs = append(errs, err)
+	}
 	if len(errs) > 0 {
 		err := fmt.Errorf("failed to process build args for git source: %w", stderrors.Join(errs...))
 		err = errdefs.WithSource(err, src._sourceMap.GetErrdefsSource())
@@ -185,4 +213,7 @@ func (src *SourceGit) doc(w io.Writer, name string) {
 	printDocLn(w, "Generated from a git repository:")
 	printDocLn(w, "	Remote:", ref.Remote)
 	printDocLn(w, "	Ref:", src.Commit)
+	if src.Checksum != "" {
+		printDocLn(w, "	Checksum:", src.Checksum)
+	}
 }

source_test.go

@@ -175,6 +175,20 @@ func TestSourceGitHTTP(t *testing.T) {
 		checkGitOp(t, ops, &src)
 	})
 
+	t.Run("with checksum and git dir", func(t *testing.T) {
+		src := Source{
+			Git: &SourceGit{
+				URL:        "https://localhost/test.git",
+				Commit:     "v1.2.3",
+				Checksum:   "0123456789abcdef",
+				KeepGitDir: true,
+			},
+		}
+
+		ops := getSourceOp(ctx, t, src)
+		checkGitOp(t, ops, &src)
+	})
+
 	t.Run("gomod auth", func(t *testing.T) {
 		const (
 			numSecrets = 2
@@ -1032,6 +1046,18 @@ func checkGitOp(t *testing.T, ops []*pb.Op, src *Source) {
 		t.Errorf("expected git.fullurl %q, got %q", src.Git.URL, op.Attrs["git.fullurl"])
 	}
 
+	if src.Git.Checksum != "" {
+		assert.Check(t, cmp.Equal(op.Attrs[pb.AttrGitChecksum], src.Git.Checksum), op.Attrs)
+	} else {
+		assert.Check(t, cmp.Equal(op.Attrs[pb.AttrGitChecksum], ""), op.Attrs)
+	}
+
+	if src.Git.KeepGitDir {
+		assert.Check(t, cmp.Equal(op.Attrs[pb.AttrKeepGitDir], "true"), op.Attrs)
+	} else {
+		assert.Check(t, cmp.Equal(op.Attrs[pb.AttrKeepGitDir], ""), op.Attrs)
+	}
+
 	const (
 		defaultAuthHeader = "GIT_AUTH_HEADER"
 		defaultAuthToken  = "GIT_AUTH_TOKEN"

test/source_test.go

@@ -13,6 +13,8 @@ import (
 	"github.com/opencontainers/go-digest"
 	"github.com/project-dalec/dalec"
 	"github.com/project-dalec/dalec/frontend/pkg/bkfs"
+	gitservices "github.com/project-dalec/dalec/test/git_services"
+	"github.com/project-dalec/dalec/test/testenv"
 	"gotest.tools/v3/assert"
 )
 
@@ -453,6 +455,91 @@ func TestSourceHTTP(t *testing.T) {
 	})
 }
 
+func TestSourceGitChecksumPreservesTagMetadata(t *testing.T) {
+	t.Parallel()
+
+	ctx := startTestSpan(baseCtx, t)
+
+	const (
+		secretName = "super-secret"
+		sourceName = "repo"
+		tagName    = "v1.2.3"
+	)
+
+	testEnv.RunTest(ctx, t, func(ctx context.Context, gwc gwclient.Client) {
+		attr := gitservices.Attributes{
+			ServerRoot:      "/",
+			PrivateRepoPath: "username/private",
+			HTTPServerPath:  "/usr/local/bin/git_http_server",
+			HTTPPort:        "8080",
+		}
+
+		testState := gitservices.NewTestState(t, gwc, &attr)
+		worker := initWorker(gwc)
+		repo := llb.Scratch().File(llb.Mkfile("foo", 0o644, []byte("bar\n")))
+		repo = worker.Dir(attr.PrivateRepoAbsPath()).Run(dalec.ShArgs(`
+			set -eux
+			export GIT_CONFIG_NOGLOBAL=true
+			git init
+			git config user.name foo
+			git config user.email foo@bar.com
+			git add -A
+			git commit -m commit --no-gpg-sign
+			git tag -a `+tagName+` -m `+tagName+`
+		`)).AddMount(attr.RepoAbsDir(), repo)
+
+		commitSt := worker.Dir(attr.PrivateRepoAbsPath()).Run(dalec.ShArgs(`
+			set -eu
+			git rev-parse HEAD > /out/commit
+		`), llb.AddMount(attr.RepoAbsDir(), repo, llb.Readonly)).AddMount("/out", llb.Scratch())
+		commitDef, err := commitSt.Marshal(ctx)
+		assert.NilError(t, err)
+
+		commitRes, err := gwc.Solve(ctx, gwclient.SolveRequest{Definition: commitDef.ToPB(), Evaluate: true})
+		assert.NilError(t, err)
+		commit := strings.TrimSpace(string(readFile(ctx, t, "commit", commitRes)))
+
+		gitHost := worker.With(hostedRepo(repo, attr.RepoAbsDir()))
+		httpServer := testState.StartHTTPGitServer(ctx, gitHost)
+
+		spec := &dalec.Spec{
+			Sources: map[string]dalec.Source{
+				sourceName: {
+					Git: &dalec.SourceGit{
+						URL:        "http://" + httpServer.IP + ":" + httpServer.Port + "/" + attr.PrivateRepoPath,
+						Commit:     tagName,
+						Checksum:   commit,
+						KeepGitDir: true,
+						Auth: dalec.GitAuth{
+							Token: secretName,
+						},
+					},
+				},
+			},
+		}
+
+		req := newSolveRequest(withBuildTarget("debug/sources"), withSpec(ctx, t, spec))
+		sourceRes := solveT(ctx, t, gwc, req)
+		verifySt := worker.Run(dalec.ShArgs(`
+			set -eu
+			git -C /src/`+sourceName+` tag --points-at HEAD > /out/tag
+			git -C /src/`+sourceName+` rev-parse HEAD > /out/head
+			git -C /src/`+sourceName+` rev-parse `+tagName+`^{} > /out/tag-commit
+			git -C /src/`+sourceName+` cat-file -t `+tagName+` > /out/tag-type
+		`), llb.AddMount("/src", resultToState(t, sourceRes), llb.Readonly)).AddMount("/out", llb.Scratch())
+		verifyDef, err := verifySt.Marshal(ctx)
+		assert.NilError(t, err)
+
+		verifyRes, err := gwc.Solve(ctx, gwclient.SolveRequest{Definition: verifyDef.ToPB(), Evaluate: true})
+		assert.NilError(t, err)
+
+		checkFile(ctx, t, "tag", verifyRes, []byte(tagName+"\n"))
+		checkFile(ctx, t, "head", verifyRes, []byte(commit+"\n"))
+		checkFile(ctx, t, "tag-commit", verifyRes, []byte(commit+"\n"))
+		checkFile(ctx, t, "tag-type", verifyRes, []byte("tag\n"))
+	}, testenv.WithSecrets(secretName, "password"))
+}
+
 // Create a very simple fake module with a limited dependency tree just to
 // keep the test as fast/reliable as possible.
 const gomodFixtureMain = `package main

website/docs/sources.md

@@ -43,7 +43,7 @@ sources:
 
 ### Git
 
-Git sources fetch a git repository at a specific commit.
+Git sources fetch a git repository at a specific commit, branch, or tag ref.
 You can use either an SSH style git URL or an HTTPS style git URL.
 
 For SSH style git URLs, if the client (such as the docker CLI) has provided
@@ -61,13 +61,24 @@ sources:
     git:
       # This uses an HTTPS style git URL.
       url: https://github.com/myOrg/myRepo.git
-      commit: 1234567890abcdef
+      commit: v1.2.3
+      checksum: 1234567890abcdef # [Optional] Verify the ref resolves to this commit.
       keepGitDir: true # [Optional] Keep the .git directory when fetching the git source. Default: false
 ```
 
 By default, Dalec will discard the `.git` directory when fetching a git source.
 You can override this behavior by setting `keepGitDir: true` in the git configuration.
 
+When `commit` is a branch or tag, `checksum` can be used to verify that the ref
+resolves to the expected commit. This lets you fetch a tag while still pinning
+the content to a known commit. BuildKit accepts a full or short hex commit hash
+for `checksum`.
+
+Fetching by tag can be useful when tools inspect git metadata during the build.
+For example, Go build metadata and Kubernetes-style version tooling can use tag
+information from `.git` when `keepGitDir: true` is set, while `checksum` still
+ensures the tag resolves to the expected commit.
+
 Git repositories are considered to be "directory" sources.
 
 Authentication will be handled using some default secret names which are fetched

website/docs/spec.md

@@ -180,7 +180,8 @@ sources:
   foo:
     git:
       url: https://github.com/foo/bar.git
-      commit: ${COMMIT}
+      commit: ${TAG}
+      checksum: ${COMMIT}
       keepGitDir: true
     generate:
     - gomod: {}