diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2a76a4f3a..2f9278a3f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -111,8 +111,6 @@ jobs: - suite: other skip: Azlinux3|Bookworm|Bullseye|Bionic|Focal|Jammy|Noble|Windows|Almalinux8|Almalinux9|Rockylinux8|Rockylinux9|Trixie - # TODO: support diff/merge - # Right now this is handled by the e2e suite, but we can migrate that here. steps: - name: Harden Runner uses: step-security/harden-runner@fe104658747b27e96e4f7e80cd0a94068e53901d # v2.16.1 @@ -127,6 +125,18 @@ jobs: - name: checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - name: Update Docker + run: | + set -e + docker version + docker info + + sudo apt update + sudo apt install -y moby-engine=29.4.* moby-buildx=0.33.* moby-cli=29.4.* moby-containerd=2.3.* + + docker version + docker info + - uses: ./.github/actions/enable-containerd - uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0 diff --git a/docs/spec.schema.json b/docs/spec.schema.json index fb2e811b4..9a1b10b20 100644 --- a/docs/spec.schema.json +++ b/docs/spec.schema.json @@ -1820,22 +1820,33 @@ ], "properties": { "auth": { - "$ref": "#/$defs/GitAuth" + "$ref": "#/$defs/GitAuth", + "description": "Auth can be used to add instructions on how to authenticate with the git repository." + }, + "checksum": { + "type": [ + "string", + "null" + ], + "description": "Checksum is the expected commit hash for the resolved ref.\nIt is useful when \"Commit\" refers to a mutable ref, such as a tag." }, "commit": { "type": [ "string" - ] + ], + "description": "Commit is the ref, which may be a commit, a tag, or even a full ref.\nNOTE: When using a commit ref, tag info is *not* fetched." }, "keepGitDir": { "type": [ "boolean" - ] + ], + "description": "KeepGitDir includes the .git directory in the source.\nNOTE: Not all git metadata may be available. The available metadata depends\non the ref specified by \"Commit\"." }, "url": { "type": [ "string" - ] + ], + "description": "URL is the URL of the git repository." } }, "additionalProperties": { diff --git a/load_test.go b/load_test.go index d3957baa5..89b7abc2b 100644 --- a/load_test.go +++ b/load_test.go @@ -41,6 +41,16 @@ func TestSourceValidation(t *testing.T) { }, expectErr: true, }, + { + title: "git source checksum accepts hex", + src: Source{ + Git: &SourceGit{ + URL: "https://example.com/repo.git", + Commit: "v1.2.3", + Checksum: "0123456789abcdef", + }, + }, + }, { title: "has multiple source types in docker-image command mount", expectErr: true, @@ -666,6 +676,7 @@ func TestSpec_SubstituteBuildArgs(t *testing.T) { const ( foo = "foo" bar = "bar" + checksum = "baddecaf" argWithDefault = "some default value" plainOleValue = "some plain old value" ) @@ -701,6 +712,13 @@ func TestSpec_SubstituteBuildArgs(t *testing.T) { Excludes: []string{"foo/${BAR}"}, Inline: &SourceInline{}, } + spec.Sources["git"] = Source{ + Git: &SourceGit{ + URL: "https://example.com/foo/${BAR}.git", + Commit: "$FOO", + Checksum: "$CHECKSUM", + }, + } spec.Patches = map[string][]PatchSpec{ "src": { @@ -795,13 +813,18 @@ func TestSpec_SubstituteBuildArgs(t *testing.T) { env["BAR"] = bar spec.Args["BAR"] = "" + spec.Args["CHECKSUM"] = "" spec.Args["VAR_WITH_DEFAULT"] = argWithDefault + env["CHECKSUM"] = checksum assert.NilError(t, spec.SubstituteArgs(env)) assert.Check(t, cmp.Equal(spec.Sources["patch"].Path, "foo/"+bar)) assert.Check(t, cmp.Equal(spec.Sources["patch"].Includes[0], "foo/"+bar)) assert.Check(t, cmp.Equal(spec.Sources["patch"].Excludes[0], "foo/"+bar)) + assert.Check(t, cmp.Equal(spec.Sources["git"].Git.URL, "https://example.com/foo/"+bar+".git")) + assert.Check(t, cmp.Equal(spec.Sources["git"].Git.Commit, foo)) + assert.Check(t, cmp.Equal(spec.Sources["git"].Git.Checksum, checksum)) assert.Check(t, cmp.Equal(spec.Patches["src"][0].Path, foo)) // Base package config @@ -1193,6 +1216,26 @@ targets: assert.Check(t, cmp.Equal(target.PackageConfig.Signer.Args["FOO"], "test")) }) + t.Run("git checksum build arg loaded from yaml", func(t *testing.T) { + dt := []byte(` +args: + COMMIT: +sources: + test: + git: + url: https://example.com/repo.git + commit: v1.2.3 + checksum: ${COMMIT} +`) + + spec, err := LoadSpec(dt) + assert.NilError(t, err) + + err = spec.SubstituteArgs(map[string]string{"COMMIT": "0123456789abcdef"}) + assert.NilError(t, err) + assert.Check(t, cmp.Equal(spec.Sources["test"].Git.Checksum, "0123456789abcdef")) + }) + t.Run("default value", func(t *testing.T) { dt := []byte(` args: diff --git a/source_git.go b/source_git.go index ae2c33306..84af5651e 100644 --- a/source_git.go +++ b/source_git.go @@ -13,10 +13,20 @@ import ( ) type SourceGit struct { - URL string `yaml:"url" json:"url"` - Commit string `yaml:"commit" json:"commit"` - KeepGitDir bool `yaml:"keepGitDir,omitempty" json:"keepGitDir,omitempty"` - Auth GitAuth `yaml:"auth,omitempty" json:"auth,omitempty"` + // URL is the URL of the git repository. + URL string `yaml:"url" json:"url"` + // Commit is the ref, which may be a commit, a tag, or even a full ref. + // NOTE: When using a commit ref, tag info is *not* fetched. + Commit string `yaml:"commit" json:"commit"` + // Checksum is the expected commit hash for the resolved ref. + // It is useful when "Commit" refers to a mutable ref, such as a tag. + Checksum string `yaml:"checksum,omitempty" json:"checksum,omitempty"` + // KeepGitDir includes the .git directory in the source. + // NOTE: Not all git metadata may be available. The available metadata depends + // on the ref specified by "Commit". + KeepGitDir bool `yaml:"keepGitDir,omitempty" json:"keepGitDir,omitempty"` + // Auth can be used to add instructions on how to authenticate with the git repository. + Auth GitAuth `yaml:"auth,omitempty" json:"auth,omitempty"` _sourceMap *sourceMap `yaml:"-" json:"-"` } @@ -118,6 +128,9 @@ func (src *SourceGit) baseState(opts fetchOptions) llb.State { if src.KeepGitDir { gOpts = append(gOpts, llb.KeepGitDir()) } + if src.Checksum != "" { + gOpts = append(gOpts, llb.GitChecksum(src.Checksum)) + } gOpts = append(gOpts, WithConstraints(opts.Constraints...)) gOpts = append(gOpts, &src.Auth) gOpts = append(gOpts, src._sourceMap.GetRootLocation()) @@ -162,6 +175,12 @@ func (src *SourceGit) processBuildArgs(lex *shell.Lex, args map[string]string, a if err != nil { errs = append(errs, err) } + + updated, err = expandArgs(lex, src.Checksum, args, allowArg) + src.Checksum = updated + if err != nil { + errs = append(errs, err) + } if len(errs) > 0 { err := fmt.Errorf("failed to process build args for git source: %w", stderrors.Join(errs...)) err = errdefs.WithSource(err, src._sourceMap.GetErrdefsSource()) @@ -185,4 +204,7 @@ func (src *SourceGit) doc(w io.Writer, name string) { printDocLn(w, "Generated from a git repository:") printDocLn(w, " Remote:", ref.Remote) printDocLn(w, " Ref:", src.Commit) + if src.Checksum != "" { + printDocLn(w, " Checksum:", src.Checksum) + } } diff --git a/source_test.go b/source_test.go index faa1d8a22..39a157a9a 100644 --- a/source_test.go +++ b/source_test.go @@ -175,6 +175,20 @@ func TestSourceGitHTTP(t *testing.T) { checkGitOp(t, ops, &src) }) + t.Run("with checksum and git dir", func(t *testing.T) { + src := Source{ + Git: &SourceGit{ + URL: "https://localhost/test.git", + Commit: "v1.2.3", + Checksum: "0123456789abcdef", + KeepGitDir: true, + }, + } + + ops := getSourceOp(ctx, t, src) + checkGitOp(t, ops, &src) + }) + t.Run("gomod auth", func(t *testing.T) { const ( numSecrets = 2 @@ -1032,6 +1046,18 @@ func checkGitOp(t *testing.T, ops []*pb.Op, src *Source) { t.Errorf("expected git.fullurl %q, got %q", src.Git.URL, op.Attrs["git.fullurl"]) } + if src.Git.Checksum != "" { + assert.Check(t, cmp.Equal(op.Attrs[pb.AttrGitChecksum], src.Git.Checksum), op.Attrs) + } else { + assert.Check(t, cmp.Equal(op.Attrs[pb.AttrGitChecksum], ""), op.Attrs) + } + + if src.Git.KeepGitDir { + assert.Check(t, cmp.Equal(op.Attrs[pb.AttrKeepGitDir], "true"), op.Attrs) + } else { + assert.Check(t, cmp.Equal(op.Attrs[pb.AttrKeepGitDir], ""), op.Attrs) + } + const ( defaultAuthHeader = "GIT_AUTH_HEADER" defaultAuthToken = "GIT_AUTH_TOKEN" diff --git a/test/source_test.go b/test/source_test.go index fa9e0a827..86eed8fb5 100644 --- a/test/source_test.go +++ b/test/source_test.go @@ -13,6 +13,8 @@ import ( "github.com/opencontainers/go-digest" "github.com/project-dalec/dalec" "github.com/project-dalec/dalec/frontend/pkg/bkfs" + gitservices "github.com/project-dalec/dalec/test/git_services" + "github.com/project-dalec/dalec/test/testenv" "gotest.tools/v3/assert" ) @@ -453,6 +455,123 @@ func TestSourceHTTP(t *testing.T) { }) } +func TestSourceGitChecksumPreservesTagMetadata(t *testing.T) { + t.Parallel() + + parentCtx := startTestSpan(baseCtx, t) + + const ( + secretName = "super-secret" + sourceName = "repo" + tagName = "v1.2.3" + ) + + runGitServerTest := func(ctx context.Context, t *testing.T, f func(context.Context, gwclient.Client, llb.State, string, func(string) *dalec.Spec)) { + t.Helper() + + testEnv.RunTest(ctx, t, func(ctx context.Context, gwc gwclient.Client) { + t.Helper() + + attr := gitservices.Attributes{ + ServerRoot: "/", + PrivateRepoPath: "username/private", + HTTPServerPath: "/usr/local/bin/git_http_server", + HTTPPort: "8080", + } + + testState := gitservices.NewTestState(t, gwc, &attr) + worker := initWorker(gwc) + repo := llb.Scratch().File(llb.Mkfile("foo", 0o644, []byte("bar\n"))) + repo = worker.Dir(attr.PrivateRepoAbsPath()).Run(dalec.ShArgs(` + set -eux + export GIT_CONFIG_NOGLOBAL=true + git init + git config user.name foo + git config user.email foo@bar.com + git add -A + git commit -m commit --no-gpg-sign + git tag -a `+tagName+` -m `+tagName+` + `)).AddMount(attr.RepoAbsDir(), repo) + + commitSt := worker.Dir(attr.PrivateRepoAbsPath()).Run(dalec.ShArgs(` + set -eu + git rev-parse HEAD > /out/commit + `), llb.AddMount(attr.RepoAbsDir(), repo, llb.Readonly)).AddMount("/out", llb.Scratch()) + commitDef, err := commitSt.Marshal(ctx) + assert.NilError(t, err) + + commitRes, err := gwc.Solve(ctx, gwclient.SolveRequest{Definition: commitDef.ToPB(), Evaluate: true}) + assert.NilError(t, err) + commit := strings.TrimSpace(string(readFile(ctx, t, "commit", commitRes))) + + gitHost := worker.With(hostedRepo(repo, attr.RepoAbsDir())) + httpServer := testState.StartHTTPGitServer(ctx, gitHost) + + newSpec := func(checksum string) *dalec.Spec { + return &dalec.Spec{ + Sources: map[string]dalec.Source{ + sourceName: { + Git: &dalec.SourceGit{ + URL: "http://" + httpServer.IP + ":" + httpServer.Port + "/" + attr.PrivateRepoPath, + Commit: tagName, + Checksum: checksum, + KeepGitDir: true, + Auth: dalec.GitAuth{ + Token: secretName, + }, + }, + }, + }, + } + } + + f(ctx, gwc, worker, commit, newSpec) + }, testenv.WithSecrets(secretName, "password")) + } + + t.Run("preserves tag metadata", func(t *testing.T) { + t.Parallel() + ctx := startTestSpan(parentCtx, t) + + runGitServerTest(ctx, t, func(ctx context.Context, gwc gwclient.Client, worker llb.State, commit string, newSpec func(string) *dalec.Spec) { + req := newSolveRequest(withBuildTarget("debug/sources"), withSpec(ctx, t, newSpec(commit))) + sourceRes := solveT(ctx, t, gwc, req) + verifySt := worker.Run(dalec.ShArgs(` + set -eu + git -C /src/`+sourceName+` tag --points-at HEAD > /out/tag + git -C /src/`+sourceName+` rev-parse HEAD > /out/head + git -C /src/`+sourceName+` rev-parse `+tagName+`^{} > /out/tag-commit + git -C /src/`+sourceName+` cat-file -t `+tagName+` > /out/tag-type + `), llb.AddMount("/src", resultToState(t, sourceRes), llb.Readonly)).AddMount("/out", llb.Scratch()) + verifyDef, err := verifySt.Marshal(ctx) + assert.NilError(t, err) + + verifyRes, err := gwc.Solve(ctx, gwclient.SolveRequest{Definition: verifyDef.ToPB(), Evaluate: true}) + assert.NilError(t, err) + + checkFile(ctx, t, "tag", verifyRes, []byte(tagName+"\n")) + checkFile(ctx, t, "head", verifyRes, []byte(commit+"\n")) + checkFile(ctx, t, "tag-commit", verifyRes, []byte(commit+"\n")) + checkFile(ctx, t, "tag-type", verifyRes, []byte("tag\n")) + }) + }) + + t.Run("rejects checksum mismatch", func(t *testing.T) { + t.Parallel() + ctx := startTestSpan(parentCtx, t) + + runGitServerTest(ctx, t, func(ctx context.Context, gwc gwclient.Client, worker llb.State, commit string, newSpec func(string) *dalec.Spec) { + _, err := gwc.Solve(ctx, newSolveRequest(withBuildTarget("debug/sources"), withSpec(ctx, t, newSpec("0000000000000000000000000000000000000000")))) + if err == nil { + t.Fatal("expected git checksum mismatch, but received none") + } + if !strings.Contains(err.Error(), "expected checksum to match") { + t.Fatalf("expected git checksum mismatch, got: %v", err) + } + }) + }) +} + // Create a very simple fake module with a limited dependency tree just to // keep the test as fast/reliable as possible. const gomodFixtureMain = `package main diff --git a/website/docs/sources.md b/website/docs/sources.md index 31708efd7..3516b1dc8 100644 --- a/website/docs/sources.md +++ b/website/docs/sources.md @@ -43,7 +43,7 @@ sources: ### Git -Git sources fetch a git repository at a specific commit. +Git sources fetch a git repository at a specific commit, branch, or tag ref. You can use either an SSH style git URL or an HTTPS style git URL. For SSH style git URLs, if the client (such as the docker CLI) has provided @@ -61,13 +61,27 @@ sources: git: # This uses an HTTPS style git URL. url: https://github.com/myOrg/myRepo.git - commit: 1234567890abcdef + commit: v1.2.3 + checksum: 1234567890abcdef # [Optional] Verify the ref resolves to this commit. keepGitDir: true # [Optional] Keep the .git directory when fetching the git source. Default: false ``` By default, Dalec will discard the `.git` directory when fetching a git source. You can override this behavior by setting `keepGitDir: true` in the git configuration. +When `commit` is a branch or tag, `checksum` can be used to verify that the ref +resolves to the expected commit. This lets you fetch a tag while still pinning +the content to a known commit. BuildKit accepts a full or short hex commit hash +for `checksum`. + +`checksum` requires BuildKit v0.22.0 or later. When using Docker's embedded +BuildKit backend, use Docker Engine v28.2.0 or later. + +Fetching by tag can be useful when tools inspect git metadata during the build. +For example, Go build metadata and Kubernetes-style version tooling can use tag +information from `.git` when `keepGitDir: true` is set, while `checksum` still +ensures the tag resolves to the expected commit. + Git repositories are considered to be "directory" sources. Authentication will be handled using some default secret names which are fetched diff --git a/website/docs/spec.md b/website/docs/spec.md index 3015e4756..a5d0df85a 100644 --- a/website/docs/spec.md +++ b/website/docs/spec.md @@ -180,7 +180,8 @@ sources: foo: git: url: https://github.com/foo/bar.git - commit: ${COMMIT} + commit: ${TAG} + checksum: ${COMMIT} keepGitDir: true generate: - gomod: {}