Add Go coverage reporting to CI (end-to-end)

kartikjoshi21 · kartikjoshi21 · commit 124edfa8068f · 2026-03-05T07:23:26.000Z
This PR adds automated Go coverage reporting to the CI pipeline. Unit and integration tests now generate coverage profiles (one per integration matrix suite). It also collects coverage for the Dalec BuildKit frontend by building it with coverage enabled and exporting its covdata during solves. A new coverage-report job downloads all profiles, merges them, and uploads a combined report (including an index.html coverage page) as a workflow artifact, plus a summary in the Actions run. Fixes: #889 Signed-off-by: Kartik Joshi <karikjoshi21@gmail.com>
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -270,6 +270,7 @@ jobs:
           set -eu
 
           docker buildx bake frontend
+
           if [ "${TEST_SUITE}" = "other" ]; then
             exit 0
           fi
@@ -283,19 +284,64 @@ jobs:
           docker buildx bake worker
         env:
           TEST_SUITE: ${{ matrix.suite }}
-      - name: Run integration tests
+      - name: Run integration tests (with coverage tracking)
         run: |
           set -ex
-          if [ -n "${TEST_SUITE}" ] && [ ! "${TEST_SUITE}" = "other" ]; then
+          mkdir -p coverage
+
+          # The frontend covdata files (covmeta/covcounters) are written by the test harness
+          # (writeFrontendCovdata) on the RUNNER filesystem.
+          export DALEC_FRONTEND_GOCOVERDIR="${GITHUB_WORKSPACE}/coverage/frontend-${TEST_SUITE}"
+          rm -rf "${DALEC_FRONTEND_GOCOVERDIR}"
+          mkdir -p "${DALEC_FRONTEND_GOCOVERDIR}"
+
+          run=""
+          skip=""
+          if [ -n "${TEST_SUITE}" ] && [ "${TEST_SUITE}" != "other" ]; then
             run="-run=${TEST_SUITE}"
           fi
           if [ -n "${TEST_SKIP}" ]; then
             skip="-skip=${TEST_SKIP}"
           fi
-          go test -timeout=59m -v -json ${run} ${skip} ./test | go run ./cmd/test2json2gha --slow 120s --logdir /tmp/testlogs
+
+          go test -timeout=59m -v -json \
+            -covermode=atomic -coverpkg=./... \
+            -coverprofile="coverage/integration-${TEST_SUITE}.out" \
+            ${run} ${skip} ./test \
+            | go run ./cmd/test2json2gha --slow 120s --logdir /tmp/testlogs
+
+          # Convert frontend covdata -> legacy coverprofile
+          if ! ls "${DALEC_FRONTEND_GOCOVERDIR}"/covmeta.* >/dev/null 2>&1; then
+            echo "::group::frontend coverage debug"
+            echo "DALEC_FRONTEND_GOCOVERDIR=${DALEC_FRONTEND_GOCOVERDIR}"
+            echo "Contents:"
+            ls -la "${DALEC_FRONTEND_GOCOVERDIR}" || true
+            echo "Searching workspace for covmeta/covcounters..."
+            find "${GITHUB_WORKSPACE}" \( -name 'covmeta.*' -o -name 'covcounters.*' \) 2>/dev/null | head -n 200 || true
+            echo "::endgroup::"
+            echo "::error::No frontend coverage covmeta.* found in ${DALEC_FRONTEND_GOCOVERDIR} (frontend coverage not collected)"
+            exit 1
+          fi
+
+          go tool covdata textfmt \
+            -i="${DALEC_FRONTEND_GOCOVERDIR}" \
+            -o="coverage/frontend-${TEST_SUITE}.out"
         env:
           TEST_SUITE: ${{ matrix.suite }}
           TEST_SKIP: ${{ matrix.skip }}
+
+
+      - name: Upload integration coverage profile
+        if: always()
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
+        with:
+          name: coverage-integration-${{ matrix.suite }}
+          path: |
+            coverage/integration-${{ matrix.suite }}.out
+            coverage/frontend-${{ matrix.suite }}.out  
+          if-no-files-found: ignore
+          retention-days: 7
+
       - name: Get traces
         if: always()
         run: |
@@ -354,8 +400,27 @@ jobs:
           cache: false
       - name: download deps
         run: go mod download
-      - name: Run unit tests
-        run: go test -v --test.short --json ./... | go run ./cmd/test2json2gha
+      - name: Run unit tests (with coverage tracking)
+        run: |
+          set -eux
+          mkdir -p coverage
+
+          pkgs="$(go list ./... | grep -v '/test$' | grep -v '/test/' )"
+          go test -v --test.short --json \
+            -covermode=atomic \
+            -coverprofile="coverage/unit.out" \
+            ${pkgs} \
+            | go run ./cmd/test2json2gha
+      - name: Upload unit coverage profile
+        if: always()
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
+        with:
+          name: coverage-unit
+          path: coverage/unit.out
+          if-no-files-found: ignore
+          retention-days: 7
+
+
 
   e2e:
     runs-on: ubuntu-22.04
@@ -443,3 +508,82 @@ jobs:
           path: ${{ steps.dump-logs.outputs.DOCKERD_LOG_PATH }}
           retention-days: 1
 
+  coverage-report:
+    runs-on: ubuntu-22.04
+    needs:
+      - unit
+      - integration
+
+    steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@e3f713f2d8f53843e71c69a996d56f51aa9adfb9 # v2.14.1
+        with:
+          egress-policy: audit
+
+      - name: Checkout
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+
+      - name: Setup Go
+        uses: actions/setup-go@7a3fe6cf4cb3a834922a1244abfce67bcef6a0c5 # v6.2.0
+        with:
+          go-version: "1.25"
+          cache: false
+
+      - name: Download deps
+        run: go mod download
+
+      - name: Download unit coverage artifact
+        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093
+        with:
+          name: coverage-unit
+          path: coverage
+
+      - name: Download integration coverage artifacts
+        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093
+        with:
+          path: coverage/_integration
+
+      - name: Merge coverage + generate report
+        run: |
+          set -eux
+          go install github.com/wadey/gocovmerge@latest
+
+          integration_profiles="$(find coverage/_integration -type f -name 'integration-*.out' | sort | tr '\n' ' ')"
+          frontend_profiles="$(find coverage/_integration -type f -name 'frontend-*.out' | sort | tr '\n' ' ')"
+          if [ -z "${integration_profiles}" ]; then
+            echo "::error::No integration coverage profiles found"
+            exit 1
+          fi
+
+          if [ -z "${frontend_profiles}" ]; then
+            echo "::error::No frontend coverage profiles found"
+            exit 1
+          fi
+
+          if [ ! -f coverage/unit.out ]; then
+            echo "::error::Unit coverage profile not found (coverage/unit.out)"
+            exit 1
+          fi
+
+          "$(go env GOPATH)/bin/gocovmerge" coverage/unit.out ${integration_profiles} ${frontend_profiles} > coverage/all.out
+
+          go tool cover -func=coverage/all.out | tee coverage/summary.txt
+          go tool cover -html=coverage/all.out -o coverage/index.html
+
+          total="$(tail -n 1 coverage/summary.txt | awk '{print $3}')"
+          {
+            echo "## Coverage"
+            echo
+            echo "- Total: **${total}**"
+            echo "- Profiles merged: $(echo "${integration_profiles}" | wc -w) integration + $(echo "${frontend_profiles}" | wc -w) frontend"
+          } >> "${GITHUB_STEP_SUMMARY}"
+
+      - name: Upload merged coverage report
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
+        with:
+          name: coverage-report
+          path: |
+            coverage/all.out
+            coverage/summary.txt
+            coverage/index.html
+          retention-days: 14
diff --git a/Dockerfile b/Dockerfile
@@ -5,11 +5,16 @@ WORKDIR /build
 COPY . .
 ENV CGO_ENABLED=0
 ARG TARGETARCH TARGETOS GOFLAGS=-trimpath
+ARG DALEC_FRONTEND_COVERAGE=0
 ENV GOOS=${TARGETOS} GOARCH=${TARGETARCH} GOFLAGS=${GOFLAGS}
 RUN \
     --mount=type=cache,target=/go/pkg/mod \
     --mount=type=cache,target=/root/.cache/go-build \
-    go build -o /frontend ./cmd/frontend
+    if [ "${DALEC_FRONTEND_COVERAGE}" = "1" ]; then \
+	go build -cover -covermode=atomic -coverpkg=./... -o /frontend ./cmd/frontend ; \
+    else \
+        go build -o /frontend ./cmd/frontend ; \
+    fi
 
 FROM scratch AS frontend
 COPY --from=frontend-build /frontend /frontend
diff --git a/cmd/frontend/coverage.go b/cmd/frontend/coverage.go
@@ -0,0 +1,105 @@
+// cmd/frontend/coverage.go
+package main
+
+import (
+	"bytes"
+	"compress/gzip"
+	"context"
+	"strings"
+
+	"runtime/coverage"
+
+	gwclient "github.com/moby/buildkit/frontend/gateway/client"
+)
+
+const (
+	frontendCoverageOptKey = "dalec.coverage"
+	frontendCovMetaKey     = "dalec.coverage.frontend.meta.gz"
+	frontendCovCountersKey = "dalec.coverage.frontend.counters.gz"
+)
+
+func isNoMetaErr(err error) bool {
+	if err == nil {
+		return false
+	}
+	// runtime/coverage: "no meta-data available (binary not built with -cover?)"
+	return strings.Contains(strings.ToLower(err.Error()), "no meta-data available")
+}
+
+// Enabled per solve via SolveRequest.FrontendOpt["dalec.coverage"]="1"
+func wantFrontendCoverage(c gwclient.Client) bool {
+	v, ok := c.BuildOpts().Opts[frontendCoverageOptKey]
+	if !ok {
+		return false
+	}
+	v = strings.ToLower(strings.TrimSpace(v))
+	return v == "1" || v == "true" || v == "yes" || v == "on"
+}
+
+func gzipBytes(in []byte) ([]byte, error) {
+	var buf bytes.Buffer
+	zw := gzip.NewWriter(&buf)
+	if _, err := zw.Write(in); err != nil {
+		_ = zw.Close()
+		return nil, err
+	}
+	if err := zw.Close(); err != nil {
+		return nil, err
+	}
+	return buf.Bytes(), nil
+}
+
+func attachFrontendCoverage(c gwclient.Client, res *gwclient.Result) error {
+	if res == nil || !wantFrontendCoverage(c) {
+		return nil
+	}
+	if res.Metadata == nil {
+		res.Metadata = map[string][]byte{}
+	}
+
+	var metaBuf, ctrBuf bytes.Buffer
+
+	if err := coverage.WriteMeta(&metaBuf); err != nil {
+		if isNoMetaErr(err) {
+			return nil
+		}
+		return err
+	}
+	if err := coverage.WriteCounters(&ctrBuf); err != nil {
+		if isNoMetaErr(err) {
+			return nil
+		}
+		return err
+	}
+
+	metaGz, err := gzipBytes(metaBuf.Bytes())
+	if err != nil {
+		return err
+	}
+	ctrGz, err := gzipBytes(ctrBuf.Bytes())
+	if err != nil {
+		return err
+	}
+
+	res.Metadata[frontendCovMetaKey] = metaGz
+	res.Metadata[frontendCovCountersKey] = ctrGz
+
+	// Avoid cross-solve accumulation if the frontend process is reused.
+	// Only works for binaries built with -cover (and typically atomic counters).
+	_ = coverage.ClearCounters()
+
+	return nil
+}
+
+func wrapWithCoverage(next gwclient.BuildFunc) gwclient.BuildFunc {
+	return func(ctx context.Context, c gwclient.Client) (*gwclient.Result, error) {
+		res, err := next(ctx, c)
+		if err != nil {
+			return nil, err
+		}
+		if err := attachFrontendCoverage(c, res); err != nil {
+			return nil, err
+		}
+		return res, nil
+	}
+}
diff --git a/cmd/frontend/main.go b/cmd/frontend/main.go
@@ -69,8 +69,10 @@ func dalecMain() {
 	if err != nil {
 		bklog.L.WithError(err).Fatal("error creating frontend router")
 	}
+	handler := mux.Handler(frontend.WithTargetForwardingHandler)
+	handler = wrapWithCoverage(handler)
 
-	if err := grpcclient.RunFromEnvironment(ctx, mux.Handler(frontend.WithTargetForwardingHandler)); err != nil {
+	if err := grpcclient.RunFromEnvironment(ctx, handler); err != nil {
 		bklog.L.WithError(err).Fatal("error running frontend")
 		os.Exit(70) // 70 is EX_SOFTWARE, meaning internal software error occurred
 	}
diff --git a/test/testenv/build.go b/test/testenv/build.go
@@ -50,10 +50,17 @@ func buildBaseFrontend(ctx context.Context, c gwclient.Client) (*gwclient.Result
 		return nil, errors.Wrap(err, "error marshaling Dockerfile context")
 	}
 
+	// If the test runner requested frontend coverage, build the frontend binary
+	// with coverage instrumentation (Dockerfile uses DALEC_FRONTEND_COVERAGE).
+	frontendOpt := map[string]string{}
+	if os.Getenv("DALEC_FRONTEND_GOCOVERDIR") != "" {
+		frontendOpt["build-arg:DALEC_FRONTEND_COVERAGE"] = "1"
+	}
+
 	defPB := def.ToPB()
 	return c.Solve(ctx, gwclient.SolveRequest{
 		Frontend:    "dockerfile.v0",
-		FrontendOpt: map[string]string{},
+		FrontendOpt: frontendOpt,
 		FrontendInputs: map[string]*pb.Definition{
 			dockerui.DefaultLocalNameContext:    defPB,
 			dockerui.DefaultLocalNameDockerfile: dockerfileDef.ToPB(),
diff --git a/test/testenv/buildx.go b/test/testenv/buildx.go
@@ -11,6 +11,7 @@ import (
 	"net"
 	"os"
 	"os/exec"
+	"path/filepath"
 	"strings"
 	"sync"
 	"testing"
@@ -420,14 +421,36 @@ type clientForceDalecWithInput struct {
 }
 
 func (c *clientForceDalecWithInput) Solve(ctx context.Context, req gwclient.SolveRequest) (*gwclient.Result, error) {
+	covRoot := os.Getenv("DALEC_FRONTEND_GOCOVERDIR")
 	if req.Definition == nil {
 		// Only inject the frontend when there is no "definition" set.
 		// If a definition is set, it is intended for this to go directly to the buildkit solver.
 		if err := withDalecInput(ctx, c.Client, &req); err != nil {
 			return nil, err
 		}
 	}
-	return c.Client.Solve(ctx, req)
+
+	// IMPORTANT: set this *after* withDalecInput, since it may replace/normalize FrontendOpt.
+	if covRoot != "" {
+		if req.FrontendOpt == nil {
+			req.FrontendOpt = map[string]string{}
+		}
+		// Frontend-only toggle (NOT a dalec build arg)
+		req.FrontendOpt["dalec.coverage"] = "1"
+	}
+	res, err := c.Client.Solve(ctx, req)
+	if err != nil {
+		return nil, err
+	}
+
+	if covRoot != "" {
+		// Make it suite-safe by letting CI set DALEC_FRONTEND_GOCOVERDIR per-suite.
+		// If they set a plain directory, we write directly there.
+		if err := writeFrontendCovdata(filepath.Clean(covRoot), res); err != nil {
+			return nil, err
+		}
+	}
+	return res, nil
 }
 
 // gwClientInputInject is a gwclient.Client that injects the result of a build func into the solve request as an input named by the id.
diff --git a/test/testenv/frontend_coverage.go b/test/testenv/frontend_coverage.go

Original file line number	Diff line number	Diff line change
`@@ -69,8 +69,10 @@ func dalecMain() {`
`69`	`69`	`if err != nil {`
`70`	`70`	`bklog.L.WithError(err).Fatal("error creating frontend router")`
`71`	`71`	`}`
	`72`	`+ handler := mux.Handler(frontend.WithTargetForwardingHandler)`
	`73`	`+ handler = wrapWithCoverage(handler)`
`72`	`74`
`73`		`- if err := grpcclient.RunFromEnvironment(ctx, mux.Handler(frontend.WithTargetForwardingHandler)); err != nil {`
	`75`	`+ if err := grpcclient.RunFromEnvironment(ctx, handler); err != nil {`
`74`	`76`	`bklog.L.WithError(err).Fatal("error running frontend")`
`75`	`77`	`os.Exit(70) // 70 is EX_SOFTWARE, meaning internal software error occurred`
`76`	`78`	`}`