conformance: graduate t5308-pack-detect-duplicates

Extends the conformance gate to cover pack-file processing.

- gogit cat-file: -e <oid> existence check (exit 0/1, no output) and
  --batch-check (stdin OIDs -> "<oid> <type> <size>" or "<oid> missing").
- gogit index-pack --stdin [--strict]: reads a pack from stdin and writes
  pack-<sha>.{pack,idx} into .git/objects/pack/ via go-git's
  WritePackfileToObjectStorage. --strict scans the pack with a
  WithScannerObservers-attached observer and rejects packs containing
  duplicate object IDs before any state is written.
- Harness test-tool stub gains sha1/sha256 [-b] cases (used by
  t/lib-pack.sh to compute pack trailers) and is now rewritten on every
  run so script changes take effect without manual cache invalidation.
- conformance/tests.txt: append t5308-pack-detect-duplicates.sh.

Assisted-by: Claude Opus 4.7 <noreply@anthropic.com>
Signed-off-by: Paulo Gomes <paulo@entire.io>

Author: pjbgf

cmd/gogit/cat-file.go

@@ -0,0 +1,94 @@
+package main
+
+import (
+	"bufio"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"strings"
+
+	"github.com/go-git/go-git/v6"
+	"github.com/go-git/go-git/v6/plumbing"
+	"github.com/spf13/cobra"
+)
+
+var (
+	catFileExists     bool
+	catFileBatchCheck bool
+)
+
+func init() {
+	catFileCmd.Flags().BoolVarP(&catFileExists, "exists", "e", false,
+		"Check whether object exists; exit 0 if so, 1 otherwise")
+	catFileCmd.Flags().BoolVar(&catFileBatchCheck, "batch-check", false,
+		"Read object IDs from stdin and print <oid> <type> <size> per line (or '<oid> missing')")
+	rootCmd.AddCommand(catFileCmd)
+}
+
+var catFileCmd = &cobra.Command{
+	Use:   "cat-file (-e <oid> | --batch-check)",
+	Short: "Provide content or check existence of repository objects",
+	RunE: func(cmd *cobra.Command, args []string) error {
+		if catFileExists && catFileBatchCheck {
+			return errors.New("-e and --batch-check are mutually exclusive")
+		}
+
+		if !catFileExists && !catFileBatchCheck {
+			return errors.New("one of -e or --batch-check is required")
+		}
+
+		r, err := git.PlainOpenWithOptions(".", &git.PlainOpenOptions{DetectDotGit: true})
+		if err != nil {
+			return fmt.Errorf("failed to open repository: %w", err)
+		}
+
+		if catFileExists {
+			if len(args) != 1 {
+				return errors.New("-e requires exactly one <oid> argument")
+			}
+
+			return catFileExistsCheck(r, args[0])
+		}
+
+		return catFileBatchCheckRun(cmd, r, os.Stdin)
+	},
+	DisableFlagsInUseLine: true,
+	SilenceUsage:          true,
+	SilenceErrors:         true,
+}
+
+func catFileExistsCheck(r *git.Repository, oid string) error {
+	h := plumbing.NewHash(oid)
+	if _, err := r.Storer.EncodedObject(plumbing.AnyObject, h); err != nil {
+		os.Exit(1)
+	}
+
+	return nil
+}
+
+func catFileBatchCheckRun(cmd *cobra.Command, r *git.Repository, stdin io.Reader) error {
+	w := bufio.NewWriter(cmd.OutOrStdout())
+	defer w.Flush()
+
+	scanner := bufio.NewScanner(stdin)
+	for scanner.Scan() {
+		line := strings.TrimSpace(scanner.Text())
+		if line == "" {
+			continue
+		}
+
+		h := plumbing.NewHash(line)
+
+		obj, err := r.Storer.EncodedObject(plumbing.AnyObject, h)
+		if err != nil {
+			fmt.Fprintf(w, "%s missing\n", line)
+
+			continue
+		}
+
+		fmt.Fprintf(w, "%s %s %d\n", line, obj.Type(), obj.Size())
+	}
+
+	return scanner.Err()
+}

cmd/gogit/cat-file_test.go

@@ -0,0 +1,82 @@
+package main
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+const baseBlobOID = "df967b96a579e45a18b8251732d16804b2e56a55" // sha1 of "blob 5\0base\n"
+
+func setupRepoWithBaseBlob(t *testing.T) string {
+	t.Helper()
+
+	repo := t.TempDir()
+
+	if _, _, err := runGogit(t, repo, "init"); err != nil {
+		t.Fatalf("init: %v", err)
+	}
+
+	if err := os.WriteFile(filepath.Join(repo, "file0"), []byte("base\n"), 0o644); err != nil {
+		t.Fatal(err)
+	}
+
+	if _, _, err := runGogit(t, repo, "add", "file0"); err != nil {
+		t.Fatalf("add: %v", err)
+	}
+
+	if _, _, err := runGogitEnv(t, repo, gitIdentityEnv(repo), "commit", "-m", "x"); err != nil {
+		t.Fatalf("commit: %v", err)
+	}
+
+	return repo
+}
+
+func TestCatFileExistsExitsZero(t *testing.T) {
+	t.Parallel()
+
+	repo := setupRepoWithBaseBlob(t)
+
+	if _, _, err := runGogit(t, repo, "cat-file", "-e", baseBlobOID); err != nil {
+		t.Fatalf("cat-file -e <existing>: expected exit 0, got %v", err)
+	}
+}
+
+func TestCatFileMissingExitsOne(t *testing.T) {
+	t.Parallel()
+
+	repo := t.TempDir()
+
+	if _, _, err := runGogit(t, repo, "init"); err != nil {
+		t.Fatalf("init: %v", err)
+	}
+
+	stdout, _, err := runGogit(t, repo, "cat-file", "-e", "0000000000000000000000000000000000000000")
+	if err == nil {
+		t.Fatalf("expected non-zero exit, got success")
+	}
+
+	if stdout != "" {
+		t.Fatalf("expected no stdout, got %q", stdout)
+	}
+}
+
+func TestCatFileBatchCheck(t *testing.T) {
+	t.Parallel()
+
+	repo := setupRepoWithBaseBlob(t)
+
+	const missingOID = "0000000000000000000000000000000000000000"
+
+	input := baseBlobOID + "\n" + missingOID + "\n"
+	want := baseBlobOID + " blob 5\n" + missingOID + " missing\n"
+
+	stdout, stderr, err := runGogitStdin(t, repo, input, "cat-file", "--batch-check")
+	if err != nil {
+		t.Fatalf("cat-file --batch-check failed: %v\nstderr: %s", err, stderr)
+	}
+
+	if stdout != want {
+		t.Fatalf("batch-check output mismatch:\n got: %q\nwant: %q", stdout, want)
+	}
+}

cmd/gogit/index-pack.go

@@ -0,0 +1,104 @@
+package main
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"io"
+
+	"github.com/go-git/go-git/v6"
+	"github.com/go-git/go-git/v6/plumbing"
+	"github.com/go-git/go-git/v6/plumbing/format/packfile"
+	"github.com/go-git/go-git/v6/plumbing/storer"
+	"github.com/spf13/cobra"
+)
+
+var (
+	indexPackStdin  bool
+	indexPackStrict bool
+)
+
+func init() {
+	indexPackCmd.Flags().BoolVar(&indexPackStdin, "stdin", false, "Read the pack from standard input")
+	indexPackCmd.Flags().BoolVar(&indexPackStrict, "strict", false, "Reject packs containing duplicate object IDs")
+	rootCmd.AddCommand(indexPackCmd)
+}
+
+var indexPackCmd = &cobra.Command{
+	Use:   "index-pack --stdin [--strict]",
+	Short: "Build a pack index for an existing packed archive",
+	RunE: func(cmd *cobra.Command, _ []string) error {
+		if !indexPackStdin {
+			return errors.New("--stdin is required")
+		}
+
+		r, err := git.PlainOpenWithOptions(".", &git.PlainOpenOptions{DetectDotGit: true})
+		if err != nil {
+			return fmt.Errorf("failed to open repository: %w", err)
+		}
+
+		return indexPackRun(r, cmd.InOrStdin(), indexPackStrict)
+	},
+	DisableFlagsInUseLine: true,
+	SilenceUsage:          true,
+	SilenceErrors:         true,
+}
+
+func indexPackRun(repo *git.Repository, in io.Reader, strict bool) error {
+	pw, ok := repo.Storer.(storer.PackfileWriter)
+	if !ok {
+		return errors.New("repository storer does not support packfile writes")
+	}
+
+	if !strict {
+		return packfile.WritePackfileToObjectStorage(pw, in)
+	}
+
+	buf, err := io.ReadAll(in)
+	if err != nil {
+		return fmt.Errorf("read pack: %w", err)
+	}
+
+	if err := checkPackForDuplicates(buf); err != nil {
+		return err
+	}
+
+	return packfile.WritePackfileToObjectStorage(pw, bytes.NewReader(buf))
+}
+
+func checkPackForDuplicates(pack []byte) error {
+	obs := &dupObserver{seen: make(map[plumbing.Hash]struct{})}
+	parser := packfile.NewParser(bytes.NewReader(pack), packfile.WithScannerObservers(obs))
+
+	if _, err := parser.Parse(); err != nil {
+		return fmt.Errorf("parse pack: %w", err)
+	}
+
+	if obs.dup != plumbing.ZeroHash {
+		return fmt.Errorf("duplicate object %s in pack (--strict)", obs.dup)
+	}
+
+	return nil
+}
+
+type dupObserver struct {
+	seen map[plumbing.Hash]struct{}
+	dup  plumbing.Hash
+}
+
+func (o *dupObserver) OnHeader(_ uint32) error        { return nil }
+func (o *dupObserver) OnFooter(_ plumbing.Hash) error { return nil }
+
+func (o *dupObserver) OnInflatedObjectHeader(_ plumbing.ObjectType, _, _ int64) error {
+	return nil
+}
+
+func (o *dupObserver) OnInflatedObjectContent(h plumbing.Hash, _ int64, _ uint32, _ []byte) error {
+	if _, ok := o.seen[h]; ok && o.dup == plumbing.ZeroHash {
+		o.dup = h
+	}
+
+	o.seen[h] = struct{}{}
+
+	return nil
+}

cmd/gogit/index-pack_test.go

@@ -0,0 +1,95 @@
+package main
+
+import (
+	"bytes"
+	"crypto/sha1"
+	"encoding/binary"
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+// emptyBlobPackEntry is the on-the-wire (zlib-compressed) packfile entry for the
+// canonical empty blob. Lifted verbatim from upstream's t/lib-pack.sh, which is
+// the same trick t5308 uses to construct test packs.
+var emptyBlobPackEntry = []byte{0x30, 0x78, 0x9c, 0x03, 0x00, 0x00, 0x00, 0x00, 0x01}
+
+// makePack returns a valid v2 packfile containing `count` copies of the empty
+// blob entry, with a correct SHA1 trailer.
+func makePack(t *testing.T, count int) []byte {
+	t.Helper()
+
+	var buf bytes.Buffer
+
+	buf.WriteString("PACK")
+
+	if err := binary.Write(&buf, binary.BigEndian, uint32(2)); err != nil {
+		t.Fatal(err)
+	}
+
+	if err := binary.Write(&buf, binary.BigEndian, uint32(count)); err != nil {
+		t.Fatal(err)
+	}
+
+	for range count {
+		buf.Write(emptyBlobPackEntry)
+	}
+
+	h := sha1.Sum(buf.Bytes())
+	buf.Write(h[:])
+
+	return buf.Bytes()
+}
+
+func TestIndexPackStdinAcceptsCleanPack(t *testing.T) {
+	t.Parallel()
+
+	repo := t.TempDir()
+
+	if _, _, err := runGogit(t, repo, "init"); err != nil {
+		t.Fatalf("init: %v", err)
+	}
+
+	pack := makePack(t, 1)
+	if _, stderr, err := runGogitStdin(t, repo, string(pack), "index-pack", "--stdin"); err != nil {
+		t.Fatalf("index-pack --stdin failed: %v\nstderr: %s", err, stderr)
+	}
+
+	matches, err := filepath.Glob(filepath.Join(repo, ".git", "objects", "pack", "pack-*.pack"))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if len(matches) != 1 {
+		t.Fatalf("expected exactly one pack file, got %d: %v", len(matches), matches)
+	}
+
+	idxPath := matches[0][:len(matches[0])-5] + ".idx"
+	if _, err := os.Stat(idxPath); err != nil {
+		t.Fatalf("expected idx alongside pack: %v", err)
+	}
+}
+
+func TestIndexPackStrictRejectsDuplicates(t *testing.T) {
+	t.Parallel()
+
+	repo := t.TempDir()
+
+	if _, _, err := runGogit(t, repo, "init"); err != nil {
+		t.Fatalf("init: %v", err)
+	}
+
+	pack := makePack(t, 2)
+	if _, _, err := runGogitStdin(t, repo, string(pack), "index-pack", "--strict", "--stdin"); err == nil {
+		t.Fatal("expected non-zero exit for duplicate-object pack under --strict")
+	}
+
+	matches, err := filepath.Glob(filepath.Join(repo, ".git", "objects", "pack", "pack-*.pack"))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if len(matches) != 0 {
+		t.Fatalf("expected no pack file left behind, got %d: %v", len(matches), matches)
+	}
+}