fix: git access tar methods (#1454)

The method to tar a filesystem that is currently used, also flattens the
directory structure to a simple list of files. As this is not what we
need, we will use another method.

Additionally, the tar-function includes the "last-modification" date of
the files. This leads to false results (e.g. digest calculation) as the
modification date for files is resetted when downloaded.

---------

Co-authored-by: Jakob Möller <jakob.moeller@sap.com>

Author: frewilhelm

api/utils/blobaccess/git/access.go

@@ -2,6 +2,7 @@ package git
 
 import (
 	"context"
+	"fmt"
 
 	gogit "github.com/go-git/go-git/v5"
 	"github.com/mandelsoft/goutils/errors"
@@ -87,8 +88,8 @@ func BlobAccess(opt ...Option) (_ bpi.BlobAccess, rerr error) {
 	dw := iotools.NewDigestWriterWith(digest.SHA256, tgz)
 	finalize.Close(dw)
 
-	if err := tarutils.TgzFs(filteredRepositoryFS, dw); err != nil {
-		return nil, err
+	if err := tarutils.TgzFs(filteredRepositoryFS, dw, tarutils.TarFileSystemOptions{ZeroModTime: true}); err != nil {
+		return nil, fmt.Errorf("failed to create tgz: %w", err)
 	}
 
 	log.Debug("created", "file", tgz.Name())

api/utils/blobaccess/git/access_test.go

@@ -2,13 +2,13 @@ package git_test
 
 import (
 	"embed"
+	_ "embed"
 	"fmt"
 	"io"
+	"io/fs"
 	"os"
 	"time"
 
-	_ "embed"
-
 	. "github.com/mandelsoft/goutils/testutils"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
@@ -56,26 +56,39 @@ var _ = Describe("git Blob Access", func() {
 			repoTestData, err := testData.ReadDir(repoBase)
 			Expect(err).ToNot(HaveOccurred())
 
-			for _, entry := range repoTestData {
-				path := filepath.Join(repoBase, entry.Name())
-				repoPath := filepath.Join(repoDir, entry.Name())
-
-				file, err := testData.Open(path)
-				Expect(err).ToNot(HaveOccurred())
-
-				fileInRepo, err := os.OpenFile(
-					repoPath,
-					os.O_CREATE|os.O_RDWR|os.O_TRUNC,
-					0o600,
-				)
-				Expect(err).ToNot(HaveOccurred())
-
-				_, err = io.Copy(fileInRepo, file)
-				Expect(err).ToNot(HaveOccurred())
-
-				Expect(fileInRepo.Close()).To(Succeed())
-				Expect(file.Close()).To(Succeed())
+			var process func(base string, entries []fs.DirEntry)
+			process = func(base string, infos []fs.DirEntry) {
+				for _, entry := range infos {
+					path := filepath.Join(base, entry.Name())
+					repoPath := filepath.Join(repoDir, entry.Name())
+
+					file, err := testData.Open(path)
+					Expect(err).ToNot(HaveOccurred())
+					fi, err := file.Stat()
+					Expect(err).ToNot(HaveOccurred())
+					if fi.IsDir() {
+						Expect(os.MkdirAll(repoPath, 0o700)).ToNot(HaveOccurred())
+						entries, err := testData.ReadDir(path)
+						Expect(err).ToNot(HaveOccurred())
+						process(filepath.Join(base, fi.Name()), entries)
+						continue
+					}
+
+					fileInRepo, err := os.OpenFile(
+						repoPath,
+						os.O_CREATE|os.O_RDWR|os.O_TRUNC,
+						0o600,
+					)
+					Expect(err).ToNot(HaveOccurred())
+
+					_, err = io.Copy(fileInRepo, file)
+					Expect(err).ToNot(HaveOccurred())
+
+					Expect(fileInRepo.Close()).To(Succeed())
+					Expect(file.Close()).To(Succeed())
+				}
 			}
+			process(repoBase, repoTestData)
 
 			wt, err := repo.Worktree()
 			Expect(err).ToNot(HaveOccurred())
@@ -100,7 +113,22 @@ var _ = Describe("git Blob Access", func() {
 			))
 			defer Close(b)
 			files := Must(tarutils.ListArchiveContentFromReader(Must(b.Reader())))
-			Expect(files).To(ConsistOf("file_in_repo"))
+			Expect(files).To(ConsistOf("file_in_repo", "file_in_dir_in_repo"))
+
+			data1, err := b.Reader()
+			Expect(err).ToNot(HaveOccurred())
+
+			By("testing equivalence of byte streams from equivalent accesses")
+			b2 := Must(gitblob.BlobAccess(
+				gitblob.WithURL(url),
+				gitblob.WithLoggingContext(ctx),
+				gitblob.WithCachingContext(ctx),
+			))
+			defer Close(b2)
+			data2, err := b2.Reader()
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(data1).ToNot(BeIdenticalTo(data2))
 		})
 	})
 
@@ -151,4 +179,5 @@ var _ = Describe("git Blob Access", func() {
 			Expect(files).To(ConsistOf("README", "CONTRIBUTING.md"))
 		})
 	})
+
 })

api/utils/blobaccess/git/testdata/repo/dir/file_in_dir_in_repo

@@ -0,0 +1 @@
+Foobar

api/utils/blobaccess/maven/utils.go

@@ -118,7 +118,7 @@ func (s *spec) getBlobAccess() (_ bpi.BlobAccess, rerr error) {
 	dw := iotools.NewDigestWriterWith(digest.SHA256, tgz)
 	finalize.Close(dw)
 
-	err = tarutils.TgzFs(tmpfs, dw)
+	err = tarutils.TgzFlatFs(tmpfs, dw)
 	if err != nil {
 		return nil, err
 	}

api/utils/tarutils/pack.go

@@ -12,6 +12,7 @@ import (
 	"time"
 
 	"github.com/mandelsoft/filepath/pkg/filepath"
+	"github.com/mandelsoft/goutils/errors"
 	"github.com/mandelsoft/goutils/finalizer"
 	"github.com/mandelsoft/goutils/general"
 	"github.com/mandelsoft/vfs/pkg/osfs"
@@ -46,7 +47,11 @@ type TarFileSystemOptions struct {
 	ExcludeFiles []string
 	// PreserveDir defines that the directory specified in the Path field should be included in the blob.
 	// Only supported for Type dir.
-	PreserveDir    bool
+	PreserveDir bool
+	// ZeroModTime defines that the modtime of the files added to the tar should be zeroed. This is important
+	// if the TAR archives need to be comparable on byte level (e.g. for hashing). To get fully byte-equivalent
+	// TAR archives at different timestamps, the mod time needs to be set to 0.
+	ZeroModTime    bool
 	FollowSymlinks bool
 
 	root string
@@ -119,6 +124,10 @@ func addFileToTar(fs vfs.FileSystem, tw *tar.Writer, path string, realPath strin
 	}
 	header.Name = path
 
+	if opts.ZeroModTime {
+		header.ModTime = time.Time{}
+	}
+
 	switch {
 	case info.IsDir():
 		// do not write root header
@@ -241,9 +250,18 @@ func ListSortedFilesInDir(fs vfs.FileSystem, root string, flat bool) ([]string,
 	return files, err
 }
 
-// TgzFs creates a tar.gz archive from a filesystem with all files being in the root of the zipped archive.
+// TgzFs works like PackFsIntoTar, but compresses the tar archive with [gzip.NewWriter].
+func TgzFs(fs vfs.FileSystem, writer io.Writer, options TarFileSystemOptions) (err error) {
+	zip := gzip.NewWriter(writer)
+	defer func() {
+		err = errors.Join(err, zip.Close())
+	}()
+	return PackFsIntoTar(fs, "", zip, options)
+}
+
+// TgzFlatFs creates a tar.gz archive from a filesystem with all files being in the root of the zipped archive.
 // The writer is closed after the archive is written. The TAR-headers are normalized, see RegularFileInfoHeader.
-func TgzFs(fs vfs.FileSystem, writer io.Writer) error {
+func TgzFlatFs(fs vfs.FileSystem, writer io.Writer) error {
 	zip := gzip.NewWriter(writer)
 	err := TarFlatFs(fs, zip)
 	if err != nil {

api/utils/tarutils/pack_test.go

@@ -1,11 +1,13 @@
 package tarutils_test
 
 import (
+	"bytes"
 	"io/fs"
 	"os"
 	"runtime"
 
 	. "github.com/mandelsoft/goutils/testutils"
+	"github.com/mandelsoft/vfs/pkg/memoryfs"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 
@@ -55,4 +57,29 @@ var _ = Describe("tar utils mapping", func() {
 			Expect(err.Error()).To(ContainSubstring("no such file or directory"))
 		}
 	})
+
+	It("test byte-equivalent compressed archives", func() {
+		fs := memoryfs.New()
+		f1, err := fs.Create("some file")
+		Expect(err).ToNot(HaveOccurred())
+		_, err = f1.Write([]byte("some content"))
+		Expect(err).ToNot(HaveOccurred())
+		Expect(f1.Close()).ToNot(HaveOccurred())
+
+		var buf1, buf2 bytes.Buffer
+
+		Expect(tarutils.TgzFs(fs, &buf1, tarutils.TarFileSystemOptions{
+			ZeroModTime: true,
+		})).To(Succeed())
+
+		Expect(tarutils.TgzFs(fs, &buf2, tarutils.TarFileSystemOptions{
+			ZeroModTime: true,
+		})).To(Succeed())
+
+		Expect(buf1.Bytes()).To(Equal(buf2.Bytes()))
+
+		var buf3 bytes.Buffer
+		Expect(tarutils.TgzFs(fs, &buf3, tarutils.TarFileSystemOptions{})).To(Succeed())
+		Expect(buf1.Bytes()).ToNot(Equal(buf3.Bytes()))
+	})
 })