Merge pull request #1170 from fluxcd/feat-targz

tar: Add support for creating gzip-compressed tarballs

Author: stefanprodan

http/fetch/archive_fetcher.go

@@ -46,7 +46,7 @@ type ArchiveFetcher struct {
 	retries           int
 	maxDownloadSize   int
 	fileMode          fs.FileMode
-	untarOpts         []tar.TarOption
+	untarOpts         []tar.Option
 	hostnameOverwrite string
 	filename          string
 	logger            any
@@ -75,9 +75,9 @@ func WithMaxDownloadSize(maxDownloadSize int) Option {
 }
 
 // WithUntar tells the ArchiveFetcher to untar the archive expecting it to be a tarball.
-func WithUntar(opts ...tar.TarOption) Option {
+func WithUntar(opts ...tar.Option) Option {
 	return func(a *ArchiveFetcher) {
-		a.untarOpts = append([]tar.TarOption{}, opts...) // to make sure a.untarOpts won't be nil
+		a.untarOpts = append([]tar.Option{}, opts...) // to make sure a.untarOpts won't be nil
 	}
 }
 

oci/build.go

@@ -17,17 +17,15 @@ limitations under the License.
 package oci
 
 import (
-	"archive/tar"
-	"compress/gzip"
 	"fmt"
 	"io"
 	"os"
 	"path/filepath"
 	"strings"
-	"time"
 
 	"github.com/fluxcd/pkg/oci/internal/fs"
 	"github.com/fluxcd/pkg/sourceignore"
+	"github.com/fluxcd/pkg/tar"
 )
 
 // Build archives the given directory as a tarball to the given local path.
@@ -37,17 +35,20 @@ func (c *Client) Build(artifactPath, sourceDir string, ignorePaths []string) (er
 }
 
 func build(artifactPath, sourceDir string, ignorePaths []string) (err error) {
-	absDir, err := filepath.Abs(sourceDir)
+	absSrc, err := filepath.Abs(sourceDir)
 	if err != nil {
 		return err
 	}
 
-	dirStat, err := os.Stat(absDir)
-	if os.IsNotExist(err) {
-		return fmt.Errorf("invalid source dir path: %s", absDir)
+	srcInfo, err := os.Stat(absSrc)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return fmt.Errorf("source path does not exist: %s", absSrc)
+		}
+		return fmt.Errorf("invalid source path %s: %w", absSrc, err)
 	}
 
-	tf, err := os.CreateTemp(filepath.Split(absDir))
+	tf, err := os.CreateTemp(filepath.Split(absSrc))
 	if err != nil {
 		return err
 	}
@@ -58,110 +59,60 @@ func build(artifactPath, sourceDir string, ignorePaths []string) (err error) {
 		}
 	}()
 
-	ignore := strings.Join(ignorePaths, "\n")
-	domain := strings.Split(filepath.Clean(absDir), string(filepath.Separator))
-	ps := sourceignore.ReadPatterns(strings.NewReader(ignore), domain)
-	matcher := sourceignore.NewMatcher(ps)
-	filter := func(p string, fi os.FileInfo) bool {
-		return matcher.Match(strings.Split(p, string(filepath.Separator)), fi.IsDir())
-	}
-
-	sz := &writeCounter{}
-	mw := io.MultiWriter(tf, sz)
-
-	gw := gzip.NewWriter(mw)
-	tw := tar.NewWriter(gw)
-	if err := filepath.Walk(absDir, func(p string, fi os.FileInfo, err error) error {
-		if err != nil {
-			return err
-		}
-
-		// Ignore anything that is not a file or directories e.g. symlinks
-		if m := fi.Mode(); !(m.IsRegular() || m.IsDir()) {
-			return nil
-		}
-
-		if len(ignorePaths) > 0 && filter(p, fi) {
-			return nil
+	// If the source is a single file, stage it in a temp dir so Tar can
+	// archive it as a directory tree containing that one entry.
+	tarDir := absSrc
+	if !srcInfo.IsDir() {
+		stage, stageErr := os.MkdirTemp("", "oci-build-")
+		if stageErr != nil {
+			tf.Close()
+			return stageErr
 		}
+		defer os.RemoveAll(stage)
 
-		header, err := tar.FileInfoHeader(fi, p)
-		if err != nil {
-			return err
-		}
-		if dirStat.IsDir() {
-			// The name needs to be modified to maintain directory structure
-			// as tar.FileInfoHeader only has access to the base name of the file.
-			// Ref: https://golang.org/src/archive/tar/common.go?#L6264
-			//
-			// we only want to do this if a directory was passed in
-			relFilePath, err := filepath.Rel(absDir, p)
-			if err != nil {
-				return err
-			}
-			// Normalize file path so it works on windows
-			header.Name = filepath.ToSlash(relFilePath)
-		}
-
-		// Remove any environment specific data.
-		header.Gid = 0
-		header.Uid = 0
-		header.Uname = ""
-		header.Gname = ""
-		header.ModTime = time.Time{}
-		header.AccessTime = time.Time{}
-		header.ChangeTime = time.Time{}
-
-		if err := tw.WriteHeader(header); err != nil {
-			return err
-		}
-
-		if !fi.Mode().IsRegular() {
-			return nil
-		}
-		f, err := os.Open(p)
-		if err != nil {
-			f.Close()
-			return err
-		}
-		if _, err := io.Copy(tw, f); err != nil {
-			f.Close()
+		if err := copyFileContents(filepath.Join(stage, srcInfo.Name()), absSrc, srcInfo.Mode()); err != nil {
+			tf.Close()
 			return err
 		}
-		return f.Close()
-	}); err != nil {
-		tw.Close()
-		gw.Close()
-		tf.Close()
-		return err
+		tarDir = stage
 	}
 
-	if err := tw.Close(); err != nil {
-		gw.Close()
-		tf.Close()
-		return err
+	ignore := strings.Join(ignorePaths, "\n")
+	domain := strings.Split(filepath.Clean(tarDir), string(filepath.Separator))
+	ps := sourceignore.ReadPatterns(strings.NewReader(ignore), domain)
+	matcher := sourceignore.NewMatcher(ps)
+	filter := func(p string, fi os.FileInfo) bool {
+		return matcher.Match(strings.Split(p, string(filepath.Separator)), fi.IsDir())
 	}
-	if err := gw.Close(); err != nil {
+
+	if _, err := tar.Tar(tarDir, tf, tar.WithFilter(filter)); err != nil {
 		tf.Close()
 		return err
 	}
 	if err := tf.Close(); err != nil {
 		return err
 	}
-
 	if err := os.Chmod(tmpName, 0o640); err != nil {
 		return err
 	}
-
 	return fs.RenameWithFallback(tmpName, artifactPath)
 }
 
-type writeCounter struct {
-	written int64
-}
-
-func (wc *writeCounter) Write(p []byte) (int, error) {
-	n := len(p)
-	wc.written += int64(n)
-	return n, nil
+func copyFileContents(dst, src string, mode os.FileMode) (err error) {
+	sf, err := os.Open(src)
+	if err != nil {
+		return err
+	}
+	defer sf.Close()
+	df, err := os.OpenFile(dst, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, mode.Perm())
+	if err != nil {
+		return err
+	}
+	defer func() {
+		if closeErr := df.Close(); closeErr != nil && err == nil {
+			err = closeErr
+		}
+	}()
+	_, err = io.Copy(df, sf)
+	return err
 }

tar/doc.go

@@ -0,0 +1,96 @@
+/*
+Copyright 2026 The Flux authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package tar provides utilities for creating and extracting tar
+// archives, with optional gzip compression. Tar writes a sanitized
+// archive of a directory tree, skipping symlinks and other non-regular,
+// non-directory entries; use ResolveSymlinks (or the confined
+// ResolveSymlinksRoot) to materialize symlink targets before archiving.
+// Untar safely extracts a tar archive into a target directory,
+// rejecting path traversal and capping the total decompressed size.
+//
+// # Creating an archive
+//
+// Archive a directory tree to a file as a gzip-compressed tarball:
+//
+//	f, err := os.Create("archive.tar.gz")
+//	if err != nil {
+//		return err
+//	}
+//	defer f.Close()
+//
+//	if _, err := tar.Tar("/path/to/dir", f); err != nil {
+//		return err
+//	}
+//
+// Exclude entries with a filter and write a plain (non-gzipped) tar:
+//
+//	skipHidden := func(p string, fi os.FileInfo) bool {
+//		return strings.HasPrefix(fi.Name(), ".")
+//	}
+//	_, err := tar.Tar("/path/to/dir", f,
+//		tar.WithFilter(skipHidden),
+//		tar.WithSkipGzip(),
+//	)
+//
+// # Extracting an archive
+//
+// Extract a gzip-compressed tarball into a directory:
+//
+//	f, err := os.Open("archive.tar.gz")
+//	if err != nil {
+//		return err
+//	}
+//	defer f.Close()
+//
+//	if err := tar.Untar(f, "/path/to/target"); err != nil {
+//		return err
+//	}
+//
+// Raise the size limit and tolerate symlinks in the archive:
+//
+//	err := tar.Untar(f, "/path/to/target",
+//		tar.WithMaxUntarSize(500<<20), // 500 MiB
+//		tar.WithSkipSymlinks(),
+//	)
+//
+// # Archiving with symlinks resolved
+//
+// By default Tar skips symlinks. For inputs where files live behind
+// symlinks (for example, manifest trees generated by Nix), stage the
+// source into a caller-owned directory with ResolveSymlinks first,
+// then archive the resolved tree:
+//
+//	tmpDir, err := os.MkdirTemp("", "resolve-")
+//	if err != nil {
+//		return err
+//	}
+//	defer os.RemoveAll(tmpDir)
+//
+//	if err := tar.ResolveSymlinks("/path/to/dir", tmpDir); err != nil {
+//		return err
+//	}
+//
+//	if _, err := tar.Tar(tmpDir, w); err != nil {
+//		return err
+//	}
+//
+// For untrusted source trees, use ResolveSymlinksRoot to confine every
+// symlink target inside a caller-supplied rootDir. Targets that resolve
+// outside rootDir cause the call to fail without materializing them:
+//
+//	err := tar.ResolveSymlinksRoot("/path/to/root", "/path/to/root/src", tmpDir)
+package tar

tar/options.go

@@ -0,0 +1,81 @@
+/*
+Copyright 2026 The Flux authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tar
+
+import "os"
+
+// Option configures the behavior of Tar and Untar. Options are
+// silently ignored by operations they do not apply to.
+type Option func(*tarOpts)
+
+type tarOpts struct {
+	// maxUntarSize represents the limit size (bytes) for archives being decompressed by Untar.
+	// When max is a negative value the size checks are disabled.
+	maxUntarSize int
+
+	// skipSymlinks ignores symlinks instead of failing the decompression.
+	skipSymlinks bool
+
+	// skipGzip disables gzip compression: Tar writes a plain tar stream,
+	// and Untar reads one.
+	skipGzip bool
+
+	// filter is called for each entry during archiving or extraction.
+	// If it returns true, the entry is excluded.
+	filter func(path string, fi os.FileInfo) bool
+}
+
+// WithMaxUntarSize sets the limit size for archives being decompressed by Untar.
+// When max is equal or less than 0 disables size checks.
+func WithMaxUntarSize(max int) Option {
+	return func(t *tarOpts) {
+		t.maxUntarSize = max
+	}
+}
+
+// WithSkipSymlinks allows for symlinks to be present
+// in the tarball and skips them when decompressing.
+func WithSkipSymlinks() Option {
+	return func(t *tarOpts) {
+		t.skipSymlinks = true
+	}
+}
+
+// WithSkipGzip disables gzip compression: Tar writes a plain tar stream,
+// and Untar reads one.
+func WithSkipGzip() Option {
+	return func(t *tarOpts) {
+		t.skipGzip = true
+	}
+}
+
+// WithFilter sets a predicate called for each entry during archiving
+// or extraction. Entries for which fn returns true are excluded. During
+// Tar the path is the absolute filesystem path; during Untar it is the
+// slash-separated name from the tar header.
+func WithFilter(fn func(path string, fi os.FileInfo) bool) Option {
+	return func(t *tarOpts) {
+		t.filter = fn
+	}
+}
+
+// applyOpts applies the given Option to t.
+func (t *tarOpts) applyOpts(opts ...Option) {
+	for _, opt := range opts {
+		opt(t)
+	}
+}