If applied, this commit will add an Extraction strategy to the ArtifactGenerator copy operations which will extract the content of matched tarball files to the output artifact. This is in reponse to the issue detailed here #301

Signed-off-by: Chris Morandi <chris.morandi@velocitaconsultancy.uk>

Author: Chris Morandi

api/v1beta1/artifactgenerator_types.go

@@ -37,6 +37,7 @@ const (
 	SourceFetchFailedReason          = "SourceFetchFailed"
 	OverwriteStrategy                = "Overwrite"
 	MergeStrategy                    = "Merge"
+	ExtractStrategy                  = "Extract"
 	EnabledValue                     = "enabled"
 	DisabledValue                    = "disabled"
 )
@@ -149,9 +150,13 @@ type CopyOperation struct {
 	// Strategy specifies the copy strategy to use.
 	// 'Overwrite' will overwrite existing files in the destination.
 	// 'Merge' is for merging YAML files using Helm values merge strategy.
+	// 'Extract' is for extracting the contents of tarball archives (.tar.gz, .tgz)
+	// built with flux build artifact or helm package. When using glob patterns,
+	// non-tarball files are silently skipped. For single file sources, the file
+	// must be a tarball or an error is returned. Directories are not supported.
 	// If not specified, defaults to 'Overwrite'.
 	// +optional
-	// +kubebuilder:validation:Enum=Overwrite;Merge
+	// +kubebuilder:validation:Enum=Overwrite;Merge;Extract
 	Strategy string `json:"strategy,omitempty"`
 }
 

config/crd/bases/source.extensions.fluxcd.io_artifactgenerators.yaml

@@ -85,10 +85,15 @@ spec:
                               Strategy specifies the copy strategy to use.
                               'Overwrite' will overwrite existing files in the destination.
                               'Merge' is for merging YAML files using Helm values merge strategy.
+                              'Extract' is for extracting the contents of tarball archives (.tar.gz, .tgz)
+                              built with flux build artifact or helm package. When using glob patterns,
+                              non-tarball files are silently skipped. For single file sources, the file
+                              must be a tarball or an error is returned. Directories are not supported.
                               If not specified, defaults to 'Overwrite'.
                             enum:
                             - Overwrite
                             - Merge
+                            - Extract
                             type: string
                           to:
                             description: |-

docs/spec/v1beta1/artifactgenerators.md

@@ -268,8 +268,8 @@ Each copy operation specifies how to copy files from sources into the generated
   the root of the generated artifact and `path` is the relative path to a file or directory.
 - `exclude` (optional): A list of glob patterns to filter out from the source selection.
   Any file matched by `from` that also matches an exclude pattern will be ignored.
-- `strategy` (optional): Defines how to handle existing files at the destination, 
-  either `Overwrite` (default) or `Merge` (for YAML files only).
+- `strategy` (optional): Defines how to handle files during copy operations:
+  `Overwrite` (default), `Merge` (for YAML files), or `Extract` (for tarball archives).
 
 Copy operations use `cp`-like semantics:
 
@@ -327,6 +327,35 @@ Example of copy with `Merge` strategy:
 **Note** that the merge strategy will replace _arrays_ entirely, the behavior is
 identical to how Helm merges `values.yaml` files when using multiple `--values` flags.
 
+##### Extract Strategy
+
+The `Extract` strategy is used for extracting the contents of tarball archives (`.tar.gz`, `.tgz`)
+built with `flux build artifact` or `helm package`. The tarball contents are extracted
+to the destination while preserving their internal directory structure.
+
+Example of copy with `Extract` strategy:
+
+```yaml
+# Extract a Helm chart tarball built with `helm package`
+- from: "@oci/podinfo-6.7.0.tgz"
+  to: "@artifact/"
+  strategy: Extract
+
+# Extract multiple tarballs using glob patterns
+- from: "@source/charts/*.tgz"
+  to: "@artifact/charts/"
+  strategy: Extract
+
+# Extract tarballs recursively from nested directories
+- from: "@source/releases/**/*.tgz"
+  to: "@artifact/"
+  strategy: Extract
+```
+
+**Note** that when using glob patterns (including recursive `**` patterns) with the `Extract`
+strategy, non-tarball files are silently skipped. For single file sources, the file must have
+a `.tar.gz` or `.tgz` extension. Directories are not supported with this strategy.
+
 ## Working with ArtifactGenerators
 
 ### Suspend and Resume Reconciliation

internal/builder/builder.go

@@ -127,6 +127,18 @@ func applyCopyOperations(ctx context.Context,
 	return nil
 }
 
+// If the copy operation uses the Extract strategy, it uses doublestar.Glob as we do not need to walk the whole tree
+// otherwise we us std fs.Glob
+func getGlobMatchingEntries(op swapi.CopyOperation, srcRoot *os.Root, srcPattern string) ([]string, error) {
+	if op.Strategy == swapi.ExtractStrategy {
+		// Use doublestar.Glob for recursive and advanced glob patterns (e.g., **/*.tar.gz)
+		return doublestar.Glob(srcRoot.FS(), srcPattern)
+	} else {
+		// Use fs.Glob for simple, non-recursive glob patterns
+		return fs.Glob(srcRoot.FS(), srcPattern)
+	}
+}
+
 // applyCopyOperation applies a single copy operation from the sources to the staging directory.
 // This function implements cp-like semantics by first analyzing the source pattern to determine
 // if it's a glob, direct file/directory reference, or wildcard pattern, then making copy decisions
@@ -175,11 +187,11 @@ func applyCopyOperation(ctx context.Context,
 
 	if !isGlobPattern {
 		// Direct path reference - check what it actually is first (cp-like behavior)
-		return applySingleSourceCopy(ctx, op, srcRoot, srcPattern, stagingRoot, destRelPath, destEndsWithSlash)
+		return applySingleSourceCopy(ctx, op, srcRoot, srcPattern, stagingRoot, stagingDir, destRelPath, destEndsWithSlash)
 	}
 
-	// Glob pattern - find all matches and copy each
-	matches, err := fs.Glob(srcRoot.FS(), srcPattern)
+	matches, err := getGlobMatchingEntries(op, srcRoot, srcPattern)
+
 	if err != nil {
 		return fmt.Errorf("invalid glob pattern '%s': %w", srcPattern, err)
 	}
@@ -188,12 +200,19 @@ func applyCopyOperation(ctx context.Context,
 		return fmt.Errorf("no files match pattern '%s' in source '%s'", srcPattern, srcAlias)
 	}
 
-	// Filter out excluded files
+	// Filter out excluded files and special directory entries
 	filteredMatches := make([]string, 0, len(matches))
 	for _, match := range matches {
-		if !shouldExclude(match, op.Exclude) {
-			filteredMatches = append(filteredMatches, match)
+		// Skip current directory and parent directory references
+		// doublestar.Glob returns "." for patterns like "**" which would
+		// cause the entire source to be copied, bypassing per-file strategies
+		if match == "." || match == ".." {
+			continue
+		}
+		if shouldExclude(match, op.Exclude) {
+			continue
 		}
+		filteredMatches = append(filteredMatches, match)
 	}
 
 	if len(filteredMatches) == 0 {
@@ -206,11 +225,24 @@ func applyCopyOperation(ctx context.Context,
 			return err
 		}
 
-		// Calculate destination path based on glob pattern type
-		destFile := calculateGlobDestination(srcPattern, match, destRelPath)
-		if err := copyFileWithRoots(ctx, op, srcRoot, match, stagingRoot, destFile); err != nil {
-			return fmt.Errorf("failed to copy file '%s' to '%s': %w", match, destFile, err)
+		// Handle Extract strategy for tarballs
+		if op.Strategy == swapi.ExtractStrategy {
+			if !isTarball(match) {
+				// Ignore files that are not tarball archives and directories
+				continue
+			}
+			if err := extractTarball(ctx, srcRoot, match, stagingDir, destRelPath); err != nil {
+				return fmt.Errorf("failed to extract tarball '%s' to '%s': %w", match, destRelPath, err)
+			}
+		} else {
+			// Calculate destination path based on glob pattern type
+			destFile := calculateGlobDestination(srcPattern, match, destRelPath)
+
+			if err := copyFileWithRoots(ctx, op, srcRoot, match, stagingRoot, destFile); err != nil {
+				return fmt.Errorf("failed to copy file '%s' to '%s': %w", match, destFile, err)
+			}
 		}
+
 	}
 
 	return nil
@@ -223,6 +255,7 @@ func applySingleSourceCopy(ctx context.Context,
 	srcRoot *os.Root,
 	srcPath string,
 	stagingRoot *os.Root,
+	stagingDir string,
 	destPath string,
 	destEndsWithSlash bool) error {
 	// Clean the source path to handle trailing slashes
@@ -238,10 +271,14 @@ func applySingleSourceCopy(ctx context.Context,
 	}
 
 	if srcInfo.IsDir() {
+		// Extract strategy is not supported for directories
+		if op.Strategy == swapi.ExtractStrategy {
+			return fmt.Errorf("extract strategy is not supported for directories, got '%s'", srcPath)
+		}
 		return applySingleDirectoryCopy(ctx, op, srcRoot, srcPath, stagingRoot, destPath)
-	} else {
-		return applySingleFileCopy(ctx, op, srcRoot, srcPath, stagingRoot, destPath, destEndsWithSlash)
 	}
+
+	return applySingleFileCopy(ctx, op, srcRoot, srcPath, stagingRoot, stagingDir, destPath, destEndsWithSlash)
 }
 
 // applySingleFileCopy handles copying a single file using cp-like semantics:
@@ -252,12 +289,22 @@ func applySingleFileCopy(ctx context.Context,
 	srcRoot *os.Root,
 	srcPath string,
 	stagingRoot *os.Root,
+	stagingDir string,
 	destPath string,
 	destEndsWithSlash bool) error {
 	// Check if the file should be excluded
 	if shouldExclude(srcPath, op.Exclude) {
 		return nil // Skip excluded file
 	}
+
+	// Handle Extract strategy for tarballs
+	if op.Strategy == swapi.ExtractStrategy {
+		if !isTarball(srcPath) {
+			return fmt.Errorf("extract strategy requires tarball file (.tar.gz or .tgz), got '%s'", srcPath)
+		}
+		return extractTarball(ctx, srcRoot, srcPath, stagingDir, destPath)
+	}
+
 	var finalDestPath string
 
 	if destEndsWithSlash {
@@ -303,6 +350,7 @@ func containsGlobChars(path string) bool {
 // - dir/** patterns strip the directory prefix (like cp -r dir/** dest/)
 // - other patterns preserve the full match path
 func calculateGlobDestination(pattern, match, destPath string) string {
+
 	// Check if pattern ends with /** (recursive contents pattern)
 	if strings.HasSuffix(pattern, "/**") {
 		// Extract the directory prefix from pattern (everything before /**)
@@ -545,12 +593,21 @@ func shouldExclude(filePath string, excludePatterns []string) bool {
 		return false
 	}
 
+	fileName := filepath.Base(filePath)
+
 	for _, pattern := range excludePatterns {
 		// We validate the patterns when parsing the copy operation,
 		// so it's safe to use MatchUnvalidated here.
 		if doublestar.MatchUnvalidated(pattern, filePath) {
 			return true
 		}
+		// For simple patterns without path separators (e.g., "*.md"),
+		// also match against just the filename. This provides a more
+		// intuitive user experience where "*.md" excludes all markdown
+		// files regardless of their directory depth.
+		if !strings.Contains(pattern, "/") && doublestar.MatchUnvalidated(pattern, fileName) {
+			return true
+		}
 	}
 
 	return false

internal/builder/extract.go

@@ -0,0 +1,86 @@
+/*
+Copyright 2025 The Flux authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package builder
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/fluxcd/pkg/tar"
+)
+
+// tarballExtensions defines the recognized tarball file extensions.
+// These are the formats produced by:
+//   - flux build artifact
+//   - helm package
+//
+// Currently supported: .tar.gz and .tgz (gzip-compressed tar archives)
+var tarballExtensions = []string{".tar.gz", ".tgz"}
+
+// isTarball checks if a file path has a recognized tarball extension.
+// The check is case-insensitive to handle variations like .TGZ or .Tar.Gz.
+func isTarball(path string) bool {
+	lowerPath := strings.ToLower(path)
+	for _, ext := range tarballExtensions {
+		if strings.HasSuffix(lowerPath, ext) {
+			return true
+		}
+	}
+	return false
+}
+
+// extractTarball extracts a tarball archive to the destination directory.
+// It uses fluxcd/pkg/tar.Untar for secure extraction which provides:
+//   - Automatic gzip decompression
+//   - Path traversal attack prevention
+//   - Symlink security validation
+//   - File permission preservation
+//
+// The tarball contents are extracted maintaining their internal directory structure.
+// If the destination directory doesn't exist, it will be created with 0755 permissions.
+func extractTarball(ctx context.Context,
+	srcRoot *os.Root,
+	srcPath string,
+	stagingDir string,
+	destPath string) error {
+	if err := ctx.Err(); err != nil {
+		return err
+	}
+
+	// Open the tarball through the source root for secure file access
+	srcFile, err := srcRoot.Open(srcPath)
+	if err != nil {
+		return fmt.Errorf("failed to open tarball %q: %w", srcPath, err)
+	}
+	defer srcFile.Close()
+
+	// Create the full destination path
+	fullDestPath := filepath.Join(stagingDir, destPath)
+	if err := os.MkdirAll(fullDestPath, 0o755); err != nil {
+		return fmt.Errorf("failed to create destination directory %q: %w", fullDestPath, err)
+	}
+
+	// Use fluxcd/pkg/tar.Untar for secure extraction
+	if err := tar.Untar(srcFile, fullDestPath); err != nil {
+		return fmt.Errorf("failed to extract tarball %q to %q: %w", srcPath, fullDestPath, err)
+	}
+
+	return nil
+}