Merge pull request #302 from chris-morandi/copy-operations-tarball-301

Extend copy operations with tarball extraction capabilities

Author: matheuscscp

api/v1beta1/artifactgenerator_types.go

@@ -37,6 +37,7 @@ const (
 	SourceFetchFailedReason          = "SourceFetchFailed"
 	OverwriteStrategy                = "Overwrite"
 	MergeStrategy                    = "Merge"
+	ExtractStrategy                  = "Extract"
 	EnabledValue                     = "enabled"
 	DisabledValue                    = "disabled"
 )
@@ -149,9 +150,12 @@ type CopyOperation struct {
 	// Strategy specifies the copy strategy to use.
 	// 'Overwrite' will overwrite existing files in the destination.
 	// 'Merge' is for merging YAML files using Helm values merge strategy.
+	// 'Extract' is for extracting the contents of tarball archives (.tar.gz, .tgz)
+	// When using glob patterns, non-tarball files are silently skipped. For single file sources,
+	// the file must be a tarball or an error is returned. Directories are not supported.
 	// If not specified, defaults to 'Overwrite'.
 	// +optional
-	// +kubebuilder:validation:Enum=Overwrite;Merge
+	// +kubebuilder:validation:Enum=Overwrite;Merge;Extract
 	Strategy string `json:"strategy,omitempty"`
 }
 

config/crd/bases/source.extensions.fluxcd.io_artifactgenerators.yaml

@@ -85,10 +85,14 @@ spec:
                               Strategy specifies the copy strategy to use.
                               'Overwrite' will overwrite existing files in the destination.
                               'Merge' is for merging YAML files using Helm values merge strategy.
+                              'Extract' is for extracting the contents of tarball archives (.tar.gz, .tgz)
+                              When using glob patterns, non-tarball files are silently skipped. For single file sources,
+                              the file must be a tarball or an error is returned. Directories are not supported.
                               If not specified, defaults to 'Overwrite'.
                             enum:
                             - Overwrite
                             - Merge
+                            - Extract
                             type: string
                           to:
                             description: |-

docs/spec/v1beta1/artifactgenerators.md

@@ -268,8 +268,8 @@ Each copy operation specifies how to copy files from sources into the generated
   the root of the generated artifact and `path` is the relative path to a file or directory.
 - `exclude` (optional): A list of glob patterns to filter out from the source selection.
   Any file matched by `from` that also matches an exclude pattern will be ignored.
-- `strategy` (optional): Defines how to handle existing files at the destination, 
-  either `Overwrite` (default) or `Merge` (for YAML files only).
+- `strategy` (optional): Defines how to handle files during copy operations:
+  `Overwrite` (default), `Merge` (for YAML files), or `Extract` (for tarball archives).
 
 Copy operations use `cp`-like semantics:
 
@@ -327,6 +327,35 @@ Example of copy with `Merge` strategy:
 **Note** that the merge strategy will replace _arrays_ entirely, the behavior is
 identical to how Helm merges `values.yaml` files when using multiple `--values` flags.
 
+##### Extract Strategy
+
+The `Extract` strategy is used for extracting the contents of tarball archives (`.tar.gz`, `.tgz`)
+built with `flux build artifact` or `helm package`. The tarball contents are extracted
+to the destination while preserving their internal directory structure.
+
+Example of copy with `Extract` strategy:
+
+```yaml
+# Extract a Helm chart tarball built with `helm package`
+- from: "@oci/podinfo-6.7.0.tgz"
+  to: "@artifact/"
+  strategy: Extract
+
+# Extract multiple tarballs using glob patterns
+- from: "@source/charts/*.tgz"
+  to: "@artifact/charts/"
+  strategy: Extract
+
+# Extract tarballs recursively from nested directories
+- from: "@source/releases/**/*.tgz"
+  to: "@artifact/"
+  strategy: Extract
+```
+
+**Note** that when using glob patterns (including recursive `**` patterns) with the `Extract`
+strategy, non-tarball files are silently skipped. For single file sources, the file must have
+a `.tar.gz` or `.tgz` extension. Directories are not supported with this strategy.
+
 ## Working with ArtifactGenerators
 
 ### Suspend and Resume Reconciliation

internal/builder/builder.go

@@ -127,6 +127,18 @@ func applyCopyOperations(ctx context.Context,
 	return nil
 }
 
+// If the copy operation uses the Extract strategy, it uses doublestar.Glob as we do not need to walk the whole tree
+// otherwise we us std fs.Glob
+func getGlobMatchingEntries(op swapi.CopyOperation, srcRoot *os.Root, srcPattern string) ([]string, error) {
+	if op.Strategy == swapi.ExtractStrategy {
+		// Use doublestar.Glob for recursive and advanced glob patterns (e.g., **/*.tar.gz)
+		return doublestar.Glob(srcRoot.FS(), srcPattern)
+	} else {
+		// Use fs.Glob for simple, non-recursive glob patterns
+		return fs.Glob(srcRoot.FS(), srcPattern)
+	}
+}
+
 // applyCopyOperation applies a single copy operation from the sources to the staging directory.
 // This function implements cp-like semantics by first analyzing the source pattern to determine
 // if it's a glob, direct file/directory reference, or wildcard pattern, then making copy decisions
@@ -175,11 +187,11 @@ func applyCopyOperation(ctx context.Context,
 
 	if !isGlobPattern {
 		// Direct path reference - check what it actually is first (cp-like behavior)
-		return applySingleSourceCopy(ctx, op, srcRoot, srcPattern, stagingRoot, destRelPath, destEndsWithSlash)
+		return applySingleSourceCopy(ctx, op, srcRoot, srcPattern, stagingRoot, stagingDir, destRelPath, destEndsWithSlash)
 	}
 
-	// Glob pattern - find all matches and copy each
-	matches, err := fs.Glob(srcRoot.FS(), srcPattern)
+	matches, err := getGlobMatchingEntries(op, srcRoot, srcPattern)
+
 	if err != nil {
 		return fmt.Errorf("invalid glob pattern '%s': %w", srcPattern, err)
 	}
@@ -188,12 +200,19 @@ func applyCopyOperation(ctx context.Context,
 		return fmt.Errorf("no files match pattern '%s' in source '%s'", srcPattern, srcAlias)
 	}
 
-	// Filter out excluded files
+	// Filter out excluded files and special directory entries
 	filteredMatches := make([]string, 0, len(matches))
 	for _, match := range matches {
-		if !shouldExclude(match, op.Exclude) {
-			filteredMatches = append(filteredMatches, match)
+		// Skip current directory and parent directory references
+		// doublestar.Glob returns "." for patterns like "**" which would
+		// cause the entire source to be copied, bypassing per-file strategies
+		if match == "." || match == ".." {
+			continue
+		}
+		if shouldExclude(match, op.Exclude) {
+			continue
 		}
+		filteredMatches = append(filteredMatches, match)
 	}
 
 	if len(filteredMatches) == 0 {
@@ -206,10 +225,22 @@ func applyCopyOperation(ctx context.Context,
 			return err
 		}
 
-		// Calculate destination path based on glob pattern type
-		destFile := calculateGlobDestination(srcPattern, match, destRelPath)
-		if err := copyFileWithRoots(ctx, op, srcRoot, match, stagingRoot, destFile); err != nil {
-			return fmt.Errorf("failed to copy file '%s' to '%s': %w", match, destFile, err)
+		// Handle Extract strategy for tarballs
+		if op.Strategy == swapi.ExtractStrategy {
+			if !isTarball(match) {
+				// Ignore files that are not tarball archives and directories
+				continue
+			}
+			if err := extractTarball(ctx, srcRoot, match, stagingDir, destRelPath); err != nil {
+				return fmt.Errorf("failed to extract tarball '%s' to '%s': %w", match, destRelPath, err)
+			}
+		} else {
+			// Calculate destination path based on glob pattern type
+			destFile := calculateGlobDestination(srcPattern, match, destRelPath)
+
+			if err := copyFileWithRoots(ctx, op, srcRoot, match, stagingRoot, destFile); err != nil {
+				return fmt.Errorf("failed to copy file '%s' to '%s': %w", match, destFile, err)
+			}
 		}
 	}
 
@@ -223,6 +254,7 @@ func applySingleSourceCopy(ctx context.Context,
 	srcRoot *os.Root,
 	srcPath string,
 	stagingRoot *os.Root,
+	stagingDir string,
 	destPath string,
 	destEndsWithSlash bool) error {
 	// Clean the source path to handle trailing slashes
@@ -238,10 +270,14 @@ func applySingleSourceCopy(ctx context.Context,
 	}
 
 	if srcInfo.IsDir() {
+		// Extract strategy is not supported for directories
+		if op.Strategy == swapi.ExtractStrategy {
+			return fmt.Errorf("extract strategy is not supported for directories, got '%s'", srcPath)
+		}
 		return applySingleDirectoryCopy(ctx, op, srcRoot, srcPath, stagingRoot, destPath)
-	} else {
-		return applySingleFileCopy(ctx, op, srcRoot, srcPath, stagingRoot, destPath, destEndsWithSlash)
 	}
+
+	return applySingleFileCopy(ctx, op, srcRoot, srcPath, stagingRoot, stagingDir, destPath, destEndsWithSlash)
 }
 
 // applySingleFileCopy handles copying a single file using cp-like semantics:
@@ -252,12 +288,22 @@ func applySingleFileCopy(ctx context.Context,
 	srcRoot *os.Root,
 	srcPath string,
 	stagingRoot *os.Root,
+	stagingDir string,
 	destPath string,
 	destEndsWithSlash bool) error {
 	// Check if the file should be excluded
 	if shouldExclude(srcPath, op.Exclude) {
 		return nil // Skip excluded file
 	}
+
+	// Handle Extract strategy for tarballs
+	if op.Strategy == swapi.ExtractStrategy {
+		if !isTarball(srcPath) {
+			return fmt.Errorf("extract strategy requires tarball file (.tar.gz or .tgz), got '%s'", srcPath)
+		}
+		return extractTarball(ctx, srcRoot, srcPath, stagingDir, destPath)
+	}
+
 	var finalDestPath string
 
 	if destEndsWithSlash {
@@ -303,6 +349,7 @@ func containsGlobChars(path string) bool {
 // - dir/** patterns strip the directory prefix (like cp -r dir/** dest/)
 // - other patterns preserve the full match path
 func calculateGlobDestination(pattern, match, destPath string) string {
+
 	// Check if pattern ends with /** (recursive contents pattern)
 	if strings.HasSuffix(pattern, "/**") {
 		// Extract the directory prefix from pattern (everything before /**)
@@ -545,12 +592,21 @@ func shouldExclude(filePath string, excludePatterns []string) bool {
 		return false
 	}
 
+	fileName := filepath.Base(filePath)
+
 	for _, pattern := range excludePatterns {
 		// We validate the patterns when parsing the copy operation,
 		// so it's safe to use MatchUnvalidated here.
 		if doublestar.MatchUnvalidated(pattern, filePath) {
 			return true
 		}
+		// For simple patterns without path separators (e.g., "*.md"),
+		// also match against just the filename. This provides a more
+		// intuitive user experience where "*.md" excludes all markdown
+		// files regardless of their directory depth.
+		if !strings.Contains(pattern, "/") && doublestar.MatchUnvalidated(pattern, fileName) {
+			return true
+		}
 	}
 
 	return false

internal/builder/extract.go

@@ -0,0 +1,86 @@
+/*
+Copyright 2025 The Flux authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package builder
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/fluxcd/pkg/tar"
+)
+
+// tarballExtensions defines the recognized tarball file extensions.
+// These are the formats produced by:
+//   - flux build artifact
+//   - helm package
+//
+// Currently supported: .tar.gz and .tgz (gzip-compressed tar archives)
+var tarballExtensions = []string{".tar.gz", ".tgz"}
+
+// isTarball checks if a file path has a recognized tarball extension.
+// The check is case-insensitive to handle variations like .TGZ or .Tar.Gz.
+func isTarball(path string) bool {
+	lowerPath := strings.ToLower(path)
+	for _, ext := range tarballExtensions {
+		if strings.HasSuffix(lowerPath, ext) {
+			return true
+		}
+	}
+	return false
+}
+
+// extractTarball extracts a tarball archive to the destination directory.
+// It uses fluxcd/pkg/tar.Untar for secure extraction which provides:
+//   - Automatic gzip decompression
+//   - Path traversal attack prevention
+//   - Symlink security validation
+//   - File permission preservation
+//
+// The tarball contents are extracted maintaining their internal directory structure.
+// If the destination directory doesn't exist, it will be created with 0755 permissions.
+func extractTarball(ctx context.Context,
+	srcRoot *os.Root,
+	srcPath string,
+	stagingDir string,
+	destPath string) error {
+	if err := ctx.Err(); err != nil {
+		return err
+	}
+
+	// Open the tarball through the source root for secure file access
+	srcFile, err := srcRoot.Open(srcPath)
+	if err != nil {
+		return fmt.Errorf("failed to open tarball %q: %w", srcPath, err)
+	}
+	defer srcFile.Close()
+
+	// Create the full destination path
+	fullDestPath := filepath.Join(stagingDir, destPath)
+	if err := os.MkdirAll(fullDestPath, 0o755); err != nil {
+		return fmt.Errorf("failed to create destination directory %q: %w", fullDestPath, err)
+	}
+
+	// Use fluxcd/pkg/tar.Untar for secure extraction
+	if err := tar.Untar(srcFile, fullDestPath); err != nil {
+		return fmt.Errorf("failed to extract tarball %q to %q: %w", srcPath, fullDestPath, err)
+	}
+
+	return nil
+}