Address extraction edge cases re: duplicate file names (#967)

egibs · web-flow · commit 6a433231ed93 · 2025-05-29T02:10:46.000Z
* Address extraction edge cases re: duplicate file names

Signed-off-by: egibs &lt;20933572+egibs@users.noreply.github.com&gt;
diff --git a/pkg/action/archive_test.go b/pkg/action/archive_test.go
@@ -260,7 +260,7 @@ func TestScanArchive(t *testing.T) {
 }
 
 func extractError(e error) error {
-	if strings.Contains(e.Error(), "not a valid gzip archive") || strings.Contains(e.Error(), "not a valid zip file") {
+	if strings.Contains(e.Error(), "not a valid gzip archive") || strings.Contains(e.Error(), "not a valid zip archive") {
 		return nil
 	}
 	return e
diff --git a/pkg/archive/archive.go b/pkg/archive/archive.go
@@ -166,62 +166,37 @@ func ExtractArchiveToTempDir(ctx context.Context, path string) (string, error) {
 	}
 
 	var extractedFiles sync.Map
-	files, err := os.ReadDir(tmpDir)
-	if err != nil {
-		return "", fmt.Errorf("failed to read files in directory %s: %w", tmpDir, err)
-	}
-	for _, file := range files {
-		extractedFiles.Store(filepath.Join(tmpDir, file.Name()), false)
-	}
 
-	extractedFiles.Range(func(key, _ any) bool {
-		if key == nil {
-			return true
+	err = filepath.WalkDir(tmpDir, func(path string, d os.DirEntry, err error) error {
+		if err != nil {
+			return err
 		}
-		//nolint: nestif // ignoring complexity of 11
-		if file, ok := key.(string); ok {
-			ext := programkind.GetExt(file)
-			info, err := os.Stat(file)
-			if err != nil {
-				return false
-			}
-			switch mode := info.Mode(); {
-			case mode.IsDir():
-				err = filepath.WalkDir(file, func(path string, d os.DirEntry, err error) error {
-					if err != nil {
-						return err
-					}
-					rel, err := filepath.Rel(tmpDir, path)
-					if err != nil {
-						return fmt.Errorf("filepath.Rel: %w", err)
-					}
-					if !d.IsDir() {
-						if err := extractNestedArchive(ctx, tmpDir, rel, &extractedFiles); err != nil {
-							return fmt.Errorf("failed to extract nested archive %s: %w", rel, err)
-						}
-					}
-
-					return nil
-				})
-				if err != nil {
-					return false
-				}
-				return true
-			case mode.IsRegular():
-				if _, ok := programkind.ArchiveMap[ext]; ok {
-					rel, err := filepath.Rel(tmpDir, file)
-					if err != nil {
-						return false
-					}
-					if err := extractNestedArchive(ctx, tmpDir, rel, &extractedFiles); err != nil {
-						return false
-					}
-				}
-				return true
+
+		if d.IsDir() {
+			return nil
+		}
+
+		if path == tmpDir {
+			return nil
+		}
+
+		rel, err := filepath.Rel(tmpDir, path)
+		if err != nil {
+			return fmt.Errorf("filepath.Rel: %w", err)
+		}
+
+		ext := programkind.GetExt(path)
+		if _, ok := programkind.ArchiveMap[ext]; ok {
+			if err := extractNestedArchive(ctx, tmpDir, rel, &extractedFiles); err != nil {
+				return err
 			}
 		}
-		return true
+
+		return nil
 	})
+	if err != nil {
+		return "", fmt.Errorf("failed to walk directory: %w", err)
+	}
 
 	return tmpDir, nil
 }
@@ -275,7 +250,10 @@ func handleFile(target string, tr *tar.Reader) error {
 
 	written, err := io.CopyBuffer(out, io.LimitReader(tr, maxBytes), buf)
 	if err != nil {
-		return fmt.Errorf("failed to copy file: %w", err)
+		if (strings.Contains(err.Error(), "unexpected EOF") && written == 0) ||
+			!strings.Contains(err.Error(), "unexpected EOF") {
+			return fmt.Errorf("failed to copy file: %w", err)
+		}
 	}
 	if written >= maxBytes {
 		return fmt.Errorf("file exceeds maximum allowed size (%d bytes): %s", maxBytes, target)
diff --git a/pkg/archive/bz2.go b/pkg/archive/bz2.go
@@ -45,7 +45,7 @@ func ExtractBz2(ctx context.Context, d, f string) error {
 	br := bzip2.NewReader(ctx, tf)
 	uncompressed := strings.TrimSuffix(filepath.Base(f), ".bz2")
 	uncompressed = strings.TrimSuffix(uncompressed, ".bzip2")
-	target := filepath.Join(d, uncompressed)
+	target := filepath.Join(d, filepath.Base(filepath.Dir(f)), uncompressed)
 	if !IsValidPath(target, d) {
 		return fmt.Errorf("invalid file path: %s", target)
 	}
diff --git a/pkg/archive/gzip.go b/pkg/archive/gzip.go
@@ -13,7 +13,7 @@ import (
 	gzip "github.com/klauspost/pgzip"
 )
 
-var gzMIME = map[string]struct{}{
+var GzMIME = map[string]struct{}{
 	"application/gzip":              {},
 	"application/gzip-compressed":   {},
 	"application/gzipped":           {},
@@ -32,13 +32,13 @@ func ExtractGzip(ctx context.Context, d string, f string) error {
 	// Check whether the provided file is a valid gzip archive
 	var isGzip bool
 	if ft, err := programkind.File(f); err == nil && ft != nil {
-		if _, ok := gzMIME[ft.MIME]; ok {
+		if _, ok := GzMIME[ft.MIME]; ok {
 			isGzip = true
 		}
 	}
 
 	if !isGzip {
-		return nil
+		return fmt.Errorf("not a valid gzip archive: %s", f)
 	}
 
 	logger := clog.FromContext(ctx).With("dir", d, "file", f)
diff --git a/pkg/archive/tar.go b/pkg/archive/tar.go
@@ -64,7 +64,7 @@ func ExtractTar(ctx context.Context, d string, f string) error {
 	isTGZ := strings.Contains(f, ".tar.gz") || strings.Contains(f, ".tgz")
 	var isGzip bool
 	if ft, err := programkind.File(f); err == nil && ft != nil {
-		if ft.MIME == "application/gzip" {
+		if _, ok := GzMIME[ft.MIME]; ok {
 			isGzip = true
 		}
 	}
@@ -96,13 +96,11 @@ func ExtractTar(ctx context.Context, d string, f string) error {
 			return fmt.Errorf("failed to create xz reader: %w", err)
 		}
 		uncompressed := strings.Trim(filepath.Base(f), ".xz")
-		target := filepath.Join(d, uncompressed)
+		target := filepath.Join(d, filepath.Base(filepath.Dir(f)), uncompressed)
 		if err := os.MkdirAll(filepath.Dir(target), 0o700); err != nil {
 			return fmt.Errorf("failed to create directory for file: %w", err)
 		}
 
-		// #nosec G115 // ignore Type conversion which leads to integer overflow
-		// header.Mode is int64 and FileMode is uint32
 		out, err := os.OpenFile(target, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o600)
 		if err != nil {
 			return fmt.Errorf("failed to create file: %w", err)
@@ -120,6 +118,9 @@ func ExtractTar(ctx context.Context, d string, f string) error {
 				break
 			}
 			if err != nil {
+				if strings.Contains(err.Error(), "unexpected EOF") && n > 0 {
+					break
+				}
 				return fmt.Errorf("failed to read file contents: %w", err)
 			}
 
@@ -136,7 +137,7 @@ func ExtractTar(ctx context.Context, d string, f string) error {
 	case strings.Contains(filename, ".tar.bz2") || strings.Contains(filename, ".tbz"):
 		br := bzip2.NewReader(ctx, tf)
 		uncompressed := strings.Trim(filepath.Base(f), programkind.GetExt(filename))
-		target := filepath.Join(d, uncompressed)
+		target := filepath.Join(d, filepath.Base(filepath.Dir(f)), uncompressed)
 		if err := os.MkdirAll(filepath.Dir(target), 0o700); err != nil {
 			return fmt.Errorf("failed to create directory for file: %w", err)
 		}
diff --git a/pkg/archive/zip.go b/pkg/archive/zip.go
@@ -57,7 +57,7 @@ func ExtractZip(ctx context.Context, d string, f string) error {
 	}
 
 	if !isZip {
-		return nil
+		return fmt.Errorf("not a valid zip archive: %s", f)
 	}
 
 	read, err := zip.OpenReader(f)
diff --git a/pkg/archive/zstd.go b/pkg/archive/zstd.go
@@ -34,7 +34,7 @@ func ExtractZstd(ctx context.Context, d string, f string) error {
 
 	uncompressed := strings.TrimSuffix(filepath.Base(f), ".zstd")
 	uncompressed = strings.TrimSuffix(uncompressed, ".zst")
-	target := filepath.Join(d, uncompressed)
+	target := filepath.Join(d, filepath.Base(filepath.Dir(f)), uncompressed)
 
 	if !IsValidPath(target, d) {
 		return fmt.Errorf("invalid zstd decompression file path: %s", target)
diff --git a/pkg/programkind/programkind.go b/pkg/programkind/programkind.go
@@ -46,14 +46,11 @@ var ArchiveMap = map[string]bool{
 var supportedKind = map[string]string{
 	"7z":      "application/x-7z-compressed",
 	"Z":       "application/zlib",
-	"apk":     "application/gzip",
 	"asm":     "",
 	"bash":    "application/x-bsh",
 	"bat":     "application/bat",
 	"beam":    "application/x-erlang-binary",
 	"bin":     "application/octet-stream",
-	"bz2":     "application/x-bzip2",
-	"bzip2":   "application/x-bzip2",
 	"c":       "text/x-c",
 	"cc":      "text/x-c",
 	"class":   "application/java-vm",
@@ -63,21 +60,16 @@ var supportedKind = map[string]string{
 	"crontab": "text/x-crontab",
 	"csh":     "application/x-csh",
 	"cxx":     "text/x-c",
-	"deb":     "application/vnd.debian.binary-package",
 	"dll":     "application/octet-stream",
 	"dylib":   "application/x-sharedlib",
 	"elf":     "application/x-elf",
 	"exe":     "application/octet-stream",
 	"expect":  "text/x-expect",
 	"fish":    "text/x-fish",
-	"gem":     "application/octet-stream",
 	"go":      "text/x-go",
-	"gzip":    "application/gzip",
-	"gz":      "application/gzip",
 	"h":       "text/x-h",
 	"hh":      "text/x-h",
 	"html":    "",
-	"jar":     "application/java-archive",
 	"java":    "text/x-java",
 	"js":      "application/javascript",
 	"ko":      "application/x-object",
@@ -97,29 +89,19 @@ var supportedKind = map[string]string{
 	"py":      "text/x-python",
 	"pyc":     "application/x-python-code",
 	"rb":      "text/x-ruby",
-	"rpm":     "application/x-rpm",
 	"rs":      "text/x-rust",
 	"scpt":    "application/x-applescript",
 	"scptd":   "application/x-applescript",
 	"script":  "text/x-generic-script",
 	"service": "text/x-systemd",
 	"sh":      "application/x-sh",
 	"so":      "application/x-sharedlib",
-	"tar":     "application/x-tar",
-	"tar.gz":  "application/gzip",
-	"tar.xz":  "application/x-xz",
-	"tgz":     "application/gzip",
 	"ts":      "application/typescript",
 	"upx":     "application/x-upx",
-	"whl":     "application/x-wheel+zip",
-	"xz":      "application/x-xz",
 	"yaml":    "",
 	"yara":    "",
 	"yml":     "",
-	"zip":     "application/zip",
 	"zsh":     "application/x-zsh",
-	"zst":     "application/zstd",
-	"zstd":    "application/zstd",
 }
 
 type FileType struct {
@@ -213,6 +195,11 @@ func IsValidUPX(header []byte, path string) (bool, error) {
 func makeFileType(path string, ext string, mime string) *FileType {
 	ext = strings.TrimPrefix(ext, ".")
 
+	// Archives are supported
+	if _, ok := ArchiveMap[GetExt(path)]; ok {
+		return &FileType{Ext: ext, MIME: mime}
+	}
+
 	// the only JSON files we currently scan are NPM package metadata, which ends in *package.json
 	if strings.HasSuffix(path, "package.json") {
 		return &FileType{
@@ -339,8 +326,7 @@ func File(path string) (*FileType, error) {
 
 // Path returns a filetype based strictly on file path.
 func Path(path string) *FileType {
-	// Trim the leading '.'
-	ext := strings.TrimPrefix(GetExt(path), ".")
+	ext := strings.ReplaceAll(filepath.Ext(path), ".", "")
 	mime := supportedKind[ext]
 	return makeFileType(path, ext, mime)
 }
diff --git a/tests/linux/clean/minio_x86_64.md b/tests/linux/clean/minio_x86_64.md
diff --git a/tests/linux/clean/neuvector_agent_aarch64.md b/tests/linux/clean/neuvector_agent_aarch64.md
diff --git a/tests/linux/clean/vitess/vtadmin.simple b/tests/linux/clean/vitess/vtadmin.simple
diff --git a/tests/linux/clean/vitess/vtclient.simple b/tests/linux/clean/vitess/vtclient.simple

Original file line number	Diff line number	Diff line change
`@@ -260,7 +260,7 @@ func TestScanArchive(t *testing.T) {`
`260`	`260`	`}`
`261`	`261`
`262`	`262`	`func extractError(e error) error {`
`263`		`- if strings.Contains(e.Error(), "not a valid gzip archive") \|\| strings.Contains(e.Error(), "not a valid zip file") {`
	`263`	`+ if strings.Contains(e.Error(), "not a valid gzip archive") \|\| strings.Contains(e.Error(), "not a valid zip archive") {`
`264`	`264`	`return nil`
`265`	`265`	`}`
`266`	`266`	`return e`
Original file line number	Diff line number	Diff line change
`@@ -45,7 +45,7 @@ func ExtractBz2(ctx context.Context, d, f string) error {`
`45`	`45`	`br := bzip2.NewReader(ctx, tf)`
`46`	`46`	`uncompressed := strings.TrimSuffix(filepath.Base(f), ".bz2")`
`47`	`47`	`uncompressed = strings.TrimSuffix(uncompressed, ".bzip2")`
`48`		`- target := filepath.Join(d, uncompressed)`
	`48`	`+ target := filepath.Join(d, filepath.Base(filepath.Dir(f)), uncompressed)`
`49`	`49`	`if !IsValidPath(target, d) {`
`50`	`50`	`return fmt.Errorf("invalid file path: %s", target)`
`51`	`51`	`}`
Original file line number	Diff line number	Diff line change
`@@ -13,7 +13,7 @@ import (`
`13`	`13`	`gzip "github.com/klauspost/pgzip"`
`14`	`14`	`)`
`15`	`15`
`16`		`-var gzMIME = map[string]struct{}{`
	`16`	`+var GzMIME = map[string]struct{}{`
`17`	`17`	`"application/gzip": {},`
`18`	`18`	`"application/gzip-compressed": {},`
`19`	`19`	`"application/gzipped": {},`
`@@ -32,13 +32,13 @@ func ExtractGzip(ctx context.Context, d string, f string) error {`
`32`	`32`	`// Check whether the provided file is a valid gzip archive`
`33`	`33`	`var isGzip bool`
`34`	`34`	`if ft, err := programkind.File(f); err == nil && ft != nil {`
`35`		`- if _, ok := gzMIME[ft.MIME]; ok {`
	`35`	`+ if _, ok := GzMIME[ft.MIME]; ok {`
`36`	`36`	`isGzip = true`
`37`	`37`	`}`
`38`	`38`	`}`
`39`	`39`
`40`	`40`	`if !isGzip {`
`41`		`- return nil`
	`41`	`+ return fmt.Errorf("not a valid gzip archive: %s", f)`
`42`	`42`	`}`
`43`	`43`
`44`	`44`	`logger := clog.FromContext(ctx).With("dir", d, "file", f)`
Original file line number	Diff line number	Diff line change
`@@ -64,7 +64,7 @@ func ExtractTar(ctx context.Context, d string, f string) error {`
`64`	`64`	`isTGZ := strings.Contains(f, ".tar.gz") \|\| strings.Contains(f, ".tgz")`
`65`	`65`	`var isGzip bool`
`66`	`66`	`if ft, err := programkind.File(f); err == nil && ft != nil {`
`67`		`- if ft.MIME == "application/gzip" {`
	`67`	`+ if _, ok := GzMIME[ft.MIME]; ok {`
`68`	`68`	`isGzip = true`
`69`	`69`	`}`
`70`	`70`	`}`
`@@ -96,13 +96,11 @@ func ExtractTar(ctx context.Context, d string, f string) error {`
`96`	`96`	`return fmt.Errorf("failed to create xz reader: %w", err)`
`97`	`97`	`}`
`98`	`98`	`uncompressed := strings.Trim(filepath.Base(f), ".xz")`
`99`		`- target := filepath.Join(d, uncompressed)`
	`99`	`+ target := filepath.Join(d, filepath.Base(filepath.Dir(f)), uncompressed)`
`100`	`100`	`if err := os.MkdirAll(filepath.Dir(target), 0o700); err != nil {`
`101`	`101`	`return fmt.Errorf("failed to create directory for file: %w", err)`
`102`	`102`	`}`
`103`	`103`
`104`		`- // #nosec G115 // ignore Type conversion which leads to integer overflow`
`105`		`- // header.Mode is int64 and FileMode is uint32`
`106`	`104`	`out, err := os.OpenFile(target, os.O_WRONLY\|os.O_CREATE\|os.O_TRUNC, 0o600)`
`107`	`105`	`if err != nil {`
`108`	`106`	`return fmt.Errorf("failed to create file: %w", err)`
`@@ -120,6 +118,9 @@ func ExtractTar(ctx context.Context, d string, f string) error {`
`120`	`118`	`break`
`121`	`119`	`}`
`122`	`120`	`if err != nil {`
	`121`	`+ if strings.Contains(err.Error(), "unexpected EOF") && n > 0 {`
	`122`	`+ break`
	`123`	`+ }`
`123`	`124`	`return fmt.Errorf("failed to read file contents: %w", err)`
`124`	`125`	`}`
`125`	`126`
`@@ -136,7 +137,7 @@ func ExtractTar(ctx context.Context, d string, f string) error {`
`136`	`137`	`case strings.Contains(filename, ".tar.bz2") \|\| strings.Contains(filename, ".tbz"):`
`137`	`138`	`br := bzip2.NewReader(ctx, tf)`
`138`	`139`	`uncompressed := strings.Trim(filepath.Base(f), programkind.GetExt(filename))`
`139`		`- target := filepath.Join(d, uncompressed)`
	`140`	`+ target := filepath.Join(d, filepath.Base(filepath.Dir(f)), uncompressed)`
`140`	`141`	`if err := os.MkdirAll(filepath.Dir(target), 0o700); err != nil {`
`141`	`142`	`return fmt.Errorf("failed to create directory for file: %w", err)`
`142`	`143`	`}`
Original file line number	Diff line number	Diff line change
`@@ -57,7 +57,7 @@ func ExtractZip(ctx context.Context, d string, f string) error {`
`57`	`57`	`}`
`58`	`58`
`59`	`59`	`if !isZip {`
`60`		`- return nil`
	`60`	`+ return fmt.Errorf("not a valid zip archive: %s", f)`
`61`	`61`	`}`
`62`	`62`
`63`	`63`	`read, err := zip.OpenReader(f)`