Avoid failing scans outright when encountering extraction failures

cmd/mal/mal.go

@@ -50,6 +50,7 @@ var (
 	allFlag                   bool
 	concurrencyFlag           int
 	diffImageFlag             bool
+	exitExtractionFlag        bool
 	exitFirstHitFlag          bool
 	exitFirstMissFlag         bool
 	fileRiskChangeFlag        bool
@@ -262,6 +263,7 @@ func main() {
 
 			mc = malcontent.Config{
 				Concurrency:           concurrency,
+				ExitExtraction:        exitExtractionFlag,
 				ExitFirstHit:          exitFirstHitFlag,
 				ExitFirstMiss:         exitFirstMissFlag,
 				IgnoreSelf:            ignoreSelfFlag,
@@ -287,6 +289,12 @@ func main() {
 				Usage:       "Ignore nothing within a provided scan path",
 				Destination: &allFlag,
 			},
+			&cli.BoolFlag{
+				Name:        "exit-extraction",
+				Value:       true,
+				Usage:       "Exit when encountering file extraction errors",
+				Destination: &exitExtractionFlag,
+			},
 			&cli.BoolFlag{
 				Name:        "exit-first-miss",
 				Value:       false,

pkg/action/archive_test.go

@@ -11,7 +11,6 @@ import (
 	"testing"
 
 	"github.com/chainguard-dev/clog"
-	"github.com/chainguard-dev/clog/slogtest"
 	"github.com/chainguard-dev/malcontent/pkg/archive"
 	"github.com/chainguard-dev/malcontent/pkg/malcontent"
 	"github.com/chainguard-dev/malcontent/pkg/programkind"
@@ -215,7 +214,7 @@ func TestExtractNestedArchive(t *testing.T) {
 
 func TestScanArchive(t *testing.T) {
 	t.Parallel()
-	ctx := slogtest.Context(t)
+	ctx := context.Background()
 	clog.FromContext(ctx).With("test", "scan_archive")
 
 	var out bytes.Buffer
@@ -260,6 +259,95 @@ func TestScanArchive(t *testing.T) {
 	}
 }
 
+func extractError(e error) error {
+	if strings.Contains(e.Error(), "not a valid gzip archive") || strings.Contains(e.Error(), "not a valid zip file") {
+		return nil
+	}
+	return e
+}
+
+func TestScanInvalidArchive(t *testing.T) {
+	t.Parallel()
+	ctx := context.Background()
+	clog.FromContext(ctx).With("test", "scan_invalid_archive")
+
+	var out bytes.Buffer
+	r, err := render.New("json", &out)
+	if err != nil {
+		t.Fatalf("render: %v", err)
+	}
+
+	rfs := []fs.FS{rules.FS, thirdparty.FS}
+	yrs, err := CachedRules(ctx, rfs)
+	if err != nil {
+		t.Fatalf("rules: %v", err)
+	}
+
+	mc := malcontent.Config{
+		Concurrency:    runtime.NumCPU(),
+		ExitExtraction: true,
+		IgnoreSelf:     false,
+		MinFileRisk:    0,
+		MinRisk:        0,
+		Renderer:       r,
+		Rules:          yrs,
+		ScanPaths: []string{
+			"testdata/17419.zip",
+			"testdata/joblib_0.9.4.dev0_compressed_cache_size_pickle_py35_np19.gz",
+		},
+	}
+	_, err = Scan(ctx, mc)
+	err = extractError(err)
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestScanInvalidArchiveIgnore(t *testing.T) {
+	t.Parallel()
+	ctx := context.Background()
+	clog.FromContext(ctx).With("test", "scan_invalid_archive_ignore")
+
+	var out bytes.Buffer
+	r, err := render.New("json", &out)
+	if err != nil {
+		t.Fatalf("render: %v", err)
+	}
+
+	rfs := []fs.FS{rules.FS, thirdparty.FS}
+	yrs, err := CachedRules(ctx, rfs)
+	if err != nil {
+		t.Fatalf("rules: %v", err)
+	}
+
+	mc := malcontent.Config{
+		Concurrency:    runtime.NumCPU(),
+		ExitExtraction: false,
+		IgnoreSelf:     false,
+		MinFileRisk:    0,
+		MinRisk:        0,
+		Renderer:       r,
+		Rules:          yrs,
+		ScanPaths: []string{
+			"testdata/17419.zip",
+			"testdata/joblib_0.9.4.dev0_compressed_cache_size_pickle_py35_np19.gz",
+		},
+	}
+	res, err := Scan(ctx, mc)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := r.Full(ctx, nil, res); err != nil {
+		t.Fatalf("full: %v", err)
+	}
+
+	got := out.String()
+	want := "{}\n"
+	if diff := cmp.Diff(want, got); diff != "" {
+		t.Errorf("output mismatch: (-want +got):\n%s", diff)
+	}
+}
+
 func TestGetExt(t *testing.T) {
 	tests := []struct {
 		path string

pkg/action/oci_test.go

@@ -2,13 +2,13 @@ package action
 
 import (
 	"bytes"
+	"context"
 	"io/fs"
 	"os"
 	"runtime"
 	"testing"
 
 	"github.com/chainguard-dev/clog"
-	"github.com/chainguard-dev/clog/slogtest"
 	"github.com/chainguard-dev/malcontent/pkg/malcontent"
 	"github.com/chainguard-dev/malcontent/pkg/render"
 	"github.com/chainguard-dev/malcontent/rules"
@@ -18,7 +18,7 @@ import (
 
 func TestOCI(t *testing.T) {
 	t.Parallel()
-	ctx := slogtest.Context(t)
+	ctx := context.Background()
 	clog.FromContext(ctx).With("test", "scan_oci")
 
 	var out bytes.Buffer

pkg/action/scan.go

@@ -630,6 +630,11 @@ func processArchive(ctx context.Context, c malcontent.Config, rfs []fs.FS, archi
 
 	tmpRoot, err := archive.ExtractArchiveToTempDir(ctx, archivePath)
 	if err != nil {
+		// Avoid failing an entire scan when encountering problematic archives
+		// e.g., joblib_0.8.4_compressed_pickle_py27_np17.gz: not a valid gzip archive
+		if !c.ExitExtraction {
+			return nil, nil
+		}
 		return nil, fmt.Errorf("extract to temp: %w", err)
 	}
 	// Ensure that tmpRoot is removed before returning if created successfully
@@ -640,8 +645,8 @@ func processArchive(ctx context.Context, c malcontent.Config, rfs []fs.FS, archi
 	}()
 
 	// macOS will prefix temporary directories with `/private`
-	// update tmpRoot with this prefix to allow strings.TrimPrefix to work
-	if runtime.GOOS == "darwin" {
+	// update tmpRoot (if populated) with this prefix to allow strings.TrimPrefix to work
+	if runtime.GOOS == "darwin" && tmpRoot != "" {
 		tmpRoot = fmt.Sprintf("/private%s", tmpRoot)
 	}
 

pkg/action/testdata/17419.zip

@@ -0,0 +1,26 @@
+Advisory :
+
+
+Abysssec Public Exploit :
+
+This module exploits a code execution vulnerability in Mozilla
+Firefox <= 3.6.16 caused by nsTreeSelection element. The specific flaw
+exists within the way Firefox handles user defined functions of
+a nsTreeSelection element. When executing the function
+invalidateSelection it is possible to free the nsTreeSelection object
+that the function operates on. Any further operations on the freed
+object can result in remote code execution.this exploit module is only
+tested on win7 and used a Another JAVA ROPto defeat DEP/ASLR (due to
+there is no more non-aslr module in Firefox) and in my tests works
+reliably on Windows7.
+
+there is two version of this exploit XP and 7 and both use different
+method that used in MSF Exploit bounty !
+
+XP   Version: https://gitlab.com/exploit-database/exploitdb-bin-sploits/-/raw/main/bin-sploits/17419-1.zip (nsTreeRange_XP.zip)
+Win7 Version: https://gitlab.com/exploit-database/exploitdb-bin-sploits/-/raw/main/bin-sploits/17419-2.zip (nsTreeRange_7.zip)
+
+
+
+
+questions / comments : Info [at] abysssec.com

pkg/archive/archive.go

@@ -60,6 +60,11 @@ func extractNestedArchive(ctx context.Context, d string, f string, extracted *sy
 		return fmt.Errorf("failed to determine file type: %w", err)
 	}
 
+	// Empty filetypes or invalid MIME types should not be evaluated via `extract`
+	if ft == nil || (ft != nil && ft.MIME == "") {
+		return nil
+	}
+
 	switch {
 	case ft != nil && ft.MIME == "application/x-upx":
 		isArchive = true
@@ -148,6 +153,11 @@ func ExtractArchiveToTempDir(ctx context.Context, path string) (string, error) {
 		return "", fmt.Errorf("failed to determine file type: %w", err)
 	}
 
+	// Empty filetypes or invalid MIME types should not be evaluated via `extract`
+	if ft == nil || (ft != nil && ft.MIME == "") {
+		return "", fmt.Errorf("unsupported archive type: %s", path)
+	}
+
 	switch {
 	case ft != nil && ft.MIME == "application/zlib":
 		extract = ExtractZlib

pkg/archive/gzip.go

@@ -13,6 +13,16 @@ import (
 	gzip "github.com/klauspost/pgzip"
 )
 
+var gzMIME = map[string]struct{}{
+	"application/gzip":              {},
+	"application/gzip-compressed":   {},
+	"application/gzipped":           {},
+	"application/x-gunzip":          {},
+	"application/x-gzip":            {},
+	"application/x-gzip-compressed": {},
+	"gzip/document":                 {},
+}
+
 // extractGzip extracts .gz archives.
 func ExtractGzip(ctx context.Context, d string, f string) error {
 	if ctx.Err() != nil {
@@ -22,13 +32,13 @@ func ExtractGzip(ctx context.Context, d string, f string) error {
 	// Check whether the provided file is a valid gzip archive
 	var isGzip bool
 	if ft, err := programkind.File(f); err == nil && ft != nil {
-		if ft.MIME == "application/gzip" {
+		if _, ok := gzMIME[ft.MIME]; ok {
 			isGzip = true
 		}
 	}
 
 	if !isGzip {
-		return fmt.Errorf("not a valid gzip archive")
+		return nil
 	}
 
 	logger := clog.FromContext(ctx).With("dir", d, "file", f)

pkg/archive/zip.go

@@ -12,12 +12,21 @@ import (
 
 	"github.com/chainguard-dev/clog"
 	"github.com/chainguard-dev/malcontent/pkg/pool"
+	"github.com/chainguard-dev/malcontent/pkg/programkind"
 	zip "github.com/klauspost/compress/zip"
 	"golang.org/x/sync/errgroup"
 )
 
 var initZipPool sync.Once
 
+var zipMIME = map[string]struct{}{
+	"application/java-archive":     {},
+	"application/x-wheel+zip":      {},
+	"application/x-zip":            {},
+	"application/x-zip-compressed": {},
+	"application/zip":              {},
+}
+
 // ExtractZip extracts .jar and .zip archives.
 func ExtractZip(ctx context.Context, d string, f string) error {
 	if ctx.Err() != nil {
@@ -39,6 +48,17 @@ func ExtractZip(ctx context.Context, d string, f string) error {
 		return nil
 	}
 
+	var isZip bool
+	if ft, err := programkind.File(f); err == nil && ft != nil {
+		if _, ok := zipMIME[ft.MIME]; ok {
+			isZip = true
+		}
+	}
+
+	if !isZip {
+		return nil
+	}
+
 	read, err := zip.OpenReader(f)
 	if err != nil {
 		return fmt.Errorf("failed to open zip file %s: %w", f, err)

pkg/malcontent/malcontent.go

@@ -23,6 +23,7 @@ type Renderer interface {
 
 type Config struct {
 	Concurrency           int
+	ExitExtraction        bool
 	ExitFirstHit          bool
 	ExitFirstMiss         bool
 	FileRiskChange        bool

pkg/programkind/programkind.go

@@ -44,13 +44,16 @@ var ArchiveMap = map[string]bool{
 
 // file extension to MIME type, if it's a good scanning target.
 var supportedKind = map[string]string{
-	"7z":      "",
+	"7z":      "application/x-7z-compressed",
 	"Z":       "application/zlib",
+	"apk":     "application/gzip",
 	"asm":     "",
 	"bash":    "application/x-bsh",
 	"bat":     "application/bat",
 	"beam":    "application/x-erlang-binary",
 	"bin":     "application/octet-stream",
+	"bz2":     "application/x-bzip2",
+	"bzip2":   "application/x-bzip2",
 	"c":       "text/x-c",
 	"cc":      "text/x-c",
 	"class":   "application/java-vm",
@@ -60,14 +63,17 @@ var supportedKind = map[string]string{
 	"crontab": "text/x-crontab",
 	"csh":     "application/x-csh",
 	"cxx":     "text/x-c",
+	"deb":     "application/vnd.debian.binary-package",
 	"dll":     "application/octet-stream",
 	"dylib":   "application/x-sharedlib",
 	"elf":     "application/x-elf",
 	"exe":     "application/octet-stream",
 	"expect":  "text/x-expect",
 	"fish":    "text/x-fish",
+	"gem":     "application/octet-stream",
 	"go":      "text/x-go",
 	"gzip":    "application/gzip",
+	"gz":      "application/gzip",
 	"h":       "text/x-h",
 	"hh":      "text/x-h",
 	"html":    "",
@@ -91,20 +97,29 @@ var supportedKind = map[string]string{
 	"py":      "text/x-python",
 	"pyc":     "application/x-python-code",
 	"rb":      "text/x-ruby",
+	"rpm":     "application/x-rpm",
 	"rs":      "text/x-rust",
 	"scpt":    "application/x-applescript",
 	"scptd":   "application/x-applescript",
 	"script":  "text/x-generic-script",
 	"service": "text/x-systemd",
 	"sh":      "application/x-sh",
 	"so":      "application/x-sharedlib",
+	"tar":     "application/x-tar",
+	"tar.gz":  "application/gzip",
+	"tar.xz":  "application/x-xz",
+	"tgz":     "application/gzip",
 	"ts":      "application/typescript",
 	"upx":     "application/x-upx",
 	"whl":     "application/x-wheel+zip",
+	"xz":      "application/x-xz",
 	"yaml":    "",
 	"yara":    "",
 	"yml":     "",
+	"zip":     "application/zip",
 	"zsh":     "application/x-zsh",
+	"zst":     "application/zstd",
+	"zstd":    "application/zstd",
 }
 
 type FileType struct {
@@ -324,7 +339,8 @@ func File(path string) (*FileType, error) {
 
 // Path returns a filetype based strictly on file path.
 func Path(path string) *FileType {
-	ext := strings.ReplaceAll(filepath.Ext(path), ".", "")
+	// Trim the leading '.'
+	ext := strings.TrimPrefix(GetExt(path), ".")
 	mime := supportedKind[ext]
 	return makeFileType(path, ext, mime)
 }