Corrected the triage connection - Added api auth to MalwareBazaar - Additional checks added for HybridAnalysis, VsShare, and VirusExchange - Updated test cases

Author: xorhex

README.md

@@ -17,7 +17,7 @@ Use mlget to query multiple sources for a given malware hash and download it.  T
 
 MIT License
 
-Copyright (c) 2024 @xorhex
+Copyright (c) 2025 @xorhex
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal
@@ -36,3 +36,4 @@ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 SOFTWARE.
+

config.go

@@ -64,7 +64,7 @@ func (malrepo MalwareRepoType) QueryAndDownload(repos []RepositoryConfigEntry, h
 		fmt.Printf("  [*] %s: %s\n", mcr.Type, mcr.Host)
 		switch malrepo {
 		case MalwareBazaar:
-			found, filename = malwareBazaar(mcr.Host, hash, doNotExtract, "infected")
+			found, filename = malwareBazaar(mcr.Host, mcr.Api, hash, doNotExtract, "infected")
 			checkedRepo = MalwareBazaar
 		case MWDB:
 			found, filename = mwdb(mcr.Host, mcr.Api, hash)
@@ -79,7 +79,7 @@ func (malrepo MalwareRepoType) QueryAndDownload(repos []RepositoryConfigEntry, h
 			found, filename = inquestlabs(mcr.Host, mcr.Api, hash)
 			checkedRepo = InQuest
 		case HybridAnalysis:
-			found, filename = hybridAnlysis(mcr.Host, mcr.Api, hash, doNotExtract)
+			found, filename = hybridAnalysis(mcr.Host, mcr.Api, hash)
 			checkedRepo = HybridAnalysis
 		case Polyswarm:
 			found, filename = polyswarm(mcr.Host, mcr.Api, hash)
@@ -126,8 +126,7 @@ func (malrepo MalwareRepoType) QueryAndDownload(repos []RepositoryConfigEntry, h
 			found, filename = mwdb(mcr.Host, mcr.Api, hash)
 			checkedRepo = UploadMWDB
 		}
-		// So some repos we can't download from but we want to know that it exists at that service
-		// At the moment, this is just Any.Run but suspect more will be added as time goes on
+
 		if found {
 			return found, filename, checkedRepo
 		}
@@ -139,10 +138,6 @@ func (malrepo MalwareRepoType) VerifyRepoParams(repo RepositoryConfigEntry) bool
 	switch malrepo {
 	case NotSupported:
 		return false
-	case MalwareBazaar:
-		if repo.Host != "" {
-			return true
-		}
 	case ObjectiveSee:
 		if repo.Host != "" {
 			return true
@@ -181,7 +176,7 @@ func (malrepo MalwareRepoType) CreateEntry() (RepositoryConfigEntry, error) {
 	case MWDB:
 		default_url = "https://mwdb.cert.pl/api"
 	case CapeSandbox:
-		default_url = "https://www.capesandbox.com/apiv2"
+		default_url = ""
 	case JoeSandbox:
 		default_url = "https://jbxcloud.joesecurity.org/api/v2"
 	case InQuest:
@@ -193,7 +188,7 @@ func (malrepo MalwareRepoType) CreateEntry() (RepositoryConfigEntry, error) {
 	case VirusTotal:
 		default_url = "https://www.virustotal.com/api/v3"
 	case Polyswarm:
-		default_url = "https://api.polyswarm.network/v2"
+		default_url = "https://api.polyswarm.network/v3"
 	case ObjectiveSee:
 		default_url = "https://objective-see.com/malware.json"
 	case UnpacMe:
@@ -237,7 +232,7 @@ func (malrepo MalwareRepoType) CreateEntry() (RepositoryConfigEntry, error) {
 			fmt.Println("Invalid option entered")
 		}
 	}
-	if malrepo != MalwareBazaar && malrepo != ObjectiveSee {
+	if malrepo != ObjectiveSee {
 		fmt.Println("Enter API Key:")
 		fmt.Print(">> ")
 		fmt.Scanln(&api)

download.go

@@ -15,6 +15,7 @@ import (
 	"os"
 	"regexp"
 	"strings"
+	"syscall"
 
 	"github.com/yeka/zip"
 )
@@ -580,7 +581,7 @@ func polyswarm(uri string, api string, hash Hash) (bool, string) {
 }
 
 func polyswarmDownload(uri string, api string, hash Hash) (bool, string) {
-	query := "/download/" + url.PathEscape(hash.HashType.String()) + "/" + url.PathEscape(hash.Hash)
+	query := "/consumer/download/" + url.PathEscape(hash.HashType.String()) + "/" + url.PathEscape(hash.Hash)
 
 	_, error := url.ParseQuery(query)
 	if error != nil {
@@ -620,7 +621,7 @@ func polyswarmDownload(uri string, api string, hash Hash) (bool, string) {
 	}
 }
 
-func hybridAnlysis(uri string, api string, hash Hash, doNotExtract bool) (bool, string) {
+func hybridAnalysis(uri string, api string, hash Hash) (bool, string) {
 	if api == "" {
 		fmt.Println("    [!] !! Missing Key !!")
 		return false, ""
@@ -672,12 +673,12 @@ func hybridAnlysis(uri string, api string, hash Hash, doNotExtract bool) (bool,
 	}
 
 	if hash.HashType == sha256 {
-		return hybridAnlysisDownload(uri, api, hash, doNotExtract)
+		return hybridAnalysisDownload(uri, api, hash)
 	}
 	return false, ""
 }
 
-func hybridAnlysisDownload(uri string, api string, hash Hash, extract bool) (bool, string) {
+func hybridAnalysisDownload(uri string, api string, hash Hash) (bool, string) {
 	request, error := http.NewRequest("GET", uri+"/overview/"+url.PathEscape(hash.Hash)+"/sample", nil)
 
 	request.Header.Set("accept", "application/gzip")
@@ -699,6 +700,9 @@ func hybridAnlysisDownload(uri string, api string, hash Hash, extract bool) (boo
 	if response.StatusCode == http.StatusForbidden {
 		fmt.Printf("    [!] Not authorized.  Check the URL and APIKey in the config.\nCould also be that the sample is not allowed to be downloaded.\n")
 		return false, ""
+	} else if response.StatusCode == http.StatusNotFound {
+		fmt.Printf("    [!] Hash not found")
+		return false, ""
 	} else if response.StatusCode != http.StatusOK {
 		return false, ""
 	}
@@ -812,52 +816,18 @@ func traigeDownload(uri string, api string, sampleId string, hash Hash) (bool, s
 		fmt.Println(error)
 		return false, ""
 	}
-	// Triage will download an archive file that contians the hash in question sometimes versus the actual sample being requested
-	hashMatch, _ := hash.ValidateFile(hash.Hash)
+	// Triage will download the sample directly - no password protected zip file.
+	hashMatch, dhash := hash.ValidateFile(hash.Hash)
 	if !hashMatch {
-		files, err := extractPwdZip(hash.Hash, "", false, hash)
-		if err != nil {
-			fmt.Println(error)
-			return false, ""
-		}
+		fmt.Printf("      [!] Sample ID %s (%s)\n          contains the file in question, further processing of the sample is needed to get the hash requested.\n", sampleId, dhash)
+		//ok := YesNoPrompt(fmt.Sprintf("      [?] Keep the file %s or delete it and continue looking for sample?", dhash), false)
+		//if ok {
+		return true, hash.Hash
+		//	} else {
+		//return false, ""
+		//}
 
-		found := false
-
-		fmt.Printf("      [-] The downloaded file appears to be a zip file in which the requested file should be located.\n")
-		for _, f := range files {
-			fmt.Printf("        [-] Checking file: %s\n", f.Name)
-			hashMatch, _ = hash.ValidateFile(f.Name)
-			if !hashMatch {
-				err = os.Remove(f.Name)
-				if err != nil {
-					fmt.Println("        [!] Error when deleting file: ", f.Name)
-					fmt.Println(err)
-				}
-			} else {
-				fmt.Printf("        [+] %s is hash %s\n", f.Name, hash.Hash)
-				err = os.Rename(f.Name, hash.Hash)
-				if err != nil {
-					fmt.Println("        [!] Error when renaming file: ", f.Name)
-					fmt.Println(err)
-				} else {
-					found = true
-				}
-			}
-		}
-		if !found {
-			fmt.Printf("        [!] Hash %s not found\n", hash.Hash)
-			err = os.Remove(hash.Hash)
-			if err != nil {
-				fmt.Println("        [!] Error when deleting file: ", hash.Hash)
-				fmt.Println(err)
-			}
-			return false, ""
-		} else {
-			fmt.Printf("      [+] Found %s\n", hash.Hash)
-			return true, hash.Hash
-		}
 	} else {
-		fmt.Printf("    [+] Downloaded %s\n", hash.Hash)
 		return true, hash.Hash
 	}
 }
@@ -905,7 +875,7 @@ func malshareDownload(uri string, api string, hash Hash) (bool, string) {
 	}
 }
 
-func malwareBazaar(uri string, hash Hash, doNotExtract bool, password string) (bool, string) {
+func malwareBazaar(uri string, api string, hash Hash, doNotExtract bool, password string) (bool, string) {
 	if hash.HashType != sha256 {
 		fmt.Printf("    [-] Looking up sha256 hash for %s\n", hash.Hash)
 
@@ -949,19 +919,26 @@ func malwareBazaar(uri string, hash Hash, doNotExtract bool, password string) (b
 	}
 
 	if hash.HashType == sha256 {
-		return malwareBazaarDownload(uri, hash, doNotExtract, password)
+		return malwareBazaarDownload(uri, api, hash, doNotExtract, password)
 	}
 	return false, ""
 }
 
-func malwareBazaarDownload(uri string, hash Hash, doNotExtract bool, password string) (bool, string) {
+func malwareBazaarDownload(uri string, api string, hash Hash, doNotExtract bool, password string) (bool, string) {
 	query := "query=get_file&sha256_hash=" + hash.Hash
-	values, err := url.ParseQuery(query)
-	if err != nil {
-		fmt.Println(err)
+	values, error := url.ParseQuery(query)
+	if error != nil {
+		fmt.Println(error)
 		return false, ""
 	}
 
+	request, error := http.NewRequest("POST", uri, nil)
+	if error != nil {
+		fmt.Println(error)
+		return false, ""
+	}
+
+	request.Header.Set("Auth-Key", api)
 	client := &http.Client{}
 
 	response, err := client.PostForm(uri, values)
@@ -976,14 +953,13 @@ func malwareBazaarDownload(uri string, hash Hash, doNotExtract bool, password st
 		if response.StatusCode == http.StatusMethodNotAllowed {
 			if !strings.HasSuffix(uri, "/") {
 				fmt.Printf("    [!] Trying again with a trailing slash: %s/\n", uri)
-				return malwareBazaarDownload(uri+"/", hash, doNotExtract, password)
+				return malwareBazaarDownload(uri+"/", api, hash, doNotExtract, password)
 			} else {
 				fmt.Printf("    [!] Normally the response code: %s means that the provided URL %s needs a trailing slash (to avoid the redirect), but this already has a trailing slash.\nPlease file a bug report at https://github.com/xorhex/mlget/issues\n", response.Status, uri)
 			}
 		} else {
 			fmt.Printf("    [!] %s\n", response.Status)
 		}
-		return false, ""
 	}
 
 	err = writeToFile(response.Body, hash.Hash+".zip")
@@ -1108,6 +1084,9 @@ func vxsharedownload(uri string, api string, hash Hash, doNotExtract bool, passw
 
 	if response.StatusCode == 404 {
 		return false, ""
+	} else if response.StatusCode == http.StatusInternalServerError {
+		fmt.Printf("    [!] Internal service error.  Skipping.\n")
+		return false, ""
 	} else if response.StatusCode == 204 {
 		fmt.Printf("    [!] Request rate limit exceeded. You are making more requests than are allowed or have exceeded your quota.\n")
 		return false, ""
@@ -1361,8 +1340,13 @@ func assemblyline(uri string, user string, api string, ignoretlserrors bool, has
 		client := &http.Client{Transport: tr}
 		response, error := client.Do(request)
 		if error != nil {
-			fmt.Println(error)
-			return false, ""
+			if errors.Is(error, syscall.ECONNREFUSED) {
+				fmt.Println("    [!] Connection Refused.  Is the service online?")
+				return false, ""
+			} else {
+				fmt.Println(error)
+				return false, ""
+			}
 		}
 		defer response.Body.Close()
 
@@ -1519,6 +1503,7 @@ func virusexchangeDownload(uri string, hash Hash) (bool, string) {
 	defer response.Body.Close()
 
 	if response.StatusCode == http.StatusNotFound {
+		fmt.Printf("    [!] Invalid download link returned by the API.\n")
 		return false, ""
 	} else if response.StatusCode == http.StatusForbidden {
 		fmt.Printf("    [!] Not authorized for some reason.\n")

go.mod

@@ -3,11 +3,14 @@ module xorhex.com/mlget
 go 1.21.3
 
 require (
-	github.com/kr/pretty v0.1.0
-	github.com/spf13/pflag v1.0.5
+	github.com/spf13/pflag v1.0.6
 	github.com/yeka/zip v0.0.0-20180914125537-d046722c6feb
+	golang.org/x/exp v0.0.0-20230224173230-c95f2b4c22f2
+	gopkg.in/yaml.v2 v2.4.0
+)
+
+require (
+	github.com/kr/pretty v0.3.1 // indirect
 	golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 // indirect
-	golang.org/x/exp v0.0.0-20221217163422-3c43f8badb15
 	gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect
-	gopkg.in/yaml.v2 v2.4.0
 )

hashes.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"bufio"
 	"crypto"
 	"errors"
 	"fmt"
@@ -160,22 +161,12 @@ func (h Hash) Validate(bytes []byte) (bool, string) {
 }
 
 func deleteInvalidFile(filename string) {
-	if !alwaysDeleteInvalidFile {
-		var delete_file string
-		fmt.Printf("    [?] Delete invalid file? [A/Y/n] Always delete/Yes, this time/No, not this time\n")
-		fmt.Scanln(&delete_file)
-		if strings.ToUpper(delete_file) == "Y" || delete_file == "" || strings.ToUpper(delete_file) == "A" {
-			os.Remove(filename)
-			fmt.Printf("    [!] Deleted invalid file\n")
-			if strings.ToUpper(delete_file) == "A" {
-				alwaysDeleteInvalidFile = true
-			}
-		} else {
-			fmt.Printf("    [!] Keeping invalid file\n")
-		}
-	} else {
+	ok := YesNoAlwaysDeleteInvalidFilePrompt("    [?] Delete invalid file?", true)
+	if ok {
 		os.Remove(filename)
 		fmt.Printf("    [!] Deleted invalid file\n")
+	} else {
+		fmt.Printf("    [!] Keeping invalid file\n")
 	}
 }
 
@@ -220,3 +211,37 @@ func extractHashes(text string) ([]string, error) {
 
 	return hashes, nil
 }
+
+func YesNoAlwaysDeleteInvalidFilePrompt(label string, def bool) bool {
+	if alwaysDeleteInvalidFile {
+		return true
+	}
+
+	choices := "a - always /Y - Yes /n - no"
+	if !def {
+		choices = "a - always /y - yes /N - No"
+	}
+
+	r := bufio.NewReader(os.Stdin)
+	var s string
+
+	for {
+		fmt.Fprintf(os.Stderr, "%s (%s) ", label, choices)
+		s, _ = r.ReadString('\n')
+		s = strings.TrimSpace(s)
+		if s == "" {
+			return def
+		}
+		s = strings.ToLower(s)
+		if s == "y" || s == "yes" || s == "Y" {
+			return true
+		}
+		if s == "n" || s == "no" || s == "N" {
+			return false
+		}
+		if s == "a" || s == "always" || s == "A" {
+			alwaysDeleteInvalidFile = true
+			return true
+		}
+	}
+}

mlget-test-config/samples.yaml

@@ -54,7 +54,13 @@ test 19:
   hash: b78e786091f017510b44137961f3074fe7d5f950
 test 20:
   name: TestTriageV2
-  hash: 5eaaf8ac2d358c2d7065884b7994638fee3987f02474e54467f14b010a18d028
+  hash: 0d8d46ec44e737e6ef6cd7df8edf95d83807e84be825ef76089307b399a6bcbb 
 test 21:
   name: TestVirusExchange
-  hash: ad5156e1e0285de9348d9b2c4649d9f6f39bac89567f833b1a8ba3d26468fc84
+  hash: 5d11b9be5daa65fe010cc7900d5d5eead7f62a7885e862a5971a005856ae9878 
+test 22:
+  name: TestHybridAnalysisNotFound
+  hash: fc17c021f18ec73d1544ad46dde6a1f1949f126bf3e75f97e241f982e2b07c86
+test 23:
+  name: TestVirusExchangeV2
+  hash: 0cacdb88b24bd34b9d8ef600b06814f76206e60e70f975c8e4bdaa1ab7cebb80