Prefix global rules; fix include statements for tests

Signed-off-by: egibs <20933572+egibs@users.noreply.github.com>

Author: egibs

.github/workflows/style.yaml

@@ -31,8 +31,8 @@ jobs:
 
       - name: Install yara-x
         run: |
-          wget https://github.com/VirusTotal/yara-x/releases/download/v0.10.0/yara-x-v0.10.0-x86_64-unknown-linux-gnu.gzip -O yara-x.gzip
-          tar -xzvf yara-x.gzip && mv yr /usr/local/bin/ && rm yara-x.gzip
+          wget https://github.com/VirusTotal/yara-x/releases/download/v0.15.0/yara-x-v0.15.0-x86_64-unknown-linux-gnu.gz -O yara-x.gz
+          tar -xzvf yara-x.gz && mv yr /usr/local/bin/ && rm yara-x.gz
       - name: Verify yr installation
         run: |
           yr --version

pkg/compile/compile.go

@@ -4,9 +4,11 @@
 package compile
 
 import (
+	"bytes"
 	"context"
 	"fmt"
 	"io/fs"
+	"os"
 	"path/filepath"
 	"regexp"
 	"strings"
@@ -17,6 +19,11 @@ import (
 	yarax "github.com/VirusTotal/yara-x/go"
 )
 
+const (
+	globalInclude = `include "rules/global/global.yara"`
+	globalPath    = "rules/global/global.yara"
+)
+
 var FS = rules.FS
 
 // badRules are noisy 3rd party rules to silently disable.
@@ -159,16 +166,57 @@ func removeRules(data []byte, rulesToRemove []string) []byte {
 	return newlinePattern.ReplaceAll(modified, []byte("\n\n"))
 }
 
+// findRoot locates the repository root on the fly.
+func findRoot(start string) string {
+	current := start
+	for {
+		next := filepath.Join(current, "rules")
+		if _, err := os.Stat(next); err == nil {
+			return current
+		}
+
+		parent := filepath.Dir(current)
+		if parent == current {
+			return ""
+		}
+
+		current = parent
+	}
+}
+
+// replaceGlobal updates the include string to reference the absolute path of rules/global/global.yara
+// by default, the relative path is valid for local compilations and builds done from the root of the repository,
+// but this is not valid for test files located in various directories.
+func replaceGlobal(data []byte, path string) []byte {
+	modified := data
+	if bytes.Contains(data, []byte(globalInclude)) {
+		modified = bytes.Replace(data, []byte(globalInclude), []byte(fmt.Sprintf(`include "%s"`, path)), 1)
+	}
+	return modified
+}
+
 func Recursive(ctx context.Context, fss []fs.FS) (*yarax.Rules, error) {
 	if ctx.Err() != nil {
 		return nil, ctx.Err()
 	}
 
-	yxc, err := yarax.NewCompiler(yarax.ConditionOptimization(true))
+	yxc, err := yarax.NewCompiler(yarax.ConditionOptimization(true), yarax.EnableIncludes(true))
 	if err != nil {
 		return nil, fmt.Errorf("yarax compiler: %w", err)
 	}
 
+	// use the current working directory to determine the root path
+	// this only needs to be done once
+	cwd, err := os.Getwd()
+	if err != nil {
+		return nil, err
+	}
+	abs, err := filepath.Abs(cwd)
+	if err != nil {
+		return nil, err
+	}
+	rootPath := findRoot(abs)
+
 	rulesToRemove := getRulesToRemove()
 
 	for _, root := range fss {
@@ -177,14 +225,21 @@ func Recursive(ctx context.Context, fss []fs.FS) (*yarax.Rules, error) {
 				return err
 			}
 
-			if !d.IsDir() && (filepath.Ext(path) == ".yara" || filepath.Ext(path) == ".yar") {
+			if d.IsDir() {
+				return nil
+			}
+
+			if filepath.Ext(path) == ".yara" || filepath.Ext(path) == ".yar" {
 				bs, err := fs.ReadFile(root, path)
 				if err != nil {
 					return fmt.Errorf("readfile: %w", err)
 				}
 
 				bs = removeRules(bs, rulesToRemove)
 
+				globalAbs := filepath.Join(rootPath, globalPath)
+				bs = replaceGlobal(bs, globalAbs)
+
 				yxc.NewNamespace(path)
 				if err := yxc.AddSource(string(bs), yarax.WithOrigin(path)); err != nil {
 					return fmt.Errorf("failed to parse %s: %v", path, err)

rules/anti-behavior/random_behavior.yara

@@ -1,4 +1,4 @@
-include "rules/global.yara"
+include "rules/global/global.yara"
 
 import "math"
 
@@ -12,7 +12,7 @@ rule setuptools_random: critical {
     $not_easy_install = "pid = random.randint(0, sys.maxsize)"
 
   condition:
-    python_setup and $ref and none of ($not*)
+    global_python_setup and $ref and none of ($not*)
 }
 
 rule java_random: low {

rules/anti-static/elf/entropy.yara

@@ -1,4 +1,4 @@
-include "rules/global.yara"
+include "rules/global/global.yara"
 
 import "math"
 
@@ -8,7 +8,7 @@ rule higher_elf_entropy_68: medium {
     filetypes   = "elf"
 
   condition:
-    normal_elf and math.entropy(1, filesize) >= 6.95
+    global_normal_elf and math.entropy(1, filesize) >= 6.95
 }
 
 rule normal_elf_high_entropy_7_4: high {
@@ -21,7 +21,7 @@ rule normal_elf_high_entropy_7_4: high {
     $not_bazel     = "BazelLogHandler"
 
   condition:
-    filesize < 30MB and normal_elf and math.entropy(1, filesize) >= 7.4 and none of ($not*)
+    filesize < 30MB and global_normal_elf and math.entropy(1, filesize) >= 7.4 and none of ($not*)
 }
 
 rule normal_elf_high_entropy_footer_7_4: high {
@@ -30,7 +30,7 @@ rule normal_elf_high_entropy_footer_7_4: high {
     filetypes   = "elf"
 
   condition:
-    normal_elf and math.entropy(filesize - 8192, filesize) >= 7.4
+    global_normal_elf and math.entropy(filesize - 8192, filesize) >= 7.4
 }
 
 rule normal_elf_high_entropy_footer_7_4_rc4: high {
@@ -43,5 +43,5 @@ rule normal_elf_high_entropy_footer_7_4_rc4: high {
     $cmp_r_x_256 = { 48 81 f? 00 01 00 00 }  // cmp {rbx, rcx, …}, 256
 
   condition:
-    filesize < 25MB and normal_elf and math.entropy(filesize - 8192, filesize) >= 7.4 and any of them
+    filesize < 25MB and global_normal_elf and math.entropy(filesize - 8192, filesize) >= 7.4 and any of them
 }

rules/anti-static/macho/entropy.yara

@@ -1,4 +1,4 @@
-include "rules/global.yara"
+include "rules/global/global.yara"
 
 import "math"
 
@@ -8,7 +8,7 @@ rule higher_entropy_6_9: medium {
     filetypes   = "macho"
 
   condition:
-    small_macho and math.entropy(1, filesize) >= 6.9
+    global_small_macho and math.entropy(1, filesize) >= 6.9
 }
 
 rule high_entropy_7_2: high {
@@ -21,5 +21,5 @@ rule high_entropy_7_2: high {
     $bin_java = "bin/java"
 
   condition:
-    small_macho and math.entropy(1, filesize) >= 7.2 and not $bin_java
+    global_small_macho and math.entropy(1, filesize) >= 7.2 and not $bin_java
 }

rules/anti-static/macho/footer.yara

@@ -1,4 +1,4 @@
-include "rules/global.yara"
+include "rules/global/global.yara"
 
 import "math"
 
@@ -12,5 +12,5 @@ rule high_entropy_trailer: high {
     $page_zero = "_PAGEZERO"
 
   condition:
-    filesize < 10MB and macho and $page_zero and math.entropy(filesize - 1024, filesize - 1) >= 4
+    filesize < 10MB and global_macho and $page_zero and math.entropy(filesize - 1024, filesize - 1) >= 4
 }

rules/anti-static/packer/aes.yara

@@ -1,4 +1,4 @@
-include "rules/global.yara"
+include "rules/global/global.yara"
 
 import "math"
 
@@ -13,5 +13,5 @@ rule go_aes: high {
     $decrypt = "NewCFBDecrypter"
 
   condition:
-    small_binary and math.entropy(1, filesize) >= 7 and all of them
+    global_small_binary and math.entropy(1, filesize) >= 7 and all of them
 }

rules/anti-static/unmarshal/marshal.yara

@@ -1,4 +1,4 @@
-include "rules/global.yara"
+include "rules/global/global.yara"
 
 import "math"
 
@@ -20,5 +20,5 @@ rule setuptools_py_marshal: suspicious {
     filetypes   = "py"
 
   condition:
-    python_setup and unmarshal_py_marshal
+    global_python_setup and unmarshal_py_marshal
 }

rules/c2/addr/ip.yara

@@ -1,4 +1,4 @@
-include "rules/global.yara"
+include "rules/global/global.yara"
 
 rule hardcoded_ip: medium {
   meta:
@@ -45,7 +45,7 @@ rule bin_hardcoded_ip: high {
     $not_2345              = "23.45.67.89"
 
   condition:
-    filesize < 12MB and elf_or_macho and 1 of ($sus_ip*) and none of ($not*)
+    filesize < 12MB and global_elf_or_macho and 1 of ($sus_ip*) and none of ($not*)
 }
 
 rule http_hardcoded_ip: high exfil {

rules/c2/addr/url.yara

@@ -1,4 +1,4 @@
-include "rules/global.yara"
+include "rules/global/global.yara"
 
 import "math"
 
@@ -82,7 +82,7 @@ rule binary_with_url: low {
     $ref = /https*:\/\/[\w\.\/]{8,160}[\/\w\=\&]{0,32}/
 
   condition:
-    filesize < 150MB and elf_or_macho and $ref
+    filesize < 150MB and global_elf_or_macho and $ref
 }
 
 rule binary_url_with_question: high {
@@ -99,7 +99,7 @@ rule binary_url_with_question: high {
     $not_mesibo      = "https://api.mesibo.com/api.php?"
 
   condition:
-    filesize < 150MB and elf_or_macho and $ref and none of ($not*)
+    filesize < 150MB and global_elf_or_macho and $ref and none of ($not*)
 }
 
 rule script_url_with_question: high {