Use the include directive to centralize common private rules

.github/workflows/style.yaml

@@ -31,14 +31,14 @@ jobs:
 
       - name: Install yara-x
         run: |
-          wget https://github.com/VirusTotal/yara-x/releases/download/v0.10.0/yara-x-v0.10.0-x86_64-unknown-linux-gnu.gzip -O yara-x.gzip
-          tar -xzvf yara-x.gzip && mv yr /usr/local/bin/ && rm yara-x.gzip
+          wget https://github.com/VirusTotal/yara-x/releases/download/v0.15.0/yara-x-v0.15.0-x86_64-unknown-linux-gnu.gz -O yara-x.gz
+          tar -xzvf yara-x.gz && mv yr /usr/local/bin/ && rm yara-x.gz
       - name: Verify yr installation
         run: |
           yr --version
       - name: Run yr compile
         run: |
-          yr compile rules/
+          yr compile --path-as-namespace -w rules/
           ret=$?
           if [[ $ret -ne 0 ]]; then
             echo "Rule compilation failed; address findings and commit the changes"

pkg/compile/compile.go

@@ -4,9 +4,11 @@
 package compile
 
 import (
+	"bytes"
 	"context"
 	"fmt"
 	"io/fs"
+	"os"
 	"path/filepath"
 	"regexp"
 	"strings"
@@ -17,6 +19,11 @@ import (
 	yarax "github.com/VirusTotal/yara-x/go"
 )
 
+const (
+	globalInclude = `include "rules/global/global.yara"`
+	globalPath    = "rules/global/global.yara"
+)
+
 var FS = rules.FS
 
 // badRules are noisy 3rd party rules to silently disable.
@@ -159,16 +166,57 @@ func removeRules(data []byte, rulesToRemove []string) []byte {
 	return newlinePattern.ReplaceAll(modified, []byte("\n\n"))
 }
 
+// findRoot locates the repository root on the fly.
+func findRoot(start string) string {
+	current := start
+	for {
+		next := filepath.Join(current, "rules")
+		if _, err := os.Stat(next); err == nil {
+			return current
+		}
+
+		parent := filepath.Dir(current)
+		if parent == current {
+			return ""
+		}
+
+		current = parent
+	}
+}
+
+// replaceGlobal updates the include string to reference the absolute path of rules/global/global.yara
+// by default, the relative path is valid for local compilations and builds done from the root of the repository,
+// but this is not valid for test files located in various directories.
+func replaceGlobal(data []byte, path string) []byte {
+	modified := data
+	if bytes.Contains(data, []byte(globalInclude)) {
+		modified = bytes.Replace(data, []byte(globalInclude), []byte(fmt.Sprintf(`include "%s"`, path)), 1)
+	}
+	return modified
+}
+
 func Recursive(ctx context.Context, fss []fs.FS) (*yarax.Rules, error) {
 	if ctx.Err() != nil {
 		return nil, ctx.Err()
 	}
 
-	yxc, err := yarax.NewCompiler(yarax.ConditionOptimization(true))
+	yxc, err := yarax.NewCompiler(yarax.ConditionOptimization(true), yarax.EnableIncludes(true))
 	if err != nil {
 		return nil, fmt.Errorf("yarax compiler: %w", err)
 	}
 
+	// use the current working directory to determine the root path
+	// this only needs to be done once
+	cwd, err := os.Getwd()
+	if err != nil {
+		return nil, err
+	}
+	abs, err := filepath.Abs(cwd)
+	if err != nil {
+		return nil, err
+	}
+	rootPath := findRoot(abs)
+
 	rulesToRemove := getRulesToRemove()
 
 	for _, root := range fss {
@@ -177,14 +225,21 @@ func Recursive(ctx context.Context, fss []fs.FS) (*yarax.Rules, error) {
 				return err
 			}
 
-			if !d.IsDir() && (filepath.Ext(path) == ".yara" || filepath.Ext(path) == ".yar") {
+			if d.IsDir() {
+				return nil
+			}
+
+			if filepath.Ext(path) == ".yara" || filepath.Ext(path) == ".yar" {
 				bs, err := fs.ReadFile(root, path)
 				if err != nil {
 					return fmt.Errorf("readfile: %w", err)
 				}
 
 				bs = removeRules(bs, rulesToRemove)
 
+				globalAbs := filepath.Join(rootPath, globalPath)
+				bs = replaceGlobal(bs, globalAbs)
+
 				yxc.NewNamespace(path)
 				if err := yxc.AddSource(string(bs), yarax.WithOrigin(path)); err != nil {
 					return fmt.Errorf("failed to parse %s: %v", path, err)

rules/anti-behavior/random_behavior.yara

@@ -1,21 +1,6 @@
 import "math"
 
-private rule random_behavior_pythonSetup {
-  strings:
-    $if_distutils  = /from distutils.core import .{0,32}setup/
-    $if_setuptools = /from setuptools import .{0,32}setup/
-    $i_setuptools  = "import setuptools"
-    $setup         = "setup("
-
-    $not_setup_example = ">>> setup("
-    $not_setup_todict  = "setup(**config.todict()"
-    $not_import_quoted = "\"from setuptools import setup"
-    $not_setup_quoted  = "\"setup(name="
-    $not_distutils     = "from distutils.errors import"
-
-  condition:
-    filesize < 128KB and $setup and any of ($i*) and none of ($not*)
-}
+include "rules/global/global.yara"
 
 rule setuptools_random: critical {
   meta:
@@ -27,7 +12,7 @@ rule setuptools_random: critical {
     $not_easy_install = "pid = random.randint(0, sys.maxsize)"
 
   condition:
-    random_behavior_pythonSetup and $ref and none of ($not*)
+    global_python_setup and $ref and none of ($not*)
 }
 
 rule java_random: low {

rules/anti-static/elf/entropy.yara

@@ -1,22 +1,14 @@
 import "math"
 
-private rule normal_elf {
-  condition:
-    filesize < 64MB and uint32(0) == 1179403647
-}
-
-private rule small_elf {
-  condition:
-    filesize < 400KB and uint32(0) == 1179403647
-}
+include "rules/global/global.yara"
 
 rule higher_elf_entropy_68: medium {
   meta:
     description = "higher entropy ELF binary (>6.95)"
     filetypes   = "elf"
 
   condition:
-    normal_elf and math.entropy(1, filesize) >= 6.95
+    global_normal_elf and math.entropy(1, filesize) >= 6.95
 }
 
 rule normal_elf_high_entropy_7_4: high {
@@ -29,7 +21,7 @@ rule normal_elf_high_entropy_7_4: high {
     $not_bazel     = "BazelLogHandler"
 
   condition:
-    filesize < 30MB and normal_elf and math.entropy(1, filesize) >= 7.4 and none of ($not*)
+    filesize < 30MB and global_normal_elf and math.entropy(1, filesize) >= 7.4 and none of ($not*)
 }
 
 rule normal_elf_high_entropy_footer_7_4: high {
@@ -38,7 +30,7 @@ rule normal_elf_high_entropy_footer_7_4: high {
     filetypes   = "elf"
 
   condition:
-    normal_elf and math.entropy(filesize - 8192, filesize) >= 7.4
+    global_normal_elf and math.entropy(filesize - 8192, filesize) >= 7.4
 }
 
 rule normal_elf_high_entropy_footer_7_4_rc4: high {
@@ -51,5 +43,5 @@ rule normal_elf_high_entropy_footer_7_4_rc4: high {
     $cmp_r_x_256 = { 48 81 f? 00 01 00 00 }  // cmp {rbx, rcx, …}, 256
 
   condition:
-    filesize < 25MB and normal_elf and math.entropy(filesize - 8192, filesize) >= 7.4 and any of them
+    filesize < 25MB and global_normal_elf and math.entropy(filesize - 8192, filesize) >= 7.4 and any of them
 }

rules/anti-static/macho/entropy.yara

@@ -1,17 +1,14 @@
 import "math"
 
-private rule smaller_macho {
-  condition:
-    filesize < 64MB and (uint32(0) == 4277009102 or uint32(0) == 3472551422 or uint32(0) == 4277009103 or uint32(0) == 3489328638 or uint32(0) == 3405691582 or uint32(0) == 3199925962)
-}
+include "rules/global/global.yara"
 
 rule higher_entropy_6_9: medium {
   meta:
     description = "higher entropy binary (>6.9)"
     filetypes   = "macho"
 
   condition:
-    smaller_macho and math.entropy(1, filesize) >= 6.9
+    global_small_macho and math.entropy(1, filesize) >= 6.9
 }
 
 rule high_entropy_7_2: high {
@@ -24,5 +21,5 @@ rule high_entropy_7_2: high {
     $bin_java = "bin/java"
 
   condition:
-    smaller_macho and math.entropy(1, filesize) >= 7.2 and not $bin_java
+    global_small_macho and math.entropy(1, filesize) >= 7.2 and not $bin_java
 }

rules/anti-static/macho/footer.yara

@@ -1,9 +1,6 @@
 import "math"
 
-private rule anti_static_macho {
-  condition:
-    (uint32(0) == 4277009102 or uint32(0) == 3472551422 or uint32(0) == 4277009103 or uint32(0) == 3489328638 or uint32(0) == 3405691582 or uint32(0) == 3199925962 or uint32(0) == 3405691583 or uint32(0) == 3216703178)
-}
+include "rules/global/global.yara"
 
 rule high_entropy_trailer: high {
   meta:
@@ -15,5 +12,5 @@ rule high_entropy_trailer: high {
     $page_zero = "_PAGEZERO"
 
   condition:
-    filesize < 10MB and anti_static_macho and $page_zero and math.entropy(filesize - 1024, filesize - 1) >= 4
+    filesize < 10MB and global_macho and $page_zero and math.entropy(filesize - 1024, filesize - 1) >= 4
 }

rules/anti-static/packer/aes.yara

@@ -1,10 +1,6 @@
 import "math"
 
-private rule smallBinary {
-  condition:
-    // matches ELF or machO binary
-    filesize > 1MB and filesize < 8MB and (uint32(0) == 1179403647 or uint32(0) == 4277009102 or uint32(0) == 3472551422 or uint32(0) == 4277009103 or uint32(0) == 3489328638 or uint32(0) == 3405691582 or uint32(0) == 3199925962)
-}
+include "rules/global/global.yara"
 
 rule go_aes: high {
   meta:
@@ -17,5 +13,5 @@ rule go_aes: high {
     $decrypt = "NewCFBDecrypter"
 
   condition:
-    smallBinary and math.entropy(1, filesize) >= 7 and all of them
+    global_small_binary and math.entropy(1, filesize) >= 7 and all of them
 }

rules/anti-static/unmarshal/marshal.yara

@@ -1,15 +1,6 @@
 import "math"
 
-private rule pySetup {
-  strings:
-    $i_distutils    = "from distutils.core import setup"
-    $i_setuptools   = "setuptools"
-    $setup          = "setup("
-    $not_setuptools = "setuptools.command"
-
-  condition:
-    filesize < 2097152 and $setup and any of ($i*) and none of ($not*)
-}
+include "rules/global/global.yara"
 
 rule unmarshal_py_marshal: medium {
   meta:
@@ -29,5 +20,5 @@ rule setuptools_py_marshal: suspicious {
     filetypes   = "py"
 
   condition:
-    pySetup and unmarshal_py_marshal
+    global_python_setup and unmarshal_py_marshal
 }

rules/c2/addr/ip.yara

@@ -1,3 +1,5 @@
+include "rules/global/global.yara"
+
 rule hardcoded_ip: medium {
   meta:
     description = "hardcoded IP address"
@@ -19,11 +21,6 @@ rule hardcoded_ip: medium {
     filesize < 200MB and 1 of ($sus_ip*) and none of ($not*)
 }
 
-private rule ip_elf_or_macho {
-  condition:
-    uint32(0) == 1179403647 or (uint32(0) == 4277009102 or uint32(0) == 3472551422 or uint32(0) == 4277009103 or uint32(0) == 3489328638 or uint32(0) == 3405691582 or uint32(0) == 3199925962 or uint32(0) == 3405691583 or uint32(0) == 3216703178)
-}
-
 rule bin_hardcoded_ip: high {
   meta:
     description = "ELF with hardcoded IP address"
@@ -48,7 +45,7 @@ rule bin_hardcoded_ip: high {
     $not_2345              = "23.45.67.89"
 
   condition:
-    filesize < 12MB and ip_elf_or_macho and 1 of ($sus_ip*) and none of ($not*)
+    filesize < 12MB and global_elf_or_macho and 1 of ($sus_ip*) and none of ($not*)
 }
 
 rule http_hardcoded_ip: high exfil {

rules/c2/addr/url.yara

@@ -1,9 +1,6 @@
 import "math"
 
-private rule elf_or_macho {
-  condition:
-    uint32(0) == 1179403647 or (uint32(0) == 4277009102 or uint32(0) == 3472551422 or uint32(0) == 4277009103 or uint32(0) == 3489328638 or uint32(0) == 3405691582 or uint32(0) == 3199925962 or uint32(0) == 3405691583 or uint32(0) == 3216703178)
-}
+include "rules/global/global.yara"
 
 rule unusual_nodename: medium {
   meta:
@@ -85,7 +82,7 @@ rule binary_with_url: low {
     $ref = /https*:\/\/[\w\.\/]{8,160}[\/\w\=\&]{0,32}/
 
   condition:
-    filesize < 150MB and elf_or_macho and $ref
+    filesize < 150MB and global_elf_or_macho and $ref
 }
 
 rule binary_url_with_question: high {
@@ -102,7 +99,7 @@ rule binary_url_with_question: high {
     $not_mesibo      = "https://api.mesibo.com/api.php?"
 
   condition:
-    filesize < 150MB and elf_or_macho and $ref and none of ($not*)
+    filesize < 150MB and global_elf_or_macho and $ref and none of ($not*)
 }
 
 rule script_url_with_question: high {

rules/c2/tool_transfer/download.yara

@@ -1,3 +1,5 @@
+include "rules/global/global.yara"
+
 rule download_sites: high {
   meta:
     ref         = "https://github.com/ditekshen/detection/blob/e6579590779f62cbe7f5e14b5be7d77b2280f516/yara/indicator_high.yar#L1001"
@@ -113,12 +115,6 @@ rule http_archive_url: medium {
     any of ($ref*) and none of ($not*)
 }
 
-private rule smallerBinary {
-  condition:
-    // matches ELF or machO binary
-    filesize < 10MB and (uint32(0) == 1179403647 or uint32(0) == 4277009102 or uint32(0) == 3472551422 or uint32(0) == 4277009103 or uint32(0) == 3489328638 or uint32(0) == 3405691582 or uint32(0) == 3199925962)
-}
-
 rule http_archive_url_higher: high {
   meta:
     description = "accesses hardcoded archive file endpoint"
@@ -129,5 +125,5 @@ rule http_archive_url_higher: high {
     $not_foo_bar = "http://foo/bar.tar"
 
   condition:
-    smallerBinary and any of ($ref*) and none of ($not*)
+    global_small_binary and any of ($ref*) and none of ($not*)
 }

rules/c2/tool_transfer/macos.yara

@@ -1,12 +1,4 @@
-private rule tool_transfer_macho {
-  strings:
-    $not_jar   = "META-INF/"
-    $not_dwarf = "_DWARF"
-    $not_kext  = "_.SYMDEF SORTED"
-
-  condition:
-    (uint32(0) == 4277009102 or uint32(0) == 3472551422 or uint32(0) == 4277009103 or uint32(0) == 3489328638 or uint32(0) == 3405691582 or uint32(0) == 3199925962 or uint32(0) == 3405691583 or uint32(0) == 3216703178) and none of ($not*)
-}
+include "rules/global/global.yara"
 
 rule macos_chflags_hidden: critical {
   meta:
@@ -38,5 +30,5 @@ rule cocoa_bundle_dropper: critical {
     $platform = "isPlatformOrVariantPlatformVersionAtLeast" fullword
 
   condition:
-    tool_transfer_macho and $shared and 5 of them
+    global_specific_macho and $shared and 5 of them
 }