Skip to content

Commit e9da72f

Browse files
egibsegibs
authored
Use the include directive to centralize common private rules (#937)
* Use the include directive to centralize common private rules Signed-off-by: egibs <20933572+egibs@users.noreply.github.com> * Prefix global rules; fix include statements for tests Signed-off-by: egibs <20933572+egibs@users.noreply.github.com> * Import before include Signed-off-by: egibs <20933572+egibs@users.noreply.github.com> --------- Signed-off-by: egibs <20933572+egibs@users.noreply.github.com>
1 parent 215474f commit e9da72f

46 files changed

Lines changed: 519 additions & 597 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/workflows/style.yaml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -31,14 +31,14 @@ jobs:
3131

3232
- name: Install yara-x
3333
run: |
34-
wget https://github.com/VirusTotal/yara-x/releases/download/v0.10.0/yara-x-v0.10.0-x86_64-unknown-linux-gnu.gzip -O yara-x.gzip
35-
tar -xzvf yara-x.gzip && mv yr /usr/local/bin/ && rm yara-x.gzip
34+
wget https://github.com/VirusTotal/yara-x/releases/download/v0.15.0/yara-x-v0.15.0-x86_64-unknown-linux-gnu.gz -O yara-x.gz
35+
tar -xzvf yara-x.gz && mv yr /usr/local/bin/ && rm yara-x.gz
3636
- name: Verify yr installation
3737
run: |
3838
yr --version
3939
- name: Run yr compile
4040
run: |
41-
yr compile rules/
41+
yr compile --path-as-namespace -w rules/
4242
ret=$?
4343
if [[ $ret -ne 0 ]]; then
4444
echo "Rule compilation failed; address findings and commit the changes"

‎pkg/compile/compile.go‎

Lines changed: 57 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,9 +4,11 @@
44
package compile
55

66
import (
7+
"bytes"
78
"context"
89
"fmt"
910
"io/fs"
11+
"os"
1012
"path/filepath"
1113
"regexp"
1214
"strings"
@@ -17,6 +19,11 @@ import (
1719
yarax "github.com/VirusTotal/yara-x/go"
1820
)
1921

22+
const (
23+
globalInclude = `include "rules/global/global.yara"`
24+
globalPath = "rules/global/global.yara"
25+
)
26+
2027
var FS = rules.FS
2128

2229
// badRules are noisy 3rd party rules to silently disable.
@@ -159,16 +166,57 @@ func removeRules(data []byte, rulesToRemove []string) []byte {
159166
return newlinePattern.ReplaceAll(modified, []byte("\n\n"))
160167
}
161168

169+
// findRoot locates the repository root on the fly.
170+
func findRoot(start string) string {
171+
current := start
172+
for {
173+
next := filepath.Join(current, "rules")
174+
if _, err := os.Stat(next); err == nil {
175+
return current
176+
}
177+
178+
parent := filepath.Dir(current)
179+
if parent == current {
180+
return ""
181+
}
182+
183+
current = parent
184+
}
185+
}
186+
187+
// replaceGlobal updates the include string to reference the absolute path of rules/global/global.yara
188+
// by default, the relative path is valid for local compilations and builds done from the root of the repository,
189+
// but this is not valid for test files located in various directories.
190+
func replaceGlobal(data []byte, path string) []byte {
191+
modified := data
192+
if bytes.Contains(data, []byte(globalInclude)) {
193+
modified = bytes.Replace(data, []byte(globalInclude), []byte(fmt.Sprintf(`include "%s"`, path)), 1)
194+
}
195+
return modified
196+
}
197+
162198
func Recursive(ctx context.Context, fss []fs.FS) (*yarax.Rules, error) {
163199
if ctx.Err() != nil {
164200
return nil, ctx.Err()
165201
}
166202

167-
yxc, err := yarax.NewCompiler(yarax.ConditionOptimization(true))
203+
yxc, err := yarax.NewCompiler(yarax.ConditionOptimization(true), yarax.EnableIncludes(true))
168204
if err != nil {
169205
return nil, fmt.Errorf("yarax compiler: %w", err)
170206
}
171207

208+
// use the current working directory to determine the root path
209+
// this only needs to be done once
210+
cwd, err := os.Getwd()
211+
if err != nil {
212+
return nil, err
213+
}
214+
abs, err := filepath.Abs(cwd)
215+
if err != nil {
216+
return nil, err
217+
}
218+
rootPath := findRoot(abs)
219+
172220
rulesToRemove := getRulesToRemove()
173221

174222
for _, root := range fss {
@@ -177,14 +225,21 @@ func Recursive(ctx context.Context, fss []fs.FS) (*yarax.Rules, error) {
177225
return err
178226
}
179227

180-
if !d.IsDir() && (filepath.Ext(path) == ".yara" || filepath.Ext(path) == ".yar") {
228+
if d.IsDir() {
229+
return nil
230+
}
231+
232+
if filepath.Ext(path) == ".yara" || filepath.Ext(path) == ".yar" {
181233
bs, err := fs.ReadFile(root, path)
182234
if err != nil {
183235
return fmt.Errorf("readfile: %w", err)
184236
}
185237

186238
bs = removeRules(bs, rulesToRemove)
187239

240+
globalAbs := filepath.Join(rootPath, globalPath)
241+
bs = replaceGlobal(bs, globalAbs)
242+
188243
yxc.NewNamespace(path)
189244
if err := yxc.AddSource(string(bs), yarax.WithOrigin(path)); err != nil {
190245
return fmt.Errorf("failed to parse %s: %v", path, err)

‎rules/anti-behavior/random_behavior.yara‎

Lines changed: 2 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -1,21 +1,6 @@
11
import "math"
22

3-
private rule random_behavior_pythonSetup {
4-
strings:
5-
$if_distutils = /from distutils.core import .{0,32}setup/
6-
$if_setuptools = /from setuptools import .{0,32}setup/
7-
$i_setuptools = "import setuptools"
8-
$setup = "setup("
9-
10-
$not_setup_example = ">>> setup("
11-
$not_setup_todict = "setup(**config.todict()"
12-
$not_import_quoted = "\"from setuptools import setup"
13-
$not_setup_quoted = "\"setup(name="
14-
$not_distutils = "from distutils.errors import"
15-
16-
condition:
17-
filesize < 128KB and $setup and any of ($i*) and none of ($not*)
18-
}
3+
include "rules/global/global.yara"
194

205
rule setuptools_random: critical {
216
meta:
@@ -27,7 +12,7 @@ rule setuptools_random: critical {
2712
$not_easy_install = "pid = random.randint(0, sys.maxsize)"
2813
2914
condition:
30-
random_behavior_pythonSetup and $ref and none of ($not*)
15+
global_python_setup and $ref and none of ($not*)
3116
}
3217

3318
rule java_random: low {

‎rules/anti-static/elf/entropy.yara‎

Lines changed: 5 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,22 +1,14 @@
11
import "math"
22

3-
private rule normal_elf {
4-
condition:
5-
filesize < 64MB and uint32(0) == 1179403647
6-
}
7-
8-
private rule small_elf {
9-
condition:
10-
filesize < 400KB and uint32(0) == 1179403647
11-
}
3+
include "rules/global/global.yara"
124

135
rule higher_elf_entropy_68: medium {
146
meta:
157
description = "higher entropy ELF binary (>6.95)"
168
filetypes = "elf"
179

1810
condition:
19-
normal_elf and math.entropy(1, filesize) >= 6.95
11+
global_normal_elf and math.entropy(1, filesize) >= 6.95
2012
}
2113

2214
rule normal_elf_high_entropy_7_4: high {
@@ -29,7 +21,7 @@ rule normal_elf_high_entropy_7_4: high {
2921
$not_bazel = "BazelLogHandler"
3022
3123
condition:
32-
filesize < 30MB and normal_elf and math.entropy(1, filesize) >= 7.4 and none of ($not*)
24+
filesize < 30MB and global_normal_elf and math.entropy(1, filesize) >= 7.4 and none of ($not*)
3325
}
3426

3527
rule normal_elf_high_entropy_footer_7_4: high {
@@ -38,7 +30,7 @@ rule normal_elf_high_entropy_footer_7_4: high {
3830
filetypes = "elf"
3931

4032
condition:
41-
normal_elf and math.entropy(filesize - 8192, filesize) >= 7.4
33+
global_normal_elf and math.entropy(filesize - 8192, filesize) >= 7.4
4234
}
4335

4436
rule normal_elf_high_entropy_footer_7_4_rc4: high {
@@ -51,5 +43,5 @@ rule normal_elf_high_entropy_footer_7_4_rc4: high {
5143
$cmp_r_x_256 = { 48 81 f? 00 01 00 00 } // cmp {rbx, rcx, …}, 256
5244
5345
condition:
54-
filesize < 25MB and normal_elf and math.entropy(filesize - 8192, filesize) >= 7.4 and any of them
46+
filesize < 25MB and global_normal_elf and math.entropy(filesize - 8192, filesize) >= 7.4 and any of them
5547
}

‎rules/anti-static/macho/entropy.yara‎

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,14 @@
11
import "math"
22

3-
private rule smaller_macho {
4-
condition:
5-
filesize < 64MB and (uint32(0) == 4277009102 or uint32(0) == 3472551422 or uint32(0) == 4277009103 or uint32(0) == 3489328638 or uint32(0) == 3405691582 or uint32(0) == 3199925962)
6-
}
3+
include "rules/global/global.yara"
74

85
rule higher_entropy_6_9: medium {
96
meta:
107
description = "higher entropy binary (>6.9)"
118
filetypes = "macho"
129

1310
condition:
14-
smaller_macho and math.entropy(1, filesize) >= 6.9
11+
global_small_macho and math.entropy(1, filesize) >= 6.9
1512
}
1613

1714
rule high_entropy_7_2: high {
@@ -24,5 +21,5 @@ rule high_entropy_7_2: high {
2421
$bin_java = "bin/java"
2522
2623
condition:
27-
smaller_macho and math.entropy(1, filesize) >= 7.2 and not $bin_java
24+
global_small_macho and math.entropy(1, filesize) >= 7.2 and not $bin_java
2825
}

‎rules/anti-static/macho/footer.yara‎

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,6 @@
11
import "math"
22

3-
private rule anti_static_macho {
4-
condition:
5-
(uint32(0) == 4277009102 or uint32(0) == 3472551422 or uint32(0) == 4277009103 or uint32(0) == 3489328638 or uint32(0) == 3405691582 or uint32(0) == 3199925962 or uint32(0) == 3405691583 or uint32(0) == 3216703178)
6-
}
3+
include "rules/global/global.yara"
74

85
rule high_entropy_trailer: high {
96
meta:
@@ -15,5 +12,5 @@ rule high_entropy_trailer: high {
1512
$page_zero = "_PAGEZERO"
1613
1714
condition:
18-
filesize < 10MB and anti_static_macho and $page_zero and math.entropy(filesize - 1024, filesize - 1) >= 4
15+
filesize < 10MB and global_macho and $page_zero and math.entropy(filesize - 1024, filesize - 1) >= 4
1916
}

‎rules/anti-static/packer/aes.yara‎

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,6 @@
11
import "math"
22

3-
private rule smallBinary {
4-
condition:
5-
// matches ELF or machO binary
6-
filesize > 1MB and filesize < 8MB and (uint32(0) == 1179403647 or uint32(0) == 4277009102 or uint32(0) == 3472551422 or uint32(0) == 4277009103 or uint32(0) == 3489328638 or uint32(0) == 3405691582 or uint32(0) == 3199925962)
7-
}
3+
include "rules/global/global.yara"
84

95
rule go_aes: high {
106
meta:
@@ -17,5 +13,5 @@ rule go_aes: high {
1713
$decrypt = "NewCFBDecrypter"
1814
1915
condition:
20-
smallBinary and math.entropy(1, filesize) >= 7 and all of them
16+
global_small_binary and math.entropy(1, filesize) >= 7 and all of them
2117
}

‎rules/anti-static/unmarshal/marshal.yara‎

Lines changed: 2 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,6 @@
11
import "math"
22

3-
private rule pySetup {
4-
strings:
5-
$i_distutils = "from distutils.core import setup"
6-
$i_setuptools = "setuptools"
7-
$setup = "setup("
8-
$not_setuptools = "setuptools.command"
9-
10-
condition:
11-
filesize < 2097152 and $setup and any of ($i*) and none of ($not*)
12-
}
3+
include "rules/global/global.yara"
134

145
rule unmarshal_py_marshal: medium {
156
meta:
@@ -29,5 +20,5 @@ rule setuptools_py_marshal: suspicious {
2920
filetypes = "py"
3021

3122
condition:
32-
pySetup and unmarshal_py_marshal
23+
global_python_setup and unmarshal_py_marshal
3324
}

‎rules/c2/addr/ip.yara‎

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
include "rules/global/global.yara"
2+
13
rule hardcoded_ip: medium {
24
meta:
35
description = "hardcoded IP address"
@@ -19,11 +21,6 @@ rule hardcoded_ip: medium {
1921
filesize < 200MB and 1 of ($sus_ip*) and none of ($not*)
2022
}
2123

22-
private rule ip_elf_or_macho {
23-
condition:
24-
uint32(0) == 1179403647 or (uint32(0) == 4277009102 or uint32(0) == 3472551422 or uint32(0) == 4277009103 or uint32(0) == 3489328638 or uint32(0) == 3405691582 or uint32(0) == 3199925962 or uint32(0) == 3405691583 or uint32(0) == 3216703178)
25-
}
26-
2724
rule bin_hardcoded_ip: high {
2825
meta:
2926
description = "ELF with hardcoded IP address"
@@ -48,7 +45,7 @@ rule bin_hardcoded_ip: high {
4845
$not_2345 = "23.45.67.89"
4946
5047
condition:
51-
filesize < 12MB and ip_elf_or_macho and 1 of ($sus_ip*) and none of ($not*)
48+
filesize < 12MB and global_elf_or_macho and 1 of ($sus_ip*) and none of ($not*)
5249
}
5350

5451
rule http_hardcoded_ip: high exfil {

‎rules/c2/addr/url.yara‎

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,6 @@
11
import "math"
22

3-
private rule elf_or_macho {
4-
condition:
5-
uint32(0) == 1179403647 or (uint32(0) == 4277009102 or uint32(0) == 3472551422 or uint32(0) == 4277009103 or uint32(0) == 3489328638 or uint32(0) == 3405691582 or uint32(0) == 3199925962 or uint32(0) == 3405691583 or uint32(0) == 3216703178)
6-
}
3+
include "rules/global/global.yara"
74

85
rule unusual_nodename: medium {
96
meta:
@@ -85,7 +82,7 @@ rule binary_with_url: low {
8582
$ref = /https*:\/\/[\w\.\/]{8,160}[\/\w\=\&]{0,32}/
8683
8784
condition:
88-
filesize < 150MB and elf_or_macho and $ref
85+
filesize < 150MB and global_elf_or_macho and $ref
8986
}
9087

9188
rule binary_url_with_question: high {
@@ -102,7 +99,7 @@ rule binary_url_with_question: high {
10299
$not_mesibo = "https://api.mesibo.com/api.php?"
103100
104101
condition:
105-
filesize < 150MB and elf_or_macho and $ref and none of ($not*)
102+
filesize < 150MB and global_elf_or_macho and $ref and none of ($not*)
106103
}
107104

108105
rule script_url_with_question: high {

0 commit comments

Comments
 (0)