Skip to content

Commit ea450d3

Browse files
ziadhanyziadhany
committed
Add support for YARA data collection.
Add YARA tests. Signed-off-by: ziad hany <ziadhany2016@gmail.com>
1 parent 2dbbd38 commit ea450d3

8 files changed

Lines changed: 354 additions & 0 deletions

File tree

‎vulnerabilities/improvers/__init__.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@
3333
from vulnerabilities.pipelines.v2_improvers import flag_ghost_packages as flag_ghost_packages_v2
3434
from vulnerabilities.pipelines.v2_improvers import relate_severities
3535
from vulnerabilities.pipelines.v2_improvers import unfurl_version_range as unfurl_version_range_v2
36+
from vulnerabilities.pipelines.v2_improvers import yara_rules
3637
from vulnerabilities.utils import create_registry
3738

3839
IMPROVERS_REGISTRY = create_registry(
@@ -74,5 +75,6 @@
7475
compute_advisory_todo.ComputeToDo,
7576
collect_ssvc_trees.CollectSSVCPipeline,
7677
relate_severities.RelateSeveritiesPipeline,
78+
yara_rules.YaraRulesImproverPipeline,
7779
]
7880
)

‎vulnerabilities/models.py‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3649,3 +3649,45 @@ def __str__(self):
36493649

36503650
class Meta:
36513651
unique_together = ("vector", "source_advisory")
3652+
3653+
3654+
class DetectionRuleTypes(models.TextChoices):
3655+
"""Defines the supported formats for security detection rules."""
3656+
3657+
YARA = "yara", "Yara"
3658+
YARA_X = "yara-x", "Yara-X"
3659+
SIGMA = "sigma", "Sigma"
3660+
CLAMAV = "clamav", "ClamAV"
3661+
SURICATA = "suricata", "Suricata"
3662+
3663+
3664+
class DetectionRule(models.Model):
3665+
"""
3666+
A Detection Rule is code used to identify malicious activity or security threats.
3667+
"""
3668+
3669+
rule_type = models.CharField(
3670+
max_length=50,
3671+
choices=DetectionRuleTypes.choices,
3672+
help_text="The type of the detection rule content (e.g., YARA, Sigma).",
3673+
)
3674+
3675+
source_url = models.URLField(
3676+
max_length=1024, help_text="URL to the original source or reference for this rule."
3677+
)
3678+
3679+
rule_metadata = models.JSONField(
3680+
null=True,
3681+
blank=True,
3682+
help_text="Additional structured data such as tags, or author information.",
3683+
)
3684+
3685+
rule_text = models.TextField(help_text="The content of the detection signature.")
3686+
3687+
advisory = models.ForeignKey(
3688+
AdvisoryV2,
3689+
related_name="detection_rules",
3690+
on_delete=models.SET_NULL,
3691+
null=True,
3692+
blank=True,
3693+
)

‎vulnerabilities/pipelines/v2_improvers/yara_rules.py‎

Lines changed: 119 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,119 @@
1+
#
2+
# Copyright (c) nexB Inc. and others. All rights reserved.
3+
# VulnerableCode is a trademark of nexB Inc.
4+
# SPDX-License-Identifier: Apache-2.0
5+
# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
6+
# See https://github.com/aboutcode-org/vulnerablecode for support or download.
7+
# See https://aboutcode.org for more information about nexB OSS projects.
8+
#
9+
from pathlib import Path
10+
11+
from aboutcode.pipeline import LoopProgress
12+
from fetchcode.vcs import fetch_via_vcs
13+
14+
from vulnerabilities.models import DetectionRule
15+
from vulnerabilities.models import DetectionRuleTypes
16+
from vulnerabilities.pipelines import VulnerableCodePipeline
17+
18+
19+
class YaraRulesImproverPipeline(VulnerableCodePipeline):
20+
pipeline_id = "yara_rules"
21+
22+
repo_urls = [
23+
"git+https://github.com/elastic/protections-artifacts",
24+
"git+https://github.com/Yara-Rules/rules",
25+
"git+https://github.com/Xumeiquer/yara-forensics",
26+
"git+https://github.com/reversinglabs/reversinglabs-yara-rules",
27+
"git+https://github.com/advanced-threat-research/Yara-Rules",
28+
"git+https://github.com/bartblaze/Yara-rules",
29+
"git+https://github.com/godaddy/yara-rules", # archived
30+
"git+https://github.com/SupportIntelligence/Icewater",
31+
"git+https://github.com/jeFF0Falltrades/YARA-Signatures",
32+
"git+https://github.com/tjnel/yara_repo",
33+
"git+https://github.com/JPCERTCC/jpcert-yara",
34+
"git+https://github.com/mikesxrs/Open-Source-YARA-rules",
35+
"git+https://github.com/fboldewin/YARA-rules",
36+
"git+https://github.com/h3x2b/yara-rules",
37+
]
38+
39+
license_urls = """
40+
https://github.com/elastic/protections-artifacts/blob/main/LICENSE.txt
41+
https://github.com/Yara-Rules/rules/blob/master/LICENSE
42+
https://github.com/Xumeiquer/yara-forensics/blob/master/LICENSE
43+
https://github.com/reversinglabs/reversinglabs-yara-rules/blob/develop/LICENSE
44+
https://github.com/advanced-threat-research/Yara-Rules/blob/master/LICENSE
45+
https://github.com/bartblaze/Yara-rules/blob/master/LICENSE
46+
https://github.com/godaddy/yara-rules/blob/master/LICENSE.md
47+
https://github.com/SupportIntelligence/Icewater/blob/master/LICENSE
48+
https://github.com/jeFF0Falltrades/YARA-Signatures/blob/master/LICENSE.md
49+
https://github.com/tjnel/yara_repo/blob/master/LICENSE
50+
https://github.com/JPCERTCC/jpcert-yara/blob/main/LICENSE
51+
52+
NO-LICENSE: https://github.com/mikesxrs/Open-Source-YARA-rules/
53+
NO-LICENSE: https://github.com/fboldewin/YARA-rules
54+
NO-LICENSE: https://github.com/h3x2b/yara-rules
55+
"""
56+
57+
def __init__(self, *args, **kwargs):
58+
super().__init__(*args, **kwargs)
59+
self.vcs_responses = []
60+
61+
@classmethod
62+
def steps(cls):
63+
return (
64+
cls.clone_repos,
65+
cls.collect_and_store_rules,
66+
cls.clean_downloads,
67+
)
68+
69+
def clone_repos(self):
70+
for url in self.repo_urls:
71+
self.log(f"Cloning `{url}`")
72+
try:
73+
response = fetch_via_vcs(url)
74+
if response:
75+
self.vcs_responses.append((response, url))
76+
except Exception as e:
77+
self.log(f"Failed to clone {url}: {e}")
78+
79+
def collect_and_store_rules(self):
80+
for vcs_response, repo_url in self.vcs_responses:
81+
base_directory = Path(vcs_response.dest_dir)
82+
yara_files = [
83+
p
84+
for p in base_directory.rglob("*")
85+
if p.suffix in (".yar", ".yara") and p.is_file()
86+
]
87+
88+
rules_count = len(yara_files)
89+
self.log(f"Processing {rules_count:,d} rules from {repo_url}")
90+
91+
progress = LoopProgress(total_iterations=rules_count, logger=self.log)
92+
for file_path in progress.iter(yara_files):
93+
if not file_path.exists() or not file_path.is_file():
94+
self.log(
95+
f"Skipping file as it no longer exists or is not a file: {file_path}",
96+
level="warning",
97+
)
98+
continue
99+
100+
raw_text = file_path.read_text(encoding="utf-8", errors="ignore")
101+
if not raw_text:
102+
continue
103+
104+
DetectionRule.objects.update_or_create(
105+
rule_text=raw_text,
106+
rule_type=DetectionRuleTypes.YARA,
107+
advisory=None,
108+
)
109+
110+
def clean_downloads(self):
111+
for vcs_response, _ in self.vcs_responses:
112+
if vcs_response:
113+
self.log(f"Removing cloned repository: {vcs_response.dest_dir}")
114+
vcs_response.delete()
115+
116+
self.vcs_responses = []
117+
118+
def on_failure(self):
119+
self.clean_downloads()

‎vulnerabilities/tests/pipelines/v2_improvers/test_yara.py‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
#
2+
# Copyright (c) nexB Inc. and others. All rights reserved.
3+
# VulnerableCode is a trademark of nexB Inc.
4+
# SPDX-License-Identifier: Apache-2.0
5+
# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
6+
# See https://github.com/aboutcode-org/vulnerablecode for support or download.
7+
# See https://aboutcode.org for more information about nexB OSS projects.
8+
#
9+
10+
from pathlib import Path
11+
from unittest.mock import MagicMock
12+
13+
import pytest
14+
from vulnerabilities.models import DetectionRule
15+
from vulnerabilities.pipelines.v2_improvers.yara_rules import YaraRulesImproverPipeline
16+
17+
BASE_DIR = Path(__file__).resolve().parent
18+
TEST_REPO_DIR = (BASE_DIR / "../../test_data/yara").resolve()
19+
20+
21+
@pytest.mark.django_db
22+
def test_collect_and_store_rules_from_test_repo_dir():
23+
mock_vcs_response = MagicMock()
24+
mock_vcs_response.dest_dir = str(TEST_REPO_DIR)
25+
26+
improver = YaraRulesImproverPipeline()
27+
improver.vcs_responses = [
28+
(mock_vcs_response, "https://github.com/mock/repo")
29+
]
30+
improver.collect_and_store_rules()
31+
32+
assert DetectionRule.objects.exists()
33+
assert DetectionRule.objects.count() == 4

‎vulnerabilities/tests/test_data/yara/Linux_Backdoor_Bash.yar‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
rule Windows_Trojan_Xeno_f92ffb82 {
2+
meta:
3+
author = "Elastic Security"
4+
id = "f92ffb82-b743-4df1-9d6b-2afa3b7bb61f"
5+
fingerprint = "2ae1aebd652afb7da5799f46883205b1f3a5c5b01e975b526640407d9bd0d22c"
6+
creation_date = "2024-10-10"
7+
last_modified = "2024-10-24"
8+
threat_name = "Windows.Trojan.Xeno"
9+
reference_sample = "22dbdbcdd4c8b6899006f9f07e87c19b6a2947eeff8cc89c653309379b388cf4"
10+
severity = 50
11+
arch_context = "x86"
12+
scan_context = "file, memory"
13+
license = "Elastic License v2"
14+
os = "windows"
15+
strings:
16+
$a1 = { 28 00 00 0A 7D 0E 00 00 04 02 7B 0E 00 00 04 28 29 00 00 0A 07 7B 03 00 00 04 02 7B 0E 00 00 04 6F 2A 00 00 0A 3A F2 00 00 00 02 7B 07 00 00 04 02 7B 09 00 00 04 6F 32 00 00 06 6F 2B 00 00 0A }
17+
condition:
18+
all of them
19+
}
20+
21+
rule Windows_Trojan_Xeno_89f9f060 {
22+
meta:
23+
author = "Elastic Security"
24+
id = "89f9f060-afc8-427d-ad36-3672016efdf6"
25+
fingerprint = "ddc5bf8c6d5140cb9ea2fbd9b6f1aaab60f506dcd6161a26961958efa4aa42e1"
26+
creation_date = "2024-10-25"
27+
last_modified = "2024-11-26"
28+
threat_name = "Windows.Trojan.Xeno"
29+
reference_sample = "b74733d68e95220ab0630a68ddf973b0c959fd421628e639c1b91e465ba9299b"
30+
severity = 100
31+
arch_context = "x86"
32+
scan_context = "file, memory"
33+
license = "Elastic License v2"
34+
os = "windows"
35+
strings:
36+
$sc_1 = { 8B 44 24 04 89 C6 FF 56 08 68 00 04 00 00 6A 08 50 FF 16 89 C3 8B 06 89 83 2C 01 00 00 8B 46 04 89 83 30 01 00 00 8B 46 08 89 83 34 01 00 00 8B 46 0C 89 83 38 }
37+
$sc_2 = { 55 48 89 E5 48 83 EC 40 49 89 CC 41 FF 54 24 10 48 89 C1 BA 08 00 00 00 41 B8 00 04 00 00 41 FF 14 24 48 89 C3 49 8B 04 24 48 89 83 90 01 00 00 49 8B 44 24 08 }
38+
$str_1 = "SharpInjector" ascii fullword
39+
$str_2 = "HEAVENSGATE_NON_OPERATIONAL" ascii fullword
40+
$str_3 = "ChromeDecryptor" ascii fullword
41+
$str_4 = "DataExtractionStructs" ascii fullword
42+
$str_5 = "XenoStealer" ascii fullword
43+
condition:
44+
(($sc_1 or $sc_2) and ($str_1 or $str_2)) and (1 of ($str_3, $str_4, $str_5))
45+
}

‎vulnerabilities/tests/test_data/yara/Linux_Backdoor_Python.yar‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
rule Linux_Backdoor_Python_00606bac {
2+
meta:
3+
author = "Elastic Security"
4+
id = "00606bac-83eb-4a58-82d2-e4fd16d30846"
5+
fingerprint = "cce1d0e7395a74c04f15ff95f6de7fd7d5f46ede83322b832df74133912c0b17"
6+
creation_date = "2021-01-12"
7+
last_modified = "2021-09-16"
8+
threat_name = "Linux.Backdoor.Python"
9+
reference_sample = "b3e3728d43535f47a1c15b915c2d29835d9769a9dc69eb1b16e40d5ba1b98460"
10+
severity = 100
11+
arch_context = "x86"
12+
scan_context = "file, memory"
13+
license = "Elastic License v2"
14+
os = "linux"
15+
strings:
16+
$a = { F4 01 83 45 F8 01 8B 45 F8 0F B6 00 84 C0 75 F2 83 45 F8 01 8B }
17+
condition:
18+
all of them
19+
}

‎vulnerabilities/tests/test_data/yara/Linux_Exploit_CVE_2022_0847.yar‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
rule Linux_Exploit_CVE_2022_0847_e831c285 {
2+
meta:
3+
author = "Elastic Security"
4+
id = "e831c285-b2b9-49f3-a87c-3deb806e31e4"
5+
fingerprint = "376b791f9bb5f48d0f41ead4e48b5bcc74cb68002bb7c170760428ace169457e"
6+
creation_date = "2022-03-10"
7+
last_modified = "2022-03-14"
8+
threat_name = "Linux.Exploit.CVE-2022-0847"
9+
reference_sample = "c6b2cef2f2bc04e3ae33e0d368eb39eb5ea38d1bca390df47f7096117c1aecca"
10+
severity = 100
11+
arch_context = "x86"
12+
scan_context = "file, memory"
13+
license = "Elastic License v2"
14+
os = "linux"
15+
strings:
16+
$pp = "prepare_pipe"
17+
$s1 = "splice failed"
18+
$s2 = "short splice"
19+
$s3 = "short write"
20+
$s4 = "hijacking suid binary"
21+
$s5 = "Usage: %s TARGETFILE OFFSET DATA"
22+
$s6 = "Usage: %s SUID"
23+
$bs1 = { B8 00 10 00 00 81 7D EC 00 10 00 00 0F 46 45 EC 89 45 FC 8B 55 FC 48 8B 45 D8 48 83 C0 04 8B 00 48 8D 35 }
24+
$bs2 = { B8 00 10 00 00 81 7D F0 00 10 00 00 0F 46 45 F0 89 45 F8 8B 55 F8 48 8B 45 D8 8B 00 48 }
25+
condition:
26+
($pp and 2 of ($s*)) or (all of ($bs*))
27+
}

‎vulnerabilities/tests/test_data/yara/general.yara‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
// yara rules that can cross boundaries between the various sets/types... more general detection signatures
2+
import "pe"
3+
4+
rule wiper_unique_strings
5+
{
6+
meta:
7+
copyright = "2015 Novetta Solutions"
8+
author = "Novetta Threat Research & Interdiction Group - trig@novetta.com"
9+
company = "novetta"
10+
11+
strings:
12+
$a = "C!@I#%VJSIEOTQWPVz034vuA"
13+
$b = "BAISEO%$2fas9vQsfvx%$"
14+
$c = "1.2.7.f-hanba-win64-v1"
15+
$d = "md %s&copy %s\\*.* %s"
16+
$e = "%sd.e%sc n%ssh%srewa%s ad%s po%sop%sing T%s %d \"%s\""
17+
$f = "Ge.tVol. .umeIn..for mati.onW"
18+
19+
condition:
20+
$a or $b or $c or $d or $e or $f
21+
}
22+
23+
24+
rule wiper_encoded_strings
25+
{
26+
meta:
27+
copyright = "2015 Novetta Solutions"
28+
author = "Novetta Threat Research & Interdiction Group - trig@novetta.com"
29+
company = "novetta"
30+
31+
strings:
32+
$scr = {89 D4 C4 D5 00 00 00}
33+
$explorer = {E2 DF D7 CB C8 D5 C2 D5 89 C2 DF C2 00 00 00 }
34+
$kernel32 = {CC C2 D5 C9 C2 CB 94 95 89 C3 CB CB 00 00 }
35+
36+
condition:
37+
$scr or $explorer or $kernel32
38+
}
39+
40+
41+
rule createP2P
42+
{
43+
meta:
44+
copyright = "2015 Novetta Solutions"
45+
author = "Novetta Threat Research & Interdiction Group - trig@novetta.com"
46+
47+
strings:
48+
$ = "CreatP2P Thread" wide
49+
50+
condition:
51+
any of them
52+
}
53+
54+
rule firewallOpener
55+
{
56+
meta:
57+
copyright = "2015 Novetta Solutions"
58+
author = "Novetta Threat Research & Interdiction Group - trig@novetta.com"
59+
60+
strings:
61+
$ = "%sd.e%sc n%ssh%srewa%s ad%s po%sop%sing T%s %d \"%s\""
62+
63+
condition:
64+
any of them
65+
66+
}
67+

0 commit comments

Comments
 (0)