Skip to content

Commit 89a8812

Browse files
ziadhanyziadhany
committed
Update Clamav to work with the new model
Signed-off-by: ziad hany <ziadhany2016@gmail.com>
1 parent 55fff43 commit 89a8812

6 files changed

Lines changed: 149 additions & 54 deletions

File tree

‎vulnerabilities/improvers/__init__.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@
1919
from vulnerabilities.pipelines import flag_ghost_packages
2020
from vulnerabilities.pipelines import populate_vulnerability_summary_pipeline
2121
from vulnerabilities.pipelines import remove_duplicate_advisories
22-
from vulnerabilities.pipelines.v2_improvers import clamv_rules
22+
from vulnerabilities.pipelines.v2_improvers import clamav_rules
2323
from vulnerabilities.pipelines.v2_improvers import compute_advisory_todo as compute_advisory_todo_v2
2424
from vulnerabilities.pipelines.v2_improvers import compute_package_risk as compute_package_risk_v2
2525
from vulnerabilities.pipelines.v2_improvers import (
@@ -71,6 +71,6 @@
7171
compute_advisory_todo_v2.ComputeToDo,
7272
unfurl_version_range_v2.UnfurlVersionRangePipeline,
7373
compute_advisory_todo.ComputeToDo,
74-
clamv_rules.ClamVRulesImproverPipeline,
74+
clamav_rules.ClamVRulesImproverPipeline,
7575
]
7676
)

‎vulnerabilities/models.py‎

Lines changed: 30 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -3416,29 +3416,43 @@ class Meta:
34163416
unique_together = ("commit_hash", "vcs_url")
34173417

34183418

3419-
class AdvisoryDetectionRule(models.Model):
3420-
""""""
3419+
class DetectionRuleTypes(models.TextChoices):
3420+
"""Defines the supported formats for security detection rules."""
34213421

3422-
RULE_TYPES = [
3423-
("yara", "YARA"),
3424-
("sigma", "Sigma Detection Rule"),
3425-
("clamav", "ClamAV Signature"),
3426-
]
3422+
YARA = "yara", "Yara"
3423+
YARA_X = "yara-x", "Yara-X"
3424+
SIGMA = "sigma", "Sigma"
3425+
CLAMAV = "clamav", "CLAMAV"
3426+
SURICATA = "suricata", "Suricata"
34273427

3428-
advisory = models.ForeignKey(
3429-
AdvisoryV2,
3430-
related_name="detection_rules",
3431-
on_delete=models.SET_NULL,
3428+
3429+
class DetectionRule(models.Model):
3430+
"""
3431+
A Detection Rule is code used to identify malicious activity or security threats.
3432+
"""
3433+
3434+
rule_type = models.CharField(
3435+
max_length=50,
3436+
choices=DetectionRuleTypes.choices,
3437+
help_text="The type of the detection rule content (e.g., YARA, Sigma).",
3438+
)
3439+
3440+
source_url = models.URLField(
3441+
max_length=1024, help_text="URL to the original source or reference for this rule."
3442+
)
3443+
3444+
rule_metadata = models.JSONField(
34323445
null=True,
34333446
blank=True,
3447+
help_text="Additional structured data such as tags, or author information.",
34343448
)
34353449

3436-
rule_text = models.TextField(help_text="Full text of the detection rule, script, or signature.")
3437-
3438-
rule_type = models.CharField(max_length=100, choices=RULE_TYPES, blank=True)
3450+
rule_text = models.TextField(help_text="The content of the detection signature.")
34393451

3440-
source_url = models.URLField(
3452+
advisory = models.ForeignKey(
3453+
AdvisoryV2,
3454+
related_name="detection_rules",
3455+
on_delete=models.SET_NULL,
34413456
null=True,
34423457
blank=True,
3443-
help_text="URL or reference to the source of the rule (vendor feed, GitHub repo, etc.).",
34443458
)

vulnerabilities/pipelines/v2_improvers/clamv_rules.py renamed to vulnerabilities/pipelines/v2_improvers/clamav_rules.py

Lines changed: 32 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,8 @@
1919
import requests
2020

2121
from vulnerabilities.models import AdvisoryAlias
22-
from vulnerabilities.models import AdvisoryDetectionRule
22+
from vulnerabilities.models import DetectionRule
23+
from vulnerabilities.models import DetectionRuleTypes
2324
from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipelineV2
2425
from vulnerabilities.utils import find_all_cve
2526

@@ -107,7 +108,7 @@ class ClamVRulesImproverPipeline(VulnerableCodeBaseImporterPipelineV2):
107108

108109
pipeline_id = "clamv_rules"
109110
MAIN_DATABASE_URL = "https://database.clamav.net/main.cvd"
110-
license_url = ""
111+
license_url = "https://github.com/Cisco-Talos/clamav/blob/c73755d3fc130b0c60ccf4e8f8d28c62fc58c95b/README.md#licensing"
111112
license_expression = "GNU GENERAL PUBLIC LICENSE"
112113

113114
@classmethod
@@ -153,37 +154,41 @@ def extract_database(self):
153154

154155
def collect_and_store_advisories(self):
155156
"""Parse .ndb and .hdb files and store rules in the DB."""
156-
rules = {}
157-
for entry in parse_hdb_file(self.extract_cvd_dir / "main.hdb") + parse_ndb_file(
157+
158+
for rule_entry in parse_hdb_file(self.extract_cvd_dir / "main.hdb") + parse_ndb_file(
158159
self.extract_cvd_dir / "main.ndb"
159160
):
160-
name = entry.get("name", "")
161-
cve = extract_cve_id(name)
162-
if cve:
163-
rules[cve] = entry
164-
165-
rules_added = 0
166-
for cve_id, rule_text in rules.items():
167-
advisories = set()
168-
try:
169-
if alias := AdvisoryAlias.objects.get(alias=cve_id):
170-
for adv in alias.advisories.all():
171-
advisories.add(adv)
172-
except AdvisoryAlias.DoesNotExist:
173-
self.log(f"Advisory {cve_id} not found.")
174-
continue
175-
176-
for advisory in advisories:
177-
AdvisoryDetectionRule.objects.update_or_create(
178-
advisory=advisory,
179-
rule_type="clamav",
161+
name = rule_entry.get("name", "")
162+
cve_id = extract_cve_id(name)
163+
found_advisories = set()
164+
165+
if cve_id:
166+
try:
167+
if alias := AdvisoryAlias.objects.get(alias=cve_id):
168+
for adv in alias.advisories.all():
169+
found_advisories.add(adv)
170+
except AdvisoryAlias.DoesNotExist:
171+
self.log(f"Advisory {cve_id} not found.")
172+
173+
for adv in found_advisories:
174+
DetectionRule.objects.update_or_create(
175+
rule_text=str(rule_entry),
176+
rule_type=DetectionRuleTypes.CLAMAV,
177+
advisory=adv,
180178
defaults={
181-
"rule_text": str(rule_text),
179+
"source_url": self.MAIN_DATABASE_URL,
182180
},
183181
)
184182

185-
rules_added += 1
186-
self.log(f"Successfully added/updated {rules_added} rules for advisories.")
183+
if not found_advisories:
184+
DetectionRule.objects.update_or_create(
185+
rule_text=str(rule_entry),
186+
rule_type=DetectionRuleTypes.CLAMAV,
187+
advisory=None,
188+
defaults={
189+
"source_url": self.MAIN_DATABASE_URL,
190+
},
191+
)
187192

188193
def clean_downloads(self):
189194
"""Clean up downloaded files."""

‎vulnerabilities/tests/pipelines/v2_improvers/test_clamv_rules.py‎

Lines changed: 85 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -15,18 +15,18 @@
1515
import pytest
1616

1717
from vulnerabilities.models import AdvisoryAlias
18-
from vulnerabilities.models import AdvisoryDetectionRule
1918
from vulnerabilities.models import AdvisoryV2
20-
from vulnerabilities.pipelines.v2_improvers.clamv_rules import ClamVRulesImproverPipeline
19+
from vulnerabilities.models import DetectionRule
20+
from vulnerabilities.pipelines.v2_improvers.clamav_rules import ClamVRulesImproverPipeline
2121

2222
BASE_DIR = Path(__file__).resolve().parent
23-
TEST_REPO_DIR = (BASE_DIR / "../../test_data/clamv").resolve()
23+
TEST_REPO_DIR = (BASE_DIR / "../../test_data/clamav").resolve()
2424

2525

2626
@pytest.mark.django_db
27-
@mock.patch("vulnerabilities.pipelines.v2_improvers.clamv_rules.extract_cvd")
28-
@mock.patch("vulnerabilities.pipelines.v2_improvers.clamv_rules.requests.get")
29-
def test_clamv_rules_db_improver(mock_requests_get, mock_extract_cvd):
27+
@mock.patch("vulnerabilities.pipelines.v2_improvers.clamav_rules.extract_cvd")
28+
@mock.patch("vulnerabilities.pipelines.v2_improvers.clamav_rules.requests.get")
29+
def test_clamav_rules_db_improver(mock_requests_get, mock_extract_cvd):
3030
mock_resp = MagicMock()
3131
mock_resp.iter_content.return_value = [b"fake data"]
3232
mock_resp.raise_for_status.return_value = None
@@ -70,6 +70,82 @@ def test_clamv_rules_db_improver(mock_requests_get, mock_extract_cvd):
7070
improver = ClamVRulesImproverPipeline()
7171
improver.execute()
7272

73-
assert AdvisoryDetectionRule.objects.count() == 3
74-
first_rule = AdvisoryDetectionRule.objects.first()
75-
assert first_rule.rule_type == "clamav"
73+
assert DetectionRule.objects.count() == 14
74+
assert DetectionRule.objects.get(advisory=adv1)
75+
assert DetectionRule.objects.get(advisory=adv2)
76+
assert DetectionRule.objects.get(advisory=adv3)
77+
assert [
78+
(detection_rule.rule_type, detection_rule.rule_text, detection_rule.source_url)
79+
for detection_rule in DetectionRule.objects.all()
80+
] == [
81+
(
82+
"clamav",
83+
"{'hash': 'af9a2ce339b3a314cd8ce31f4e2489a5', 'file_size': '149420', 'name': 'Archive.Malware.Agent-7116646-0', 'line_num': 1}",
84+
"https://database.clamav.net/main.cvd",
85+
),
86+
(
87+
"clamav",
88+
"{'hash': 'ab51de8588946f1332d53dd53bac8056', 'file_size': '48580', 'name': 'Html.Malware.Agent-7116647-0', 'line_num': 2}",
89+
"https://database.clamav.net/main.cvd",
90+
),
91+
(
92+
"clamav",
93+
"{'hash': '3f70569ac131833698c3d1c20e0123ca', 'file_size': '676', 'name': 'Html.Malware.Agent-7116648-0', 'line_num': 3}",
94+
"https://database.clamav.net/main.cvd",
95+
),
96+
(
97+
"clamav",
98+
"{'hash': 'df6634d021a6df4d17f005e507beac88', 'file_size': '6268', 'name': 'Win.Exploit.CVE_2019_1199-7116649-2', 'line_num': 4}",
99+
"https://database.clamav.net/main.cvd",
100+
),
101+
(
102+
"clamav",
103+
"{'hash': '27ebcd8c72e6e3c7f4a64dc68b95dd8a', 'file_size': '173248', 'name': 'Html.Malware.Agent-7116650-0', 'line_num': 5}",
104+
"https://database.clamav.net/main.cvd",
105+
),
106+
(
107+
"clamav",
108+
"{'hash': '8745d432f7027e65178e92b2239bef25', 'file_size': '384634', 'name': 'Archive.Malware.Agent-7116651-0', 'line_num': 6}",
109+
"https://database.clamav.net/main.cvd",
110+
),
111+
(
112+
"clamav",
113+
"{'hash': '63d1a25066c121253febc907850b1852', 'file_size': '50185', 'name': 'Html.Malware.Agent-7116652-0', 'line_num': 7}",
114+
"https://database.clamav.net/main.cvd",
115+
),
116+
(
117+
"clamav",
118+
"{'hash': '92233ed6889cd0ba7bf632e3f45fc950', 'file_size': '97134', 'name': 'Html.Malware.Agent-7116653-0', 'line_num': 8}",
119+
"https://database.clamav.net/main.cvd",
120+
),
121+
(
122+
"clamav",
123+
"{'name': 'Win.Exploit.CVE_2020_0720-7578647-1', 'target_type': '1', 'offset': '*', 'hex_signature': '240C1400000068E8214000660F1344241C897C2414C744241805000000E80EFEFFFF83C4048D44240C50FF1544204000', 'line_num': 1}",
124+
"https://database.clamav.net/main.cvd",
125+
),
126+
(
127+
"clamav",
128+
"{'name': 'Win.Exploit.CVE_2020_0731-7583553-0', 'target_type': '1', 'offset': '*', 'hex_signature': '83C4088B55F0526AF48B45FC50FF15D4C146008945E46A006A006A108B4D', 'line_num': 2}",
129+
"https://database.clamav.net/main.cvd",
130+
),
131+
(
132+
"clamav",
133+
"{'name': 'Win.Exploit.CVE_2020_0722-7583689-1', 'target_type': '1', 'offset': '*', 'hex_signature': '488B555033C9FF15A1F100004889057AB10000488B0D73B10000FF1575F10000EB86', 'line_num': 3}",
134+
"https://database.clamav.net/main.cvd",
135+
),
136+
(
137+
"clamav",
138+
"{'name': 'Win.Ransomware.MailTo-7586723-0', 'target_type': '1', 'offset': '*', 'hex_signature': '496e746572666163345c7b62313936623238372d626162342d313031612d623639632d3030616130303334316430377d*4c616d616e74696e652e537469636b7950617373776f7264', 'line_num': 4}",
139+
"https://database.clamav.net/main.cvd",
140+
),
141+
(
142+
"clamav",
143+
"{'name': 'Win.Trojan.Emotet-7587729-1', 'target_type': '1', 'offset': '*', 'hex_signature': '565053e801000000cc5889c3402d00e016002dacb00b1005a3b00b10803bcc7519c60300bb00100000682ece177a680f9067565350e80a00000083c000894424085b58c35589e5505351568b75088b4d0cc1e9028b45108b5d1485c9740a3106011e83c60449ebf25e595b58c9c21000', 'line_num': 5}",
144+
"https://database.clamav.net/main.cvd",
145+
),
146+
(
147+
"clamav",
148+
"{'name': 'Win.Trojan.Hoplight-7587747-0', 'target_type': '1', 'offset': '*', 'hex_signature': '4e6574776f726b20554450205472616365204d616e6167656d656e742053657276696365*6d646e6574757365*554450547263537663', 'line_num': 6}",
149+
"https://database.clamav.net/main.cvd",
150+
),
151+
]

vulnerabilities/tests/test_data/clamv/main.hdb renamed to vulnerabilities/tests/test_data/clamav/main.hdb

File renamed without changes.

vulnerabilities/tests/test_data/clamv/main.ndb renamed to vulnerabilities/tests/test_data/clamav/main.ndb

File renamed without changes.

0 commit comments

Comments
 (0)