Avoid blocking self and other slips peers doing arp attacks

.github/workflows/unit-tests.yml

@@ -37,6 +37,7 @@ jobs:
           - test_whitelist.py
           - test_arp.py
           - test_arp_poisoner.py
+          - test_arp_filter.py
           - test_blocking.py
           - test_unblocker.py
           - test_flow_handler.py

.pre-commit-config.yaml

@@ -43,3 +43,11 @@ repos:
         - id: yamllint
           args: ["-d", "{rules: {line-length: {max: 100}}}"]
           files: "slips.yaml"
+
+- repo: local
+  hooks:
+    - id: vulture
+      name: vulture dead code check
+      entry: bash -c 'files=$(git diff --cached --name-only --diff-filter=ACM | grep -E "\.py$" | grep -vE "^(tests/|migrations/)"); [ -n "$files" ] && vulture --exclude "tests/*,venv/*" $files || true'
+      language: system
+      types: [python]

conftest.py

@@ -67,14 +67,6 @@ def profiler_queue():
     profiler_queue.put = do_nothing
     return profiler_queue
 
-
-@pytest.fixture
-def database():
-    db = DBManager(Output(), "output/", 6379)
-    db.print = do_nothing
-    return db
-
-
 @pytest.fixture
 def flow():
     """returns a dummy flow for testing"""

docker/Dockerfile

@@ -59,6 +59,7 @@ RUN apt update && apt install -y --no-install-recommends \
     nano \
     tree \
     tmux \
+    arp-scan \
     && echo 'deb http://download.opensuse.org/repositories/security:/zeek/xUbuntu_22.04/ /' |  tee /etc/apt/sources.list.d/security:zeek.list \
     && curl -fsSL https://download.opensuse.org/repositories/security:zeek/xUbuntu_22.04/Release.key | gpg --dearmor |  tee /etc/apt/trusted.gpg.d/security_zeek.gpg > /dev/null \
     && apt update \

docs/create_new_module.md

@@ -365,7 +365,8 @@ import json
 
 from slips_files.common.flow_classifier import FlowClassifier
 from slips_files.core.structures.evidence import
-    (
+
+(
     Evidence,
     ProfileID,
     TimeWindow,
@@ -378,7 +379,7 @@ from slips_files.core.structures.evidence import
     )
 from slips_files.common.parsers.config_parser import ConfigParser
 from slips_files.common.slips_utils import utils
-from slips_files.common.abstracts.module import IModule
+from slips_files.common.abstracts.imodule import IModule
 
 
 class LocalConnectionDetector(

install/requirements.txt

@@ -41,4 +41,5 @@ netifaces==0.11.0
 scapy==2.6.1
 pyyaml
 pytest-asyncio
+vulture
 git+https://github.com/SECEF/python-idmefv2.git

managers/process_manager.py

@@ -33,7 +33,7 @@
 import modules
 from modules.update_manager.update_manager import UpdateManager
 from slips_files.common.slips_utils import utils
-from slips_files.common.abstracts.module import (
+from slips_files.common.abstracts.imodule import (
     IModule,
 )
 
@@ -106,6 +106,7 @@ def start_profiler_process(self):
             self.main.redis_port,
             self.termination_event,
             self.main.args,
+            self.main.conf,
             is_profiler_done=self.is_profiler_done,
             profiler_queue=self.profiler_queue,
             is_profiler_done_event=self.is_profiler_done_event,
@@ -127,6 +128,7 @@ def start_evidence_process(self):
             self.main.redis_port,
             self.evidence_handler_termination_event,
             self.main.args,
+            self.main.conf,
         )
         evidence_process.start()
         self.main.print(
@@ -145,6 +147,7 @@ def start_input_process(self):
             self.main.redis_port,
             self.termination_event,
             self.main.args,
+            self.main.conf,
             is_input_done=self.is_input_done,
             profiler_queue=self.profiler_queue,
             input_type=self.main.input_type,
@@ -389,6 +392,7 @@ def load_modules(self):
                 self.main.redis_port,
                 self.termination_event,
                 self.main.args,
+                self.main.conf,
             )
             module.start()
             self.main.db.store_pid(module_name, int(module.pid))
@@ -444,6 +448,7 @@ def start_update_manager(self, local_files=False, ti_feeds=False):
                     self.main.redis_port,
                     multiprocessing.Event(),
                     self.main.args,
+                    self.main.conf,
                 )
 
                 if local_files:
@@ -821,7 +826,7 @@ def shutdown_gracefully(self):
             self.main.profilers_manager.cpu_profiler_release()
             self.main.profilers_manager.memory_profiler_release()
 
-            self.main.db.close_redis_and_sqlite()
+            self.main.db.close_all_dbs()
             if graceful_shutdown:
                 print(
                     "[Process Manager] Slips shutdown gracefully\n",

modules/arp/arp.py

@@ -8,10 +8,11 @@
 from multiprocessing import Queue
 from typing import List
 
+from modules.arp.filter import ARPEvidenceFilter
 from slips_files.common.flow_classifier import FlowClassifier
 from slips_files.common.parsers.config_parser import ConfigParser
 from slips_files.common.slips_utils import utils
-from slips_files.common.abstracts.module import IModule
+from slips_files.common.abstracts.imodule import IModule
 from slips_files.core.structures.evidence import (
     Evidence,
     ProfileID,
@@ -26,7 +27,7 @@
 
 
 class ARP(IModule):
-    # Name: short name of the module. Do not use spaces
+
     name = "ARP"
     description = "Detect ARP attacks"
     authors = ["Alya Gomaa"]
@@ -64,6 +65,7 @@ def init(self):
         # wait 10s for mmore arp scan evidence to come
         self.time_to_wait = 10
         self.is_zeek_running: bool = self.is_running_zeek()
+        self.evidence_filter = ARPEvidenceFilter(self.conf, self.args, self.db)
 
     def read_configuration(self):
         conf = ConfigParser()
@@ -247,7 +249,7 @@ def set_evidence_arp_scan(self, ts, profileid, twid, uids: List[str]):
             timestamp=ts,
         )
 
-        self.db.set_evidence(evidence)
+        self.set_evidence(evidence)
         # after we set evidence, clear the dict so we can detect if it
         # does another scan
         try:
@@ -315,7 +317,7 @@ def check_dstip_outside_localnet(self, twid, flow):
                 timestamp=flow.starttime,
                 victim=victim,
             )
-            self.db.set_evidence(evidence)
+            self.set_evidence(evidence)
             return True
 
         return False
@@ -357,7 +359,7 @@ def detect_unsolicited_arp(self, twid: str, flow):
                 timestamp=flow.starttime,
             )
 
-            self.db.set_evidence(evidence)
+            self.set_evidence(evidence)
             return True
 
     def detect_mitm_arp_attack(self, twid: str, flow):
@@ -460,7 +462,7 @@ def detect_mitm_arp_attack(self, twid: str, flow):
                 victim=victim,
             )
 
-            self.db.set_evidence(evidence)
+            self.set_evidence(evidence)
             return True
 
     def check_if_gratutitous_arp(self, flow):
@@ -514,6 +516,13 @@ def clear_arp_logfile(self):
             # update ts of the new arp.log
             self.arp_log_creation_time = time.time()
 
+    def set_evidence(self, evidence: Evidence):
+        """the goal of this function is to discard evidence of other slips
+        peers doing arp scans because that's slips attacking back attackers"""
+        if self.evidence_filter.should_discard_evidence(evidence.profile.ip):
+            return
+        self.db.set_evidence(evidence)
+
     def pre_main(self):
         """runs once before the main() is executed in a loop"""
         utils.drop_root_privs()

modules/arp/filter.py

@@ -0,0 +1,58 @@
+from typing import List
+
+from slips_files.common.slips_utils import utils
+from slips_files.core.database.database_manager import DBManager
+
+
+class ARPEvidenceFilter:
+    """
+    A class to filter ARP evidence coming from a peer slips.
+    Slips uses arp poisoning, arp spoofing, and arp scans to discover
+    attackers and isolate them from the network, we don't want this
+    instance of Slips to block other Slips instances, so we discard
+    evidence about other slips attacking.
+    """
+
+    def __init__(self, conf, slips_args, db: DBManager):
+        self.db = db
+        self.conf = conf
+        self.args = slips_args
+        # p2p needs to be enabled for slips to be able to recognize slips peers
+        self.p2p_enabled = False
+        if self.conf.use_local_p2p():
+            self.p2p_enabled = True
+        self.our_ips: List[str] = utils.get_own_ips(ret="List")
+
+    def should_discard_evidence(self, ip: str) -> bool:
+        return self.is_slips_peer(ip) or self.is_self_defense(ip)
+
+    def is_self_defense(self, ip: str):
+        """
+        slips uses arp poison to defend itself and th enetwork,
+        check arp_poison.py for more details.
+        goal of this function is to discard evidence about slips doing arp
+        attacks when it's just attacking attackers
+        """
+        loaded_modules = self.db.get_pids().keys()
+        return (
+            ip in self.our_ips
+            and self.args.blocking
+            and "ARP Poisoner" in loaded_modules
+        )
+
+    def is_slips_peer(self, ip: str) -> bool:
+        """
+        Check if the given IP address is a trusted Slips peer.
+        Trust here is defined from the p2p network (trust model).
+        Only works if the local p2p is enabled.
+
+        :param ip: The IP address to check.
+        """
+        if not self.p2p_enabled or not utils.is_private_ip(ip):
+            return False
+
+        trust = self.db.get_peer_trust(ip)
+        if not trust:
+            return False
+
+        return trust >= 0.3

modules/arp_poisoner/arp_poisoner.py

@@ -12,7 +12,7 @@
 from scapy.all import ARP, Ether
 from scapy.sendrecv import sendp, srp
 
-from slips_files.common.abstracts.module import IModule
+from slips_files.common.abstracts.imodule import IModule
 from modules.arp_poisoner.unblocker import ARPUnblocker
 from slips_files.common.slips_utils import utils
 
@@ -195,6 +195,7 @@ def _arp_poison(self, target_ip: str, first_time=False):
         repoisoning every x seconds.
         """
         fake_mac = "aa:aa:aa:aa:aa:aa"
+
         # it makes sense here to get the mac using cache, because if we
         # reached this function, means there's an alert, means slips seen
         # traffic from that target_ip and has itsmac in the arp cache.

modules/arp_poisoner/unblocker.py

@@ -1,7 +1,7 @@
 from threading import Lock
 import time
 from typing import Callable, Optional
-from slips_files.common.abstracts.unblocker import IUnblocker
+from slips_files.common.abstracts.iunblocker import IUnblocker
 from slips_files.common.printer import Printer
 from slips_files.common.slips_utils import utils
 from slips_files.core.structures.evidence import TimeWindow

modules/blocking/blocking.py

@@ -10,7 +10,7 @@
 import time
 from threading import Lock
 
-from slips_files.common.abstracts.module import IModule
+from slips_files.common.abstracts.imodule import IModule
 from slips_files.common.slips_utils import utils
 from .exec_iptables_cmd import exec_iptables_command
 from modules.blocking.unblocker import Unblocker

modules/blocking/unblocker.py

@@ -2,7 +2,7 @@
 import time
 import threading
 from typing import Dict, Callable
-from slips_files.common.abstracts.unblocker import IUnblocker
+from slips_files.common.abstracts.iunblocker import IUnblocker
 from slips_files.common.printer import Printer
 from slips_files.common.slips_utils import utils
 from slips_files.core.structures.evidence import TimeWindow

modules/cesnet/cesnet.py

@@ -9,7 +9,7 @@
 import validators
 
 from slips_files.common.parsers.config_parser import ConfigParser
-from slips_files.common.abstracts.module import IModule
+from slips_files.common.abstracts.imodule import IModule
 from slips_files.core.structures.evidence import (
     ThreatLevel,
     Evidence,

modules/cyst/cyst.py

@@ -1,6 +1,6 @@
 # SPDX-FileCopyrightText: 2021 Sebastian Garcia <sebastian.garcia@agents.fel.cvut.cz>
 # SPDX-License-Identifier: GPL-2.0-only
-from slips_files.common.abstracts.module import IModule
+from slips_files.common.abstracts.imodule import IModule
 import socket
 import json
 import os

modules/exporting_alerts/exporting_alerts.py

@@ -5,7 +5,7 @@
 from modules.exporting_alerts.slack_exporter import SlackExporter
 from modules.exporting_alerts.stix_exporter import StixExporter
 from slips_files.common.slips_utils import utils
-from slips_files.common.abstracts.module import IModule
+from slips_files.common.abstracts.imodule import IModule
 
 
 class ExportingAlerts(IModule):

modules/exporting_alerts/slack_exporter.py

@@ -3,7 +3,7 @@
 from slack import WebClient
 from slack.errors import SlackApiError
 from slips_files.common.slips_utils import utils
-from slips_files.common.abstracts.exporter import IExporter
+from slips_files.common.abstracts.iexporter import IExporter
 from slips_files.common.parsers.config_parser import ConfigParser
 
 

modules/exporting_alerts/stix_exporter.py

@@ -6,7 +6,7 @@
 import threading
 import os
 
-from slips_files.common.abstracts.exporter import IExporter
+from slips_files.common.abstracts.iexporter import IExporter
 from slips_files.common.parsers.config_parser import ConfigParser
 from slips_files.common.slips_utils import utils
 

modules/fidesModule/fidesModule.py

@@ -1,22 +1,20 @@
 import os
 import json
-from dataclasses import asdict
 from pathlib import Path
 
 from slips_files.common.slips_utils import utils
-from slips_files.common.abstracts.module import IModule
+from slips_files.common.abstracts.imodule import IModule
 from slips_files.common.parsers.config_parser import (
     ConfigParser,
 )
 from slips_files.core.structures.alerts import (
     dict_to_alert,
     Alert,
 )
-from .messaging.model import NetworkMessage
 from ..fidesModule.messaging.message_handler import MessageHandler
 from ..fidesModule.messaging.network_bridge import NetworkBridge
 from ..fidesModule.model.configuration import load_configuration
-from ..fidesModule.model.threat_intelligence import SlipsThreatIntelligence, ThreatIntelligence
+from ..fidesModule.model.threat_intelligence import SlipsThreatIntelligence
 from ..fidesModule.protocols.alert import AlertProtocol
 from ..fidesModule.protocols.initial_trusl import InitialTrustProtocol
 from ..fidesModule.protocols.opinion import OpinionAggregator
@@ -26,15 +24,13 @@
     ThreatIntelligenceProtocol,
 )
 from ..fidesModule.utils.logger import LoggerPrintCallbacks
-from ..fidesModule.messaging.redis_simplex_queue import RedisSimplexQueue, RedisDuplexQueue
+from ..fidesModule.messaging.redis_simplex_queue import RedisSimplexQueue
 from ..fidesModule.persistence.threat_intelligence_db import (
     SlipsThreatIntelligenceDatabase,
 )
 from ..fidesModule.persistence.trust_db import SlipsTrustDatabase
 from ..fidesModule.persistence.sqlite_db import SQLiteDB
 
-from ..fidesModule.model.alert import Alert as FidesAlert
-
 
 class FidesModule(IModule):
     """