Small bug fixes

dataset/port-scans/horizontal/conn.log

@@ -47,3 +47,4 @@
 960.869113	CyxM402uQN5ldQ2Td3	10.0.2.112	49248	62.193.227.35	80	tcp	-	-	-	-	S0	-	-	0	S	1	52	0	0	(empty)
 958.869114	CeSj4A1BN9nBer0fe9	10.0.2.112	49245	188.252.28.130	80	tcp	-	-	-	-	S0	-	-	0	S	1	52	0	0	(empty)
 958.869115	CeSj4A1BN9nBei0fe9	10.0.2.112	49245	188.252.30.130	80	tcp	-	-	-	-	S0	-	-	0	S	1	52	0	0	(empty)
+#close	2015-11-18-16-36-13

dataset/test18-malicious-ctu-sme-11-win/zeek-labeled/conn.log.labeled

@@ -772,3 +772,4 @@
 1677024002.966990	CWuwGW3u99uhiJZAek	192.168.1.107	49323	192.168.1.135	8009	tcp	-	496.012311	11000	11000	OTH	T	T	0	DTdtATtTt	400	38000	200	30000	-	Benign	From_benign-To_benign-Device_chromecast_tv_assistant
 1677024033.849842	CfyNXH2BVoPq6CZkWe	192.168.1.107	49752	77.88.55.55	443	tcp	-	450.657534	1	0	OTH	T	F	0	DTaT	22	902	22	1144	-	Unknown	(empty)
 1677024005.317605	Csmqcc1SQBkWaMDiah	192.168.1.107	64746	162.159.136.234	443	tcp	-	495.305609	572	11677	OTH	T	F	0	DTadtAtT	76	4184	84	26714	-	Benign	From_benign-To_benign-Application_discord
+#close	2024-08-16-09-45-01

modules/exporting_alerts/stix_exporter.py

@@ -649,13 +649,19 @@ def _build_custom_properties(
         if src_port:
             custom_properties["x_slips_src_port"] = src_port
 
+        immune_type = evidence.get("immune_type")
+        if immune_type:
+            custom_properties["x_slips_immune_type"] = src_port
+
         return {
             key: value
             for key, value in custom_properties.items()
             if value not in (None, "", [], {})
         }
 
     def _build_indicator(self, evidence: dict):
+        """this is where a single evidence is converted from slips format
+        to STIX format"""
         attacker = (evidence.get("attacker") or {}).get("value")
         if not attacker:
             attacker = (evidence.get("profile") or {}).get("ip")

modules/flow_alerts/conn.py

@@ -192,6 +192,10 @@ def check_unknown_port(self, profileid, twid, flow):
         if not flow.dport:
             return
 
+        if "unknown_transport" in flow.proto:
+            # the protocol is unrecognized by zeek. not tcp or udp.
+            return
+
         if self.db.is_a_port_scanner(flow.saddr, twid):
             # avoid setting unknown port evidence for each port
             # scanned from this attacker

modules/http_analyzer/http_analyzer.py

@@ -372,6 +372,10 @@ def check_multiple_user_agents_in_a_row(
                 return False
 
         ua: str = cached_ua.get("user_agent", "")
+        if ua == flow.user_agent:
+            # same UA is used again, nothing to alert about here
+            return False
+
         self.set_evidence.multiple_user_agents_in_a_row(flow, ua, twid)
         return True
 

modules/ip_info/ip_info.py

@@ -88,7 +88,7 @@ def subscribe_to_channels(self):
             "check_jarm_hash": self.c4,
         }
 
-    async def open_dbs(self):
+    def open_dbs(self) -> None:
         """Function to open the different offline databases used in this
         module. ASN, Country etc.."""
         # Open the maxminddb ASN offline db
@@ -116,7 +116,18 @@ async def open_dbs(self):
                 "https://dev.maxmind.com/geoip/geolite2-free-geolocation-data?lang=en. "
                 "Please note it must be the MaxMind DB version."
             )
-        self.create_task(self.read_mac_db)
+        self._start_mac_db_reader()
+
+    def _start_mac_db_reader(self) -> None:
+        """
+        Schedule the MAC vendor database reader on the active event loop.
+        """
+        try:
+            asyncio.get_running_loop()
+        except RuntimeError:
+            return
+
+        self.reading_mac_db_task = self.create_task(self.read_mac_db)
 
     async def read_mac_db(self):
         """
@@ -496,7 +507,6 @@ def get_gateway_ip_if_interface(self) -> Dict[str, str] | None:
             return
 
         interfaces: List[str] = utils.get_all_interfaces(self.args)
-
         gw_ips = {}
         for interface in interfaces:
             try:
@@ -518,6 +528,9 @@ def get_own_mac() -> str:
 
     def _get_wifi_interface_if_ap(self) -> str | None:
         ap_interfaces: str = self.db.get_wifi_interface()
+        if not ap_interfaces:
+            return None
+
         try:
             # we're now sure that we're running in AP mode
             wifi_interface = ap_interfaces["wifi_interface"]
@@ -733,18 +746,12 @@ def register_private_dns_server(self, flow: Any) -> bool:
         )
         return True
 
-    def wait_for_dbs(self):
+    def wait_for_dbs(self) -> None:
         """
         wait for update manager to finish updating the mac db and open the
         rest of dbs before starting this module
         """
-        # this is the loop that controls tasks running on open_dbs
-        loop = asyncio.new_event_loop()
-        asyncio.set_event_loop(loop)
-        # run open_dbs in the background so we don't have
-        # to wait for update manager to finish updating the mac db to start
-        # this module
-        loop.run_until_complete(self.open_dbs())
+        self.open_dbs()
 
     def set_evidence_malicious_jarm_hash(
         self,
@@ -808,7 +815,7 @@ def set_evidence_malicious_jarm_hash(
 
         self.db.set_evidence(evidence)
 
-    def pre_main(self):
+    def pre_main(self) -> None:
         utils.drop_root_privs_permanently()
         self.wait_for_dbs()
         # the following method only works when running on an interface

slips/main.py

@@ -428,6 +428,7 @@ def get_analyzed_flows_percentage(self) -> str:
         else:
             percentage = int(percentage)
 
+        percentage = min(100, percentage)  # cap at 100%
         return f"Analyzed Flows: {green(percentage)}{green('%')}. "
 
     def is_total_flows_unknown(self) -> bool:

slips_files/common/abstracts/iasync_module.py

@@ -4,6 +4,7 @@
 import traceback
 from asyncio import Task
 from typing import (
+    Any,
     Callable,
     List,
 )
@@ -49,11 +50,34 @@ def _remove_completed_task(self, task: asyncio.Task):
             # Task already removed or not tracked.
             pass
 
-    def handle_task_exception(self, task: asyncio.Task):
+    @staticmethod
+    def is_shutdown_exception(exception: BaseException | None) -> bool:
+        """
+        Check whether an exception is part of normal process shutdown.
+
+        :param exception: Exception raised while a task was running.
+        :return: True when the exception should not be logged as a task error.
+        """
+        shutdown_exceptions = (
+            KeyboardInterrupt,
+            SystemExit,
+            asyncio.CancelledError,
+        )
+        return isinstance(exception, shutdown_exceptions)
+
+    def handle_task_exception(self, task: asyncio.Task) -> None:
+        """
+        Log task exceptions while suppressing normal shutdown exceptions.
+
+        :param task: Completed asyncio task to inspect.
+        :return: None.
+        """
         try:
             exception = task.exception()
         except asyncio.CancelledError:
-            return  # Task was cancelled, not an error
+            return  # Task was cancelled, not an error.
+        if self.is_shutdown_exception(exception):
+            return
         if exception:
             self.print(f"Unhandled exception in task: {exception!r} .. ")
             self.print_traceback_from_exception(exception, task)
@@ -111,17 +135,23 @@ def run_async_function(self, func: Callable):
         loop.set_exception_handler(self.handle_loop_exception)
         return loop.run_until_complete(func())
 
-    def handle_loop_exception(self, loop, context):
+    def handle_loop_exception(
+        self, loop: asyncio.AbstractEventLoop, context: dict[str, Any]
+    ) -> None:
         """A common loop exception handler"""
         exception = context.get("exception")
         future = context.get("future")
 
         if future:
             try:
                 future.result()
-            except Exception:
+            except BaseException as error:
+                if self.is_shutdown_exception(error):
+                    return
                 self.print_traceback()
-        elif exception:
+        elif isinstance(exception, BaseException):
+            if self.is_shutdown_exception(exception):
+                return
             self.print(f"Unhandled loop exception: {exception}")
         else:
             self.print(f"Unhandled loop error: {context.get('message')}")

slips_files/common/idmefv2.py

@@ -42,7 +42,8 @@ class IDMEFv2:
     Class to convert Slips evidence and alerts to
     The Incident Detection Message Exchange Format version 2 (IDMEFv2 format).
     More Details about it here:
-    https://www.ietf.org/id/draft-lehmann-idmefv2-03.html#name-the-alert-class
+    https://datatracker.ietf.org/doc/draft-lehmann-idmefv2/#:~:text=Site%22%0A%20%20%20%20%20%20%20%7D%0A%20%20%20%20%20%5D%0A%20%20%20%7D%0A%0AAppendix%20C.-,JSON%20Validation%20Schema%20(Non%2Dnormative),-Listing%205%20contains
+
     """
 
     name = "idmefv2"
@@ -187,7 +188,7 @@ def convert_to_idmef_event(self, evidence: Evidence) -> Message:
         The Incident Detection Message Exchange Format version 2
         (IDMEFv2 format).
         More Details about it here:
-        https://www.ietf.org/id/draft-lehmann-idmefv2-03.html#name-the-alert-class
+        https://datatracker.ietf.org/doc/draft-lehmann-idmefv2/#:~:text=Site%22%0A%20%20%20%20%20%20%20%7D%0A%20%20%20%20%20%5D%0A%20%20%20%7D%0A%0AAppendix%20C.-,JSON%20Validation%20Schema%20(Non%2Dnormative),-Listing%205%20contains
         """
         try:
             now = datetime.now(utils.local_tz).isoformat("T")

slips_files/common/slips_utils.py

@@ -294,7 +294,7 @@ def get_gateway_for_iface(self, iface: str) -> Optional[str]:
         """returns the default gateway for the given interface"""
         gws = netifaces.gateways()
         for family in (netifaces.AF_INET, netifaces.AF_INET6):
-            if "default" in gws and gws["default"][family]:
+            if "default" in gws and family in gws["default"]:
                 gw, gw_iface = gws["default"][family]
                 if gw_iface == iface:
                     return gw

slips_files/core/database/redis_db/database.py

@@ -31,6 +31,7 @@
 
 import os
 import redis
+import shlex
 import time
 import json
 import subprocess
@@ -62,6 +63,7 @@ class RedisDB(
     ScanDetectionsHandler,
     Publisher,
 ):
+    name = "redis_db"
     # this db is a singelton per port. meaning no 2 instances
     # should be created for the same port at the same time
     _obj = None
@@ -467,6 +469,10 @@ def _start_redis_server(cls) -> bool:
             "--daemonize",
             "yes",
         ]
+        cls.printer.print(
+            f"Redis command: {shlex.join(cmd)}",
+            log_to_logfiles_only=True,
+        )
         process = subprocess.Popen(
             cmd,
             cwd=os.getcwd(),

slips_files/core/database/redis_db/profile_handler.py

@@ -1320,7 +1320,6 @@ def mark_profile_as_gateway(self, profileid):
         """
         Used to mark this profile as dhcp server
         """
-
         self.set_profileid_field(profileid, self.constants.GATEWAY, "true")
 
     def set_ipv6_of_profile(self, profileid, ip: list):

slips_files/core/evidence_handler_worker.py

@@ -129,6 +129,7 @@ def add_evidence_to_json_log_file(
                                 evidence.confidence
                             ),
                             "timewindow": evidence.timewindow.number,
+                            "immune_type": evidence.immune_type,
                         }
                     )
                 }

slips_files/core/input/zeek/utils/zeek_input_utils.py

@@ -4,6 +4,7 @@
 import datetime
 import json
 import os
+import shlex
 import signal
 import subprocess
 import threading
@@ -645,8 +646,8 @@ def _get_zeek_cmd_and_logs_dir(
         safe_zeek_logs_dir = utils.validate_safe_path(
             zeek_logs_dir, must_exist=True
         )
-        str_cmd = " ".join(command)
-        self.input.print(f"Zeek command: {str_cmd}", 3, 0)
+        str_cmd = shlex.join(command)
+        self.input.print(f"Zeek command: {str_cmd}", log_to_logfiles_only=True)
         return command, safe_zeek_logs_dir
 
     def _start_zeek_process(self, command, zeek_logs_dir) -> subprocess.Popen:

slips_files/core/input_profilers/zeek_to_slips_maps.py

@@ -14,6 +14,8 @@
     "history": "history",
     "orig_pkts": "spkts",
     "resp_pkts": "dpkts",
+    "orig_l2_addr": "smac",
+    "resp_l2_addr": "dmac",
     "label": "ground_truth_label",
     "detailedlabel": "detailed_ground_truth_label",
 }

slips_files/core/profiler_worker.py

@@ -394,6 +394,27 @@ def get_gw_ip_using_gw_mac(self, gw_mac) -> Optional[str]:
         # all of them are ipv6, return the first
         return gw_ips[0]
 
+    def gw_ip_belongs_to_localnet(self, gw_ip: str) -> bool:
+        """checks if the given detected gw_ip belongs to the detected local
+        network"""
+        try:
+            gw_ip_obj = ipaddress.ip_address(gw_ip)
+        except ValueError:
+            return False
+
+        for interface in utils.get_all_interfaces(self.args):
+            local_net = self.db.get_local_network(interface)
+            if not local_net:
+                continue
+            try:
+                local_net_obj = ipaddress.ip_network(local_net, strict=False)
+            except ValueError:
+                continue
+
+            if gw_ip_obj in local_net_obj:
+                return True
+        return False
+
     def get_gateway_info(self, flow):
         """
         Gets the IP and MAC of the gateway and stores them in the db
@@ -412,7 +433,7 @@ def get_gateway_info(self, flow):
         if not gw_mac_found:
             # we didnt get the MAC of the GW of this flow's interface
             # ok consider the GW MAC = any dst MAC of a flow
-            # going from a private srcip -> a public ip
+            # going from a private srcip -> a public dstip
             if (
                 utils.is_private_ip(flow.saddr)
                 and not utils.is_ignored_ip(flow.daddr)
@@ -428,8 +449,12 @@ def get_gateway_info(self, flow):
 
         # we need the mac to be set to be able to find the ip using it
         if not self.is_gw_info_detected("ip", flow.interface) and gw_mac_found:
-            gw_ip: Optional[str] = self.get_gw_ip_using_gw_mac(flow.dmac)
-            if gw_ip:
+            gw_mac: Optional[str] = self.gw_macs.get(flow.interface)
+            if not gw_mac:
+                return
+
+            gw_ip: Optional[str] = self.get_gw_ip_using_gw_mac(gw_mac)
+            if gw_ip and self.gw_ip_belongs_to_localnet(gw_ip):
                 self.gw_ips[flow.interface] = gw_ip
                 self.db.set_default_gateway("IP", gw_ip, flow.interface)
                 self.print(

slips_files/core/structures/evidence.py

@@ -116,6 +116,11 @@ def __str__(self):
         return self.name
 
 
+class ImmuneType(Enum):
+    PAMP = auto()
+    DAMP = auto()
+
+
 class Direction(Enum):
     DST = auto()
     SRC = auto()
@@ -244,7 +249,7 @@ def __repr__(self):
 class Method(Enum):
     """
     Describes how was the evidence generated. these values are IDMEFv2
-    https://www.ietf.org/id/draft-lehmann-idmefv2-03.html#section-5.3-4.20.1
+    https://datatracker.ietf.org/doc/draft-lehmann-idmefv2/#:~:text=Site%22%0A%20%20%20%20%20%20%20%7D%0A%20%20%20%20%20%5D%0A%20%20%20%7D%0A%0AAppendix%20C.-,JSON%20Validation%20Schema%20(Non%2Dnormative),-Listing%205%20contains
     """
 
     BIOMETRIC = "Biometric"
@@ -282,6 +287,7 @@ class Evidence:
     timestamp: str = field(
         metadata={"validate": lambda x: validate_timestamp(x)}
     )
+    immune_type: ImmuneType = field(default=None)
     interface: str = field(default="default")
     victim: Optional[Victim] = field(default=False)
     proto: Optional[Proto] = field(default=False)