From fc4153b1277ca06ad1f2012d9dd8293f3f15f354 Mon Sep 17 00:00:00 2001 From: alya Date: Fri, 5 Jun 2026 05:58:24 +0300 Subject: [PATCH 01/25] make sure all tab separated zeek log files in the dataset/ directory end with a #close --- dataset/port-scans/horizontal/conn.log | 1 + .../zeek-labeled/conn.log.labeled | 1 + 2 files changed, 2 insertions(+) diff --git a/dataset/port-scans/horizontal/conn.log b/dataset/port-scans/horizontal/conn.log index 54225fd2e3..9b9978017c 100644 --- a/dataset/port-scans/horizontal/conn.log +++ b/dataset/port-scans/horizontal/conn.log @@ -47,3 +47,4 @@ 960.869113 CyxM402uQN5ldQ2Td3 10.0.2.112 49248 62.193.227.35 80 tcp - - - - S0 - - 0 S 1 52 0 0 (empty) 958.869114 CeSj4A1BN9nBer0fe9 10.0.2.112 49245 188.252.28.130 80 tcp - - - - S0 - - 0 S 1 52 0 0 (empty) 958.869115 CeSj4A1BN9nBei0fe9 10.0.2.112 49245 188.252.30.130 80 tcp - - - - S0 - - 0 S 1 52 0 0 (empty) +#close 2015-11-18-16-36-13 diff --git a/dataset/test18-malicious-ctu-sme-11-win/zeek-labeled/conn.log.labeled b/dataset/test18-malicious-ctu-sme-11-win/zeek-labeled/conn.log.labeled index 65ef6c3bd2..20f3a307c4 100644 --- a/dataset/test18-malicious-ctu-sme-11-win/zeek-labeled/conn.log.labeled +++ b/dataset/test18-malicious-ctu-sme-11-win/zeek-labeled/conn.log.labeled @@ -772,3 +772,4 @@ 1677024002.966990 CWuwGW3u99uhiJZAek 192.168.1.107 49323 192.168.1.135 8009 tcp - 496.012311 11000 11000 OTH T T 0 DTdtATtTt 400 38000 200 30000 - Benign From_benign-To_benign-Device_chromecast_tv_assistant 1677024033.849842 CfyNXH2BVoPq6CZkWe 192.168.1.107 49752 77.88.55.55 443 tcp - 450.657534 1 0 OTH T F 0 DTaT 22 902 22 1144 - Unknown (empty) 1677024005.317605 Csmqcc1SQBkWaMDiah 192.168.1.107 64746 162.159.136.234 443 tcp - 495.305609 572 11677 OTH T F 0 DTadtAtT 76 4184 84 26714 - Benign From_benign-To_benign-Application_discord +#close 2024-08-16-09-45-01 From b913a7c24a73846e2cad7840b7df2b1641fa6051 Mon Sep 17 00:00:00 2001 From: alya Date: Fri, 5 Jun 2026 06:00:58 +0300 Subject: [PATCH 02/25] fix KeyError getting the GW of an interface --- slips_files/common/slips_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/slips_files/common/slips_utils.py b/slips_files/common/slips_utils.py index 8c5fca40b6..2de8ae4c58 100644 --- a/slips_files/common/slips_utils.py +++ b/slips_files/common/slips_utils.py @@ -294,7 +294,7 @@ def get_gateway_for_iface(self, iface: str) -> Optional[str]: """returns the default gateway for the given interface""" gws = netifaces.gateways() for family in (netifaces.AF_INET, netifaces.AF_INET6): - if "default" in gws and gws["default"][family]: + if "default" in gws and family in gws["default"]: gw, gw_iface = gws["default"][family] if gw_iface == iface: return gw From 76f3e15681a40b5335195fb5cde1bb309bac93ab Mon Sep 17 00:00:00 2001 From: alya Date: Fri, 5 Jun 2026 06:06:14 +0300 Subject: [PATCH 03/25] add the used redis and zeek commands to slips.log --- .../core/database/redis_db/database.py | 6 +++++ .../core/input/zeek/utils/zeek_input_utils.py | 5 ++-- .../core/database/test_database.py | 26 +++++++++++++++++++ .../core/input/test_zeek_input_utils.py | 24 +++++++++++++++++ 4 files changed, 59 insertions(+), 2 deletions(-) diff --git a/slips_files/core/database/redis_db/database.py b/slips_files/core/database/redis_db/database.py index e62c9e49df..36df1b4026 100644 --- a/slips_files/core/database/redis_db/database.py +++ b/slips_files/core/database/redis_db/database.py @@ -31,6 +31,7 @@ import os import redis +import shlex import time import json import subprocess @@ -62,6 +63,7 @@ class RedisDB( ScanDetectionsHandler, Publisher, ): + name = "redis_db" # this db is a singelton per port. meaning no 2 instances # should be created for the same port at the same time _obj = None @@ -467,6 +469,10 @@ def _start_redis_server(cls) -> bool: "--daemonize", "yes", ] + cls.printer.print( + f"Redis command: {shlex.join(cmd)}", + log_to_logfiles_only=True, + ) process = subprocess.Popen( cmd, cwd=os.getcwd(), diff --git a/slips_files/core/input/zeek/utils/zeek_input_utils.py b/slips_files/core/input/zeek/utils/zeek_input_utils.py index 34176b0b4a..123988391c 100644 --- a/slips_files/core/input/zeek/utils/zeek_input_utils.py +++ b/slips_files/core/input/zeek/utils/zeek_input_utils.py @@ -4,6 +4,7 @@ import datetime import json import os +import shlex import signal import subprocess import threading @@ -645,8 +646,8 @@ def _get_zeek_cmd_and_logs_dir( safe_zeek_logs_dir = utils.validate_safe_path( zeek_logs_dir, must_exist=True ) - str_cmd = " ".join(command) - self.input.print(f"Zeek command: {str_cmd}", 3, 0) + str_cmd = shlex.join(command) + self.input.print(f"Zeek command: {str_cmd}", log_to_logfiles_only=True) return command, safe_zeek_logs_dir def _start_zeek_process(self, command, zeek_logs_dir) -> subprocess.Popen: diff --git a/tests/unit/slips_files/core/database/test_database.py b/tests/unit/slips_files/core/database/test_database.py index 29a08d2808..5b298340c6 100644 --- a/tests/unit/slips_files/core/database/test_database.py +++ b/tests/unit/slips_files/core/database/test_database.py @@ -399,6 +399,32 @@ def create_rdb_file(*args: Any, **kwargs: Any) -> Mock: mock_run.assert_called_once() +def test_start_redis_server_logs_command_to_logfiles_only( + tmp_path: Path, monkeypatch: Any +) -> None: + """Test Redis startup command logging is restricted to log files.""" + conf_file = tmp_path / "redis server.conf" + conf_file.write_text("port 6379\n", encoding="utf-8") + printer = Mock() + process = Mock(returncode=0) + process.communicate.return_value = (b"", b"") + + monkeypatch.setattr(RedisDB, "_conf_file", str(conf_file)) + monkeypatch.setattr(RedisDB, "_options", {}, raising=False) + monkeypatch.setattr(RedisDB, "redis_port", 6379, raising=False) + monkeypatch.setattr(RedisDB, "printer", printer, raising=False) + + with patch("subprocess.Popen", return_value=process) as popen: + assert RedisDB._start_redis_server() is True + + printer.print.assert_called_once_with( + f"Redis command: redis-server '{conf_file}' --port 6379 " + "--bind 127.0.0.1 --daemonize yes", + log_to_logfiles_only=True, + ) + popen.assert_called_once() + + def test_init_p2p_trust_db_uses_permanent_dir(tmp_path, monkeypatch): db = ModuleFactory().create_db_manager_obj(6379) monkeypatch.chdir(tmp_path) diff --git a/tests/unit/slips_files/core/input/test_zeek_input_utils.py b/tests/unit/slips_files/core/input/test_zeek_input_utils.py index 1aacfa314d..b150721e75 100644 --- a/tests/unit/slips_files/core/input/test_zeek_input_utils.py +++ b/tests/unit/slips_files/core/input/test_zeek_input_utils.py @@ -3,6 +3,7 @@ import json import threading import pytest +from pathlib import Path from types import SimpleNamespace from unittest.mock import Mock from tests.module_factory import ModuleFactory @@ -41,6 +42,29 @@ def test_get_ts_from_line_returns_timestamp_for_tabs(): assert line == "1.5\tfield\n" +def test_get_zeek_cmd_and_logs_dir_logs_command_to_logfiles_only( + tmp_path: Path, +) -> None: + """Test Zeek command logging is restricted to log files.""" + input_process = ModuleFactory().create_input_obj("", InputType.PCAP) + input_process.print = Mock() + command = ["zeek", "-r", "pcaps/input file.pcap"] + input_process.zeek_utils._construct_zeek_cmd = Mock(return_value=command) + + returned_command, zeek_dir = ( + input_process.zeek_utils._get_zeek_cmd_and_logs_dir( + str(tmp_path), "pcaps/input file.pcap" + ) + ) + + assert returned_command == command + assert zeek_dir == str(tmp_path) + input_process.print.assert_called_once_with( + "Zeek command: zeek -r 'pcaps/input file.pcap'", + log_to_logfiles_only=True, + ) + + def test_read_zeek_files_drains_generated_lines_during_live_update(tmp_path): test_file = tmp_path / "conn.log" test_file.write_text( From afe78dfbe36e0f0271b8a337cf9e86afeed23488 Mon Sep 17 00:00:00 2001 From: alya Date: Fri, 5 Jun 2026 06:17:32 +0300 Subject: [PATCH 04/25] iasync_module: suppress KeyboardInterrupt,SystemExit, and asyncio.CancelledError to avoid long tracebacks on ctrl+c --- slips_files/common/abstracts/iasync_module.py | 40 +++++++++-- .../common/abstracts/test_iasync_module.py | 68 +++++++++++++++++++ 2 files changed, 103 insertions(+), 5 deletions(-) create mode 100644 tests/unit/slips_files/common/abstracts/test_iasync_module.py diff --git a/slips_files/common/abstracts/iasync_module.py b/slips_files/common/abstracts/iasync_module.py index bdb18c79da..7a5a08d36d 100644 --- a/slips_files/common/abstracts/iasync_module.py +++ b/slips_files/common/abstracts/iasync_module.py @@ -4,6 +4,7 @@ import traceback from asyncio import Task from typing import ( + Any, Callable, List, ) @@ -49,11 +50,34 @@ def _remove_completed_task(self, task: asyncio.Task): # Task already removed or not tracked. pass - def handle_task_exception(self, task: asyncio.Task): + @staticmethod + def is_shutdown_exception(exception: BaseException | None) -> bool: + """ + Check whether an exception is part of normal process shutdown. + + :param exception: Exception raised while a task was running. + :return: True when the exception should not be logged as a task error. + """ + shutdown_exceptions = ( + KeyboardInterrupt, + SystemExit, + asyncio.CancelledError, + ) + return isinstance(exception, shutdown_exceptions) + + def handle_task_exception(self, task: asyncio.Task) -> None: + """ + Log task exceptions while suppressing normal shutdown exceptions. + + :param task: Completed asyncio task to inspect. + :return: None. + """ try: exception = task.exception() except asyncio.CancelledError: - return # Task was cancelled, not an error + return # Task was cancelled, not an error. + if self.is_shutdown_exception(exception): + return if exception: self.print(f"Unhandled exception in task: {exception!r} .. ") self.print_traceback_from_exception(exception, task) @@ -111,7 +135,9 @@ def run_async_function(self, func: Callable): loop.set_exception_handler(self.handle_loop_exception) return loop.run_until_complete(func()) - def handle_loop_exception(self, loop, context): + def handle_loop_exception( + self, loop: asyncio.AbstractEventLoop, context: dict[str, Any] + ) -> None: """A common loop exception handler""" exception = context.get("exception") future = context.get("future") @@ -119,9 +145,13 @@ def handle_loop_exception(self, loop, context): if future: try: future.result() - except Exception: + except BaseException as error: + if self.is_shutdown_exception(error): + return self.print_traceback() - elif exception: + elif isinstance(exception, BaseException): + if self.is_shutdown_exception(exception): + return self.print(f"Unhandled loop exception: {exception}") else: self.print(f"Unhandled loop error: {context.get('message')}") diff --git a/tests/unit/slips_files/common/abstracts/test_iasync_module.py b/tests/unit/slips_files/common/abstracts/test_iasync_module.py new file mode 100644 index 0000000000..74f732c966 --- /dev/null +++ b/tests/unit/slips_files/common/abstracts/test_iasync_module.py @@ -0,0 +1,68 @@ +# SPDX-FileCopyrightText: 2021 Sebastian Garcia +# SPDX-License-Identifier: GPL-2.0-only +import asyncio +from unittest.mock import Mock + +import pytest + +from tests.module_factory import ModuleFactory + + +@pytest.mark.parametrize( + "exception", + [KeyboardInterrupt(), SystemExit(), asyncio.CancelledError()], +) +def test_handle_task_exception_ignores_shutdown_exceptions(exception): + """ + Verify shutdown exceptions are not logged as task failures. + + :param exception: Shutdown exception raised or returned by a task. + :return: None. + """ + module_factory = ModuleFactory() + flowalerts = module_factory.create_flowalerts_obj() + task = Mock() + task.exception.return_value = exception + + flowalerts.handle_task_exception(task) + + flowalerts.print.assert_not_called() + + +def test_handle_task_exception_ignores_cancelled_tasks(): + """ + Verify cancelled tasks are not logged as task failures. + + :return: None. + """ + module_factory = ModuleFactory() + flowalerts = module_factory.create_flowalerts_obj() + task = Mock() + task.exception.side_effect = asyncio.CancelledError() + + flowalerts.handle_task_exception(task) + + flowalerts.print.assert_not_called() + + +def test_handle_task_exception_logs_regular_exceptions(): + """ + Verify non-shutdown task exceptions are still logged. + + :return: None. + """ + module_factory = ModuleFactory() + flowalerts = module_factory.create_flowalerts_obj() + exception = ValueError("boom") + task = Mock() + task.exception.return_value = exception + flowalerts.print_traceback_from_exception = Mock() + + flowalerts.handle_task_exception(task) + + flowalerts.print.assert_called_once_with( + "Unhandled exception in task: ValueError('boom') .. " + ) + flowalerts.print_traceback_from_exception.assert_called_once_with( + exception, task + ) From 51ff7369129af3e8cca5283e79f0bb708b2fbada Mon Sep 17 00:00:00 2001 From: alya Date: Sat, 6 Jun 2026 00:44:46 +0300 Subject: [PATCH 05/25] add immmune type PAMP/DAMP to slips evidence, STIX and IDMEF evidence --- modules/exporting_alerts/stix_exporter.py | 6 ++++++ modules/flow_alerts/set_evidence.py | 2 ++ slips/main.py | 1 + slips_files/core/evidence_handler_worker.py | 1 + slips_files/core/structures/evidence.py | 6 ++++++ 5 files changed, 16 insertions(+) diff --git a/modules/exporting_alerts/stix_exporter.py b/modules/exporting_alerts/stix_exporter.py index 875f0b5854..4a4605248f 100644 --- a/modules/exporting_alerts/stix_exporter.py +++ b/modules/exporting_alerts/stix_exporter.py @@ -649,6 +649,10 @@ def _build_custom_properties( if src_port: custom_properties["x_slips_src_port"] = src_port + immune_type = evidence.get("immune_type") + if immune_type: + custom_properties["x_slips_immune_type"] = src_port + return { key: value for key, value in custom_properties.items() @@ -656,6 +660,8 @@ def _build_custom_properties( } def _build_indicator(self, evidence: dict): + """this is where a single evidence is converted from slips format + to STIX format""" attacker = (evidence.get("attacker") or {}).get("value") if not attacker: attacker = (evidence.get("profile") or {}).get("ip") diff --git a/modules/flow_alerts/set_evidence.py b/modules/flow_alerts/set_evidence.py index 1a62708d90..bc17257ec1 100644 --- a/modules/flow_alerts/set_evidence.py +++ b/modules/flow_alerts/set_evidence.py @@ -16,6 +16,7 @@ EvidenceType, IoCType, Direction, + ImmuneType, ) ESTAB = "Established" @@ -620,6 +621,7 @@ def unknown_port(self, twid, flow) -> None: confidence=confidence, src_port=flow.sport, dst_port=flow.dport, + immune_type=ImmuneType.DAMP, ) self.db.set_evidence(evidence) diff --git a/slips/main.py b/slips/main.py index fae0def48a..2c315f98a5 100644 --- a/slips/main.py +++ b/slips/main.py @@ -428,6 +428,7 @@ def get_analyzed_flows_percentage(self) -> str: else: percentage = int(percentage) + percentage = min(100, percentage) # cap at 100% return f"Analyzed Flows: {green(percentage)}{green('%')}. " def is_total_flows_unknown(self) -> bool: diff --git a/slips_files/core/evidence_handler_worker.py b/slips_files/core/evidence_handler_worker.py index cf5cd51e7a..b2b7dac44e 100644 --- a/slips_files/core/evidence_handler_worker.py +++ b/slips_files/core/evidence_handler_worker.py @@ -129,6 +129,7 @@ def add_evidence_to_json_log_file( evidence.confidence ), "timewindow": evidence.timewindow.number, + "immune_type": evidence.immune_type, } ) } diff --git a/slips_files/core/structures/evidence.py b/slips_files/core/structures/evidence.py index 0b12a45e2e..6d1e9d6a78 100644 --- a/slips_files/core/structures/evidence.py +++ b/slips_files/core/structures/evidence.py @@ -116,6 +116,11 @@ def __str__(self): return self.name +class ImmuneType(Enum): + PAMP = auto() + DAMP = auto() + + class Direction(Enum): DST = auto() SRC = auto() @@ -282,6 +287,7 @@ class Evidence: timestamp: str = field( metadata={"validate": lambda x: validate_timestamp(x)} ) + immune_type: ImmuneType = field(default=None) interface: str = field(default="default") victim: Optional[Victim] = field(default=False) proto: Optional[Proto] = field(default=False) From db22d8eb3e530c9061b41240463cc2ba0523d015 Mon Sep 17 00:00:00 2001 From: alya Date: Sat, 6 Jun 2026 00:51:02 +0300 Subject: [PATCH 06/25] fix the links to idmefv2 docs/schema --- slips_files/common/idmefv2.py | 5 +++-- slips_files/core/structures/evidence.py | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/slips_files/common/idmefv2.py b/slips_files/common/idmefv2.py index c946a69363..44118c6951 100644 --- a/slips_files/common/idmefv2.py +++ b/slips_files/common/idmefv2.py @@ -42,7 +42,8 @@ class IDMEFv2: Class to convert Slips evidence and alerts to The Incident Detection Message Exchange Format version 2 (IDMEFv2 format). More Details about it here: - https://www.ietf.org/id/draft-lehmann-idmefv2-03.html#name-the-alert-class + https://datatracker.ietf.org/doc/draft-lehmann-idmefv2/#:~:text=Site%22%0A%20%20%20%20%20%20%20%7D%0A%20%20%20%20%20%5D%0A%20%20%20%7D%0A%0AAppendix%20C.-,JSON%20Validation%20Schema%20(Non%2Dnormative),-Listing%205%20contains + """ name = "idmefv2" @@ -185,7 +186,7 @@ def convert_to_idmef_event(self, evidence: Evidence) -> Message: The Incident Detection Message Exchange Format version 2 (IDMEFv2 format). More Details about it here: - https://www.ietf.org/id/draft-lehmann-idmefv2-03.html#name-the-alert-class + https://datatracker.ietf.org/doc/draft-lehmann-idmefv2/#:~:text=Site%22%0A%20%20%20%20%20%20%20%7D%0A%20%20%20%20%20%5D%0A%20%20%20%7D%0A%0AAppendix%20C.-,JSON%20Validation%20Schema%20(Non%2Dnormative),-Listing%205%20contains """ try: now = datetime.now(utils.local_tz).isoformat("T") diff --git a/slips_files/core/structures/evidence.py b/slips_files/core/structures/evidence.py index 6d1e9d6a78..beaf4701ac 100644 --- a/slips_files/core/structures/evidence.py +++ b/slips_files/core/structures/evidence.py @@ -249,7 +249,7 @@ def __repr__(self): class Method(Enum): """ Describes how was the evidence generated. these values are IDMEFv2 - https://www.ietf.org/id/draft-lehmann-idmefv2-03.html#section-5.3-4.20.1 + https://datatracker.ietf.org/doc/draft-lehmann-idmefv2/#:~:text=Site%22%0A%20%20%20%20%20%20%20%7D%0A%20%20%20%20%20%5D%0A%20%20%20%7D%0A%0AAppendix%20C.-,JSON%20Validation%20Schema%20(Non%2Dnormative),-Listing%205%20contains """ BIOMETRIC = "Biometric" From 5adacda7b2438373383ec7877140f4358c10ba68 Mon Sep 17 00:00:00 2001 From: alya Date: Sat, 6 Jun 2026 00:53:24 +0300 Subject: [PATCH 07/25] Dont set evidence for "unknown ports" when the proto isnt recognized as tcp or udp by Zeek --- modules/flow_alerts/conn.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/modules/flow_alerts/conn.py b/modules/flow_alerts/conn.py index 329874882a..3008b60267 100644 --- a/modules/flow_alerts/conn.py +++ b/modules/flow_alerts/conn.py @@ -192,6 +192,10 @@ def check_unknown_port(self, profileid, twid, flow): if not flow.dport: return + if "unknown_transport" in flow.proto: + # the protocol is unrecognized by zeek. not tcp or udp. + return + if self.db.is_a_port_scanner(flow.saddr, twid): # avoid setting unknown port evidence for each port # scanned from this attacker From 5b6b6e036a9e3a6cb9c040c54f982a0762750e49 Mon Sep 17 00:00:00 2001 From: alya Date: Sat, 6 Jun 2026 00:58:22 +0300 Subject: [PATCH 08/25] fix same user agent detected in "Multiple user agent" evidence --- modules/http_analyzer/http_analyzer.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/modules/http_analyzer/http_analyzer.py b/modules/http_analyzer/http_analyzer.py index 40bfceec31..eb60ea1b4d 100644 --- a/modules/http_analyzer/http_analyzer.py +++ b/modules/http_analyzer/http_analyzer.py @@ -372,6 +372,10 @@ def check_multiple_user_agents_in_a_row( return False ua: str = cached_ua.get("user_agent", "") + if ua == flow.user_agent: + # same UA is used again, nothing to alert about here + return False + self.set_evidence.multiple_user_agents_in_a_row(flow, ua, twid) return True From 27f9833f2e7a5a3ac5e53adc20606f305fa1f47a Mon Sep 17 00:00:00 2001 From: alya Date: Sat, 6 Jun 2026 02:03:59 +0300 Subject: [PATCH 09/25] ensure the detected gw ip belongs to the detected localnet before reporting it. --- slips_files/core/profiler_worker.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/slips_files/core/profiler_worker.py b/slips_files/core/profiler_worker.py index 9636766af7..1f68838303 100644 --- a/slips_files/core/profiler_worker.py +++ b/slips_files/core/profiler_worker.py @@ -394,6 +394,17 @@ def get_gw_ip_using_gw_mac(self, gw_mac) -> Optional[str]: # all of them are ipv6, return the first return gw_ips[0] + def gw_ip_belongs_to_localnet(self, gw_ip: str) -> bool: + """checks if the given detected gw_ip belongs to the detected local + network""" + for interface in utils.get_all_interfaces(self.args): + local_net = self.db.get_local_network(interface) + if not local_net: + continue + if gw_ip in ipaddress.ip_network(local_net): + return True + return False + def get_gateway_info(self, flow): """ Gets the IP and MAC of the gateway and stores them in the db @@ -429,7 +440,7 @@ def get_gateway_info(self, flow): # we need the mac to be set to be able to find the ip using it if not self.is_gw_info_detected("ip", flow.interface) and gw_mac_found: gw_ip: Optional[str] = self.get_gw_ip_using_gw_mac(flow.dmac) - if gw_ip: + if gw_ip and self.gw_ip_belongs_to_localnet(gw_ip): self.gw_ips[flow.interface] = gw_ip self.db.set_default_gateway("IP", gw_ip, flow.interface) self.print( From ec1c0113f7e3a78210e5cd56a67ccebb5af891cb Mon Sep 17 00:00:00 2001 From: alya Date: Sat, 6 Jun 2026 02:20:01 +0300 Subject: [PATCH 10/25] fix problem detecting if the gw ip belongs to localnetwork --- .../core/database/redis_db/profile_handler.py | 1 - slips_files/core/profiler_worker.py | 20 ++++++++-- .../slips_files/core/test_profiler_worker.py | 37 +++++++++++++++++++ 3 files changed, 54 insertions(+), 4 deletions(-) diff --git a/slips_files/core/database/redis_db/profile_handler.py b/slips_files/core/database/redis_db/profile_handler.py index f3267fc4b0..7c08268d52 100644 --- a/slips_files/core/database/redis_db/profile_handler.py +++ b/slips_files/core/database/redis_db/profile_handler.py @@ -1320,7 +1320,6 @@ def mark_profile_as_gateway(self, profileid): """ Used to mark this profile as dhcp server """ - self.set_profileid_field(profileid, self.constants.GATEWAY, "true") def set_ipv6_of_profile(self, profileid, ip: list): diff --git a/slips_files/core/profiler_worker.py b/slips_files/core/profiler_worker.py index 1f68838303..6ab86c2005 100644 --- a/slips_files/core/profiler_worker.py +++ b/slips_files/core/profiler_worker.py @@ -397,11 +397,21 @@ def get_gw_ip_using_gw_mac(self, gw_mac) -> Optional[str]: def gw_ip_belongs_to_localnet(self, gw_ip: str) -> bool: """checks if the given detected gw_ip belongs to the detected local network""" + try: + gw_ip_obj = ipaddress.ip_address(gw_ip) + except ValueError: + return False + for interface in utils.get_all_interfaces(self.args): local_net = self.db.get_local_network(interface) if not local_net: continue - if gw_ip in ipaddress.ip_network(local_net): + try: + local_net_obj = ipaddress.ip_network(local_net, strict=False) + except ValueError: + continue + + if gw_ip_obj in local_net_obj: return True return False @@ -423,7 +433,7 @@ def get_gateway_info(self, flow): if not gw_mac_found: # we didnt get the MAC of the GW of this flow's interface # ok consider the GW MAC = any dst MAC of a flow - # going from a private srcip -> a public ip + # going from a private srcip -> a public dstip if ( utils.is_private_ip(flow.saddr) and not utils.is_ignored_ip(flow.daddr) @@ -439,7 +449,11 @@ def get_gateway_info(self, flow): # we need the mac to be set to be able to find the ip using it if not self.is_gw_info_detected("ip", flow.interface) and gw_mac_found: - gw_ip: Optional[str] = self.get_gw_ip_using_gw_mac(flow.dmac) + gw_mac: Optional[str] = self.gw_macs.get(flow.interface) + if not gw_mac: + return + + gw_ip: Optional[str] = self.get_gw_ip_using_gw_mac(gw_mac) if gw_ip and self.gw_ip_belongs_to_localnet(gw_ip): self.gw_ips[flow.interface] = gw_ip self.db.set_default_gateway("IP", gw_ip, flow.interface) diff --git a/tests/unit/slips_files/core/test_profiler_worker.py b/tests/unit/slips_files/core/test_profiler_worker.py index 14f96f8fec..6861463eb2 100644 --- a/tests/unit/slips_files/core/test_profiler_worker.py +++ b/tests/unit/slips_files/core/test_profiler_worker.py @@ -395,6 +395,7 @@ def test_get_gateway_info_sets_mac_and_ip( profiler = ModuleFactory().create_profiler_worker_obj() profiler.is_gw_info_detected = Mock(side_effect=[False, False]) profiler.get_gw_ip_using_gw_mac = Mock(return_value="8.8.8.1") + profiler.gw_ip_belongs_to_localnet = Mock(return_value=True) mock_is_private_ip.return_value = True mock_is_ignored_ip.return_value = False flow = make_conn(dmac="00:11:22:33:44:55") @@ -428,6 +429,42 @@ def test_get_gateway_info_does_not_set_gateway_for_non_private_source(_mock): profiler.db.set_default_gateway.assert_not_called() +def test_get_gateway_info_uses_detected_gateway_mac_for_ip_lookup(): + profiler = ModuleFactory().create_profiler_worker_obj() + profiler.gw_macs = {"eth0": "00:11:22:33:44:55"} + profiler.is_gw_info_detected = Mock(side_effect=[True, False]) + profiler.get_gw_ip_using_gw_mac = Mock(return_value="192.168.1.1") + profiler.gw_ip_belongs_to_localnet = Mock(return_value=True) + + profiler.get_gateway_info(make_conn(dmac="66:77:88:99:aa:bb")) + + profiler.get_gw_ip_using_gw_mac.assert_called_once_with( + "00:11:22:33:44:55" + ) + profiler.db.set_default_gateway.assert_called_once_with( + "IP", "192.168.1.1", "eth0" + ) + + +@pytest.mark.parametrize( + "gw_ip, local_net, expected", + [ + ("192.168.1.1", "192.168.1.0/24", True), + ("10.0.0.1", "192.168.1.0/24", False), + ("not-an-ip", "192.168.1.0/24", False), + ], +) +@patch("slips_files.core.profiler_worker.utils.get_all_interfaces") +def test_gw_ip_belongs_to_localnet_handles_string_ips( + mock_get_all_interfaces, gw_ip, local_net, expected +): + profiler = ModuleFactory().create_profiler_worker_obj() + mock_get_all_interfaces.return_value = ["eth0"] + profiler.db.get_local_network.return_value = local_net + + assert profiler.gw_ip_belongs_to_localnet(gw_ip) is expected + + def test_get_gw_ip_using_gw_mac_prefers_ipv4(): profiler = ModuleFactory().create_profiler_worker_obj() profiler.db.get_ip_of_mac.return_value = json.dumps( From a41aa8126d2d69f63ac7e56148868bd6b951a0f3 Mon Sep 17 00:00:00 2001 From: alya Date: Sat, 6 Jun 2026 02:43:13 +0300 Subject: [PATCH 11/25] fix hanging on the pre_main of ip_info forever --- modules/ip_info/ip_info.py | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/modules/ip_info/ip_info.py b/modules/ip_info/ip_info.py index a2dfe59073..d7e5509f94 100644 --- a/modules/ip_info/ip_info.py +++ b/modules/ip_info/ip_info.py @@ -88,7 +88,7 @@ def subscribe_to_channels(self): "check_jarm_hash": self.c4, } - async def open_dbs(self): + def open_dbs(self) -> None: """Function to open the different offline databases used in this module. ASN, Country etc..""" # Open the maxminddb ASN offline db @@ -116,7 +116,18 @@ async def open_dbs(self): "https://dev.maxmind.com/geoip/geolite2-free-geolocation-data?lang=en. " "Please note it must be the MaxMind DB version." ) - self.create_task(self.read_mac_db) + self._start_mac_db_reader() + + def _start_mac_db_reader(self) -> None: + """ + Schedule the MAC vendor database reader on the active event loop. + """ + try: + asyncio.get_running_loop() + except RuntimeError: + return + + self.reading_mac_db_task = self.create_task(self.read_mac_db) async def read_mac_db(self): """ @@ -496,7 +507,6 @@ def get_gateway_ip_if_interface(self) -> Dict[str, str] | None: return interfaces: List[str] = utils.get_all_interfaces(self.args) - gw_ips = {} for interface in interfaces: try: @@ -518,6 +528,9 @@ def get_own_mac() -> str: def _get_wifi_interface_if_ap(self) -> str | None: ap_interfaces: str = self.db.get_wifi_interface() + if not ap_interfaces: + return None + try: # we're now sure that we're running in AP mode wifi_interface = ap_interfaces["wifi_interface"] @@ -733,18 +746,12 @@ def register_private_dns_server(self, flow: Any) -> bool: ) return True - def wait_for_dbs(self): + def wait_for_dbs(self) -> None: """ wait for update manager to finish updating the mac db and open the rest of dbs before starting this module """ - # this is the loop that controls tasks running on open_dbs - loop = asyncio.new_event_loop() - asyncio.set_event_loop(loop) - # run open_dbs in the background so we don't have - # to wait for update manager to finish updating the mac db to start - # this module - loop.run_until_complete(self.open_dbs()) + self.open_dbs() def set_evidence_malicious_jarm_hash( self, @@ -808,7 +815,7 @@ def set_evidence_malicious_jarm_hash( self.db.set_evidence(evidence) - def pre_main(self): + def pre_main(self) -> None: utils.drop_root_privs_permanently() self.wait_for_dbs() # the following method only works when running on an interface From 14e615f6e1c8e3248f254309c39c6bbc9b5fe02f Mon Sep 17 00:00:00 2001 From: alya Date: Mon, 22 Jun 2026 22:04:59 +0300 Subject: [PATCH 12/25] update unit tests --- .../input_profilers/zeek_to_slips_maps.py | 2 + tests/unit/modules/ip_info/test_ip_info.py | 22 ++++ .../common/abstracts/test_iasync_module.py | 102 ++++++++++++++---- .../core/input/test_zeek_input_profiler.py | 100 ++++++++++++++++- 4 files changed, 206 insertions(+), 20 deletions(-) diff --git a/slips_files/core/input_profilers/zeek_to_slips_maps.py b/slips_files/core/input_profilers/zeek_to_slips_maps.py index da5b534337..3914b9dd25 100644 --- a/slips_files/core/input_profilers/zeek_to_slips_maps.py +++ b/slips_files/core/input_profilers/zeek_to_slips_maps.py @@ -14,6 +14,8 @@ "history": "history", "orig_pkts": "spkts", "resp_pkts": "dpkts", + "orig_l2_addr": "smac", + "resp_l2_addr": "dmac", "label": "ground_truth_label", "detailedlabel": "detailed_ground_truth_label", } diff --git a/tests/unit/modules/ip_info/test_ip_info.py b/tests/unit/modules/ip_info/test_ip_info.py index d3eae3ae7b..cbf9a1d10a 100644 --- a/tests/unit/modules/ip_info/test_ip_info.py +++ b/tests/unit/modules/ip_info/test_ip_info.py @@ -26,6 +26,28 @@ ) +def test_start_mac_db_reader_returns_without_running_loop() -> None: + module_factory = ModuleFactory() + ip_info = module_factory.create_ip_info_obj() + ip_info.create_task = Mock() + + ip_info._start_mac_db_reader() + + ip_info.create_task.assert_not_called() + + +async def test_start_mac_db_reader_schedules_reader_in_running_loop() -> None: + module_factory = ModuleFactory() + ip_info = module_factory.create_ip_info_obj() + created_task = Mock() + ip_info.create_task = Mock(return_value=created_task) + + ip_info._start_mac_db_reader() + + ip_info.create_task.assert_called_once_with(ip_info.read_mac_db) + assert ip_info.reading_mac_db_task == created_task + + @pytest.mark.parametrize( "ip_address, expected_geocountry", [ # Testcase 1: Valid IP address diff --git a/tests/unit/slips_files/common/abstracts/test_iasync_module.py b/tests/unit/slips_files/common/abstracts/test_iasync_module.py index 74f732c966..baafaee2cf 100644 --- a/tests/unit/slips_files/common/abstracts/test_iasync_module.py +++ b/tests/unit/slips_files/common/abstracts/test_iasync_module.py @@ -12,13 +12,9 @@ "exception", [KeyboardInterrupt(), SystemExit(), asyncio.CancelledError()], ) -def test_handle_task_exception_ignores_shutdown_exceptions(exception): - """ - Verify shutdown exceptions are not logged as task failures. - - :param exception: Shutdown exception raised or returned by a task. - :return: None. - """ +def test_handle_task_exception_ignores_shutdown_exceptions( + exception: BaseException, +) -> None: module_factory = ModuleFactory() flowalerts = module_factory.create_flowalerts_obj() task = Mock() @@ -29,12 +25,7 @@ def test_handle_task_exception_ignores_shutdown_exceptions(exception): flowalerts.print.assert_not_called() -def test_handle_task_exception_ignores_cancelled_tasks(): - """ - Verify cancelled tasks are not logged as task failures. - - :return: None. - """ +def test_handle_task_exception_ignores_cancelled_tasks() -> None: module_factory = ModuleFactory() flowalerts = module_factory.create_flowalerts_obj() task = Mock() @@ -45,12 +36,7 @@ def test_handle_task_exception_ignores_cancelled_tasks(): flowalerts.print.assert_not_called() -def test_handle_task_exception_logs_regular_exceptions(): - """ - Verify non-shutdown task exceptions are still logged. - - :return: None. - """ +def test_handle_task_exception_logs_regular_exceptions() -> None: module_factory = ModuleFactory() flowalerts = module_factory.create_flowalerts_obj() exception = ValueError("boom") @@ -66,3 +52,81 @@ def test_handle_task_exception_logs_regular_exceptions(): flowalerts.print_traceback_from_exception.assert_called_once_with( exception, task ) + + +@pytest.mark.parametrize( + ("exception", "expected"), + [ + (KeyboardInterrupt(), True), + (SystemExit(), True), + (asyncio.CancelledError(), True), + (ValueError("boom"), False), + (None, False), + ], +) +def test_is_shutdown_exception( + exception: BaseException | None, expected: bool +) -> None: + module_factory = ModuleFactory() + flowalerts = module_factory.create_flowalerts_obj() + + assert flowalerts.is_shutdown_exception(exception) is expected + + +def test_handle_loop_exception_ignores_shutdown_future() -> None: + module_factory = ModuleFactory() + flowalerts = module_factory.create_flowalerts_obj() + future = Mock() + future.result.side_effect = asyncio.CancelledError() + flowalerts.print_traceback = Mock() + + flowalerts.handle_loop_exception(Mock(), {"future": future}) + + flowalerts.print.assert_not_called() + flowalerts.print_traceback.assert_not_called() + + +def test_handle_loop_exception_logs_future_error() -> None: + module_factory = ModuleFactory() + flowalerts = module_factory.create_flowalerts_obj() + future = Mock() + future.result.side_effect = ValueError("boom") + flowalerts.print_traceback = Mock() + + flowalerts.handle_loop_exception(Mock(), {"future": future}) + + flowalerts.print_traceback.assert_called_once_with() + + +@pytest.mark.parametrize( + "exception", + [KeyboardInterrupt(), SystemExit(), asyncio.CancelledError()], +) +def test_handle_loop_exception_ignores_shutdown_exception( + exception: BaseException, +) -> None: + module_factory = ModuleFactory() + flowalerts = module_factory.create_flowalerts_obj() + + flowalerts.handle_loop_exception(Mock(), {"exception": exception}) + + flowalerts.print.assert_not_called() + + +def test_handle_loop_exception_logs_regular_exception() -> None: + module_factory = ModuleFactory() + flowalerts = module_factory.create_flowalerts_obj() + exception = ValueError("boom") + + flowalerts.handle_loop_exception(Mock(), {"exception": exception}) + + flowalerts.print.assert_called_once_with("Unhandled loop exception: boom") + + +def test_handle_loop_exception_logs_message() -> None: + module_factory = ModuleFactory() + flowalerts = module_factory.create_flowalerts_obj() + + flowalerts.handle_loop_exception(Mock(), {"message": "boom"}) + + flowalerts.print.assert_called_once_with("Unhandled loop error: boom") diff --git a/tests/unit/slips_files/core/input/test_zeek_input_profiler.py b/tests/unit/slips_files/core/input/test_zeek_input_profiler.py index 29f6d7fe80..5f4bbbdc48 100644 --- a/tests/unit/slips_files/core/input/test_zeek_input_profiler.py +++ b/tests/unit/slips_files/core/input/test_zeek_input_profiler.py @@ -3,7 +3,9 @@ from unittest.mock import Mock -from slips_files.core.input_profilers.zeek import ZeekJSON +import pytest + +from slips_files.core.input_profilers.zeek import ZeekJSON, ZeekTabs from tests.module_factory import ModuleFactory @@ -99,3 +101,99 @@ def test_zeek_json_maps_login_log_fields(): assert flow.confused is False assert flow.saddr == "147.32.80.40" assert flow.daddr == "147.32.80.37" + + +@pytest.mark.parametrize( + "src_mac,dst_mac", + [("00:0c:29:66:c7:82", "00:90:0b:7a:15:eb")], +) +def test_zeek_json_maps_conn_l2_addresses_to_mac_fields( + src_mac: str, dst_mac: str +) -> None: + """ + Test conn.log JSON l2 address fields are converted to MAC fields. + + :param src_mac: Source layer-2 address from Zeek conn.log. + :param dst_mac: Destination layer-2 address from Zeek conn.log. + :return: None. + """ + module_factory = ModuleFactory() + parser = ZeekJSON(module_factory.logger) + + flow, err = parser.process_line( + { + "type": "conn.log", + "interface": "default", + "data": { + "ts": 279.103822, + "uid": "CNybJS33LDUfyyg1Pi", + "id.orig_h": "10.0.2.15", + "id.orig_p": 44927, + "id.resp_h": "1.1.1.1", + "id.resp_p": 80, + "proto": "tcp", + "service": "http", + "duration": 0.5273809432983398, + "orig_bytes": 656, + "resp_bytes": 12310, + "conn_state": "SF", + "history": "ShADadFf", + "orig_pkts": 7, + "resp_pkts": 14, + "orig_l2_addr": src_mac, + "resp_l2_addr": dst_mac, + }, + } + ) + + assert err == "" + assert flow.smac == src_mac + assert flow.dmac == dst_mac + + +@pytest.mark.parametrize( + "src_mac,dst_mac", + [("08:00:27:ef:ee:34", "52:54:00:12:35:02")], +) +def test_zeek_tabs_maps_conn_l2_addresses_to_mac_fields( + src_mac: str, dst_mac: str +) -> None: + """ + Test conn.log tab l2 address fields are converted to MAC fields. + + :param src_mac: Source layer-2 address from Zeek conn.log. + :param dst_mac: Destination layer-2 address from Zeek conn.log. + :return: None. + """ + module_factory = ModuleFactory() + db = module_factory.logger + db.channels.NEW_ZEEK_FIELDS_LINE = "new_zeek_fields_line" + parser = ZeekTabs(db) + fields_line = ( + "#fields\tts\tuid\tid.orig_h\tid.orig_p\tid.resp_h\tid.resp_p\t" + "proto\tservice\tduration\torig_bytes\tresp_bytes\tconn_state\t" + "history\torig_pkts\tresp_pkts\torig_l2_addr\tresp_l2_addr" + ) + + flow, err = parser.process_line( + {"type": "conn.log", "interface": "default", "data": fields_line} + ) + assert flow is False + assert err == "Field line processed" + + flow, err = parser.process_line( + { + "type": "conn.log", + "interface": "default", + "data": ( + "904728.025376\tCIhV323VBG6udE1Ho3\t10.0.2.19\t" + "1701\t78.6.164.6\t2928\tudp\t-\t0.11151099996641278\t" + "196\t118\tSF\tDd\t1\t1\t" + f"{src_mac}\t{dst_mac}" + ), + } + ) + + assert err == "" + assert flow.smac == src_mac + assert flow.dmac == dst_mac From 4e567bdff9adfd81c07d4a14fa9741831d311f46 Mon Sep 17 00:00:00 2001 From: alya Date: Fri, 5 Jun 2026 05:58:24 +0300 Subject: [PATCH 13/25] make sure all tab separated zeek log files in the dataset/ directory end with a #close --- dataset/port-scans/horizontal/conn.log | 1 + .../zeek-labeled/conn.log.labeled | 1 + 2 files changed, 2 insertions(+) diff --git a/dataset/port-scans/horizontal/conn.log b/dataset/port-scans/horizontal/conn.log index 54225fd2e3..9b9978017c 100644 --- a/dataset/port-scans/horizontal/conn.log +++ b/dataset/port-scans/horizontal/conn.log @@ -47,3 +47,4 @@ 960.869113 CyxM402uQN5ldQ2Td3 10.0.2.112 49248 62.193.227.35 80 tcp - - - - S0 - - 0 S 1 52 0 0 (empty) 958.869114 CeSj4A1BN9nBer0fe9 10.0.2.112 49245 188.252.28.130 80 tcp - - - - S0 - - 0 S 1 52 0 0 (empty) 958.869115 CeSj4A1BN9nBei0fe9 10.0.2.112 49245 188.252.30.130 80 tcp - - - - S0 - - 0 S 1 52 0 0 (empty) +#close 2015-11-18-16-36-13 diff --git a/dataset/test18-malicious-ctu-sme-11-win/zeek-labeled/conn.log.labeled b/dataset/test18-malicious-ctu-sme-11-win/zeek-labeled/conn.log.labeled index 65ef6c3bd2..20f3a307c4 100644 --- a/dataset/test18-malicious-ctu-sme-11-win/zeek-labeled/conn.log.labeled +++ b/dataset/test18-malicious-ctu-sme-11-win/zeek-labeled/conn.log.labeled @@ -772,3 +772,4 @@ 1677024002.966990 CWuwGW3u99uhiJZAek 192.168.1.107 49323 192.168.1.135 8009 tcp - 496.012311 11000 11000 OTH T T 0 DTdtATtTt 400 38000 200 30000 - Benign From_benign-To_benign-Device_chromecast_tv_assistant 1677024033.849842 CfyNXH2BVoPq6CZkWe 192.168.1.107 49752 77.88.55.55 443 tcp - 450.657534 1 0 OTH T F 0 DTaT 22 902 22 1144 - Unknown (empty) 1677024005.317605 Csmqcc1SQBkWaMDiah 192.168.1.107 64746 162.159.136.234 443 tcp - 495.305609 572 11677 OTH T F 0 DTadtAtT 76 4184 84 26714 - Benign From_benign-To_benign-Application_discord +#close 2024-08-16-09-45-01 From b88f7d8964c0171a77dbc3a0828a887ef3d4dd3d Mon Sep 17 00:00:00 2001 From: alya Date: Fri, 5 Jun 2026 06:00:58 +0300 Subject: [PATCH 14/25] fix KeyError getting the GW of an interface --- slips_files/common/slips_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/slips_files/common/slips_utils.py b/slips_files/common/slips_utils.py index 8c5fca40b6..2de8ae4c58 100644 --- a/slips_files/common/slips_utils.py +++ b/slips_files/common/slips_utils.py @@ -294,7 +294,7 @@ def get_gateway_for_iface(self, iface: str) -> Optional[str]: """returns the default gateway for the given interface""" gws = netifaces.gateways() for family in (netifaces.AF_INET, netifaces.AF_INET6): - if "default" in gws and gws["default"][family]: + if "default" in gws and family in gws["default"]: gw, gw_iface = gws["default"][family] if gw_iface == iface: return gw From adb447132bec100b60b5a7b3eb0f47aa126fa520 Mon Sep 17 00:00:00 2001 From: alya Date: Fri, 5 Jun 2026 06:06:14 +0300 Subject: [PATCH 15/25] add the used redis and zeek commands to slips.log --- .../core/database/redis_db/database.py | 6 +++++ .../core/input/zeek/utils/zeek_input_utils.py | 5 ++-- .../core/database/test_database.py | 26 +++++++++++++++++++ .../core/input/test_zeek_input_utils.py | 24 +++++++++++++++++ 4 files changed, 59 insertions(+), 2 deletions(-) diff --git a/slips_files/core/database/redis_db/database.py b/slips_files/core/database/redis_db/database.py index e62c9e49df..36df1b4026 100644 --- a/slips_files/core/database/redis_db/database.py +++ b/slips_files/core/database/redis_db/database.py @@ -31,6 +31,7 @@ import os import redis +import shlex import time import json import subprocess @@ -62,6 +63,7 @@ class RedisDB( ScanDetectionsHandler, Publisher, ): + name = "redis_db" # this db is a singelton per port. meaning no 2 instances # should be created for the same port at the same time _obj = None @@ -467,6 +469,10 @@ def _start_redis_server(cls) -> bool: "--daemonize", "yes", ] + cls.printer.print( + f"Redis command: {shlex.join(cmd)}", + log_to_logfiles_only=True, + ) process = subprocess.Popen( cmd, cwd=os.getcwd(), diff --git a/slips_files/core/input/zeek/utils/zeek_input_utils.py b/slips_files/core/input/zeek/utils/zeek_input_utils.py index 34176b0b4a..123988391c 100644 --- a/slips_files/core/input/zeek/utils/zeek_input_utils.py +++ b/slips_files/core/input/zeek/utils/zeek_input_utils.py @@ -4,6 +4,7 @@ import datetime import json import os +import shlex import signal import subprocess import threading @@ -645,8 +646,8 @@ def _get_zeek_cmd_and_logs_dir( safe_zeek_logs_dir = utils.validate_safe_path( zeek_logs_dir, must_exist=True ) - str_cmd = " ".join(command) - self.input.print(f"Zeek command: {str_cmd}", 3, 0) + str_cmd = shlex.join(command) + self.input.print(f"Zeek command: {str_cmd}", log_to_logfiles_only=True) return command, safe_zeek_logs_dir def _start_zeek_process(self, command, zeek_logs_dir) -> subprocess.Popen: diff --git a/tests/unit/slips_files/core/database/test_database.py b/tests/unit/slips_files/core/database/test_database.py index 29a08d2808..5b298340c6 100644 --- a/tests/unit/slips_files/core/database/test_database.py +++ b/tests/unit/slips_files/core/database/test_database.py @@ -399,6 +399,32 @@ def create_rdb_file(*args: Any, **kwargs: Any) -> Mock: mock_run.assert_called_once() +def test_start_redis_server_logs_command_to_logfiles_only( + tmp_path: Path, monkeypatch: Any +) -> None: + """Test Redis startup command logging is restricted to log files.""" + conf_file = tmp_path / "redis server.conf" + conf_file.write_text("port 6379\n", encoding="utf-8") + printer = Mock() + process = Mock(returncode=0) + process.communicate.return_value = (b"", b"") + + monkeypatch.setattr(RedisDB, "_conf_file", str(conf_file)) + monkeypatch.setattr(RedisDB, "_options", {}, raising=False) + monkeypatch.setattr(RedisDB, "redis_port", 6379, raising=False) + monkeypatch.setattr(RedisDB, "printer", printer, raising=False) + + with patch("subprocess.Popen", return_value=process) as popen: + assert RedisDB._start_redis_server() is True + + printer.print.assert_called_once_with( + f"Redis command: redis-server '{conf_file}' --port 6379 " + "--bind 127.0.0.1 --daemonize yes", + log_to_logfiles_only=True, + ) + popen.assert_called_once() + + def test_init_p2p_trust_db_uses_permanent_dir(tmp_path, monkeypatch): db = ModuleFactory().create_db_manager_obj(6379) monkeypatch.chdir(tmp_path) diff --git a/tests/unit/slips_files/core/input/test_zeek_input_utils.py b/tests/unit/slips_files/core/input/test_zeek_input_utils.py index 1aacfa314d..b150721e75 100644 --- a/tests/unit/slips_files/core/input/test_zeek_input_utils.py +++ b/tests/unit/slips_files/core/input/test_zeek_input_utils.py @@ -3,6 +3,7 @@ import json import threading import pytest +from pathlib import Path from types import SimpleNamespace from unittest.mock import Mock from tests.module_factory import ModuleFactory @@ -41,6 +42,29 @@ def test_get_ts_from_line_returns_timestamp_for_tabs(): assert line == "1.5\tfield\n" +def test_get_zeek_cmd_and_logs_dir_logs_command_to_logfiles_only( + tmp_path: Path, +) -> None: + """Test Zeek command logging is restricted to log files.""" + input_process = ModuleFactory().create_input_obj("", InputType.PCAP) + input_process.print = Mock() + command = ["zeek", "-r", "pcaps/input file.pcap"] + input_process.zeek_utils._construct_zeek_cmd = Mock(return_value=command) + + returned_command, zeek_dir = ( + input_process.zeek_utils._get_zeek_cmd_and_logs_dir( + str(tmp_path), "pcaps/input file.pcap" + ) + ) + + assert returned_command == command + assert zeek_dir == str(tmp_path) + input_process.print.assert_called_once_with( + "Zeek command: zeek -r 'pcaps/input file.pcap'", + log_to_logfiles_only=True, + ) + + def test_read_zeek_files_drains_generated_lines_during_live_update(tmp_path): test_file = tmp_path / "conn.log" test_file.write_text( From 117058f504f3af9edad31234165ba5b01717b0bb Mon Sep 17 00:00:00 2001 From: alya Date: Fri, 5 Jun 2026 06:17:32 +0300 Subject: [PATCH 16/25] iasync_module: suppress KeyboardInterrupt,SystemExit, and asyncio.CancelledError to avoid long tracebacks on ctrl+c --- slips_files/common/abstracts/iasync_module.py | 40 +++++++++-- .../common/abstracts/test_iasync_module.py | 68 +++++++++++++++++++ 2 files changed, 103 insertions(+), 5 deletions(-) create mode 100644 tests/unit/slips_files/common/abstracts/test_iasync_module.py diff --git a/slips_files/common/abstracts/iasync_module.py b/slips_files/common/abstracts/iasync_module.py index bdb18c79da..7a5a08d36d 100644 --- a/slips_files/common/abstracts/iasync_module.py +++ b/slips_files/common/abstracts/iasync_module.py @@ -4,6 +4,7 @@ import traceback from asyncio import Task from typing import ( + Any, Callable, List, ) @@ -49,11 +50,34 @@ def _remove_completed_task(self, task: asyncio.Task): # Task already removed or not tracked. pass - def handle_task_exception(self, task: asyncio.Task): + @staticmethod + def is_shutdown_exception(exception: BaseException | None) -> bool: + """ + Check whether an exception is part of normal process shutdown. + + :param exception: Exception raised while a task was running. + :return: True when the exception should not be logged as a task error. + """ + shutdown_exceptions = ( + KeyboardInterrupt, + SystemExit, + asyncio.CancelledError, + ) + return isinstance(exception, shutdown_exceptions) + + def handle_task_exception(self, task: asyncio.Task) -> None: + """ + Log task exceptions while suppressing normal shutdown exceptions. + + :param task: Completed asyncio task to inspect. + :return: None. + """ try: exception = task.exception() except asyncio.CancelledError: - return # Task was cancelled, not an error + return # Task was cancelled, not an error. + if self.is_shutdown_exception(exception): + return if exception: self.print(f"Unhandled exception in task: {exception!r} .. ") self.print_traceback_from_exception(exception, task) @@ -111,7 +135,9 @@ def run_async_function(self, func: Callable): loop.set_exception_handler(self.handle_loop_exception) return loop.run_until_complete(func()) - def handle_loop_exception(self, loop, context): + def handle_loop_exception( + self, loop: asyncio.AbstractEventLoop, context: dict[str, Any] + ) -> None: """A common loop exception handler""" exception = context.get("exception") future = context.get("future") @@ -119,9 +145,13 @@ def handle_loop_exception(self, loop, context): if future: try: future.result() - except Exception: + except BaseException as error: + if self.is_shutdown_exception(error): + return self.print_traceback() - elif exception: + elif isinstance(exception, BaseException): + if self.is_shutdown_exception(exception): + return self.print(f"Unhandled loop exception: {exception}") else: self.print(f"Unhandled loop error: {context.get('message')}") diff --git a/tests/unit/slips_files/common/abstracts/test_iasync_module.py b/tests/unit/slips_files/common/abstracts/test_iasync_module.py new file mode 100644 index 0000000000..74f732c966 --- /dev/null +++ b/tests/unit/slips_files/common/abstracts/test_iasync_module.py @@ -0,0 +1,68 @@ +# SPDX-FileCopyrightText: 2021 Sebastian Garcia +# SPDX-License-Identifier: GPL-2.0-only +import asyncio +from unittest.mock import Mock + +import pytest + +from tests.module_factory import ModuleFactory + + +@pytest.mark.parametrize( + "exception", + [KeyboardInterrupt(), SystemExit(), asyncio.CancelledError()], +) +def test_handle_task_exception_ignores_shutdown_exceptions(exception): + """ + Verify shutdown exceptions are not logged as task failures. + + :param exception: Shutdown exception raised or returned by a task. + :return: None. + """ + module_factory = ModuleFactory() + flowalerts = module_factory.create_flowalerts_obj() + task = Mock() + task.exception.return_value = exception + + flowalerts.handle_task_exception(task) + + flowalerts.print.assert_not_called() + + +def test_handle_task_exception_ignores_cancelled_tasks(): + """ + Verify cancelled tasks are not logged as task failures. + + :return: None. + """ + module_factory = ModuleFactory() + flowalerts = module_factory.create_flowalerts_obj() + task = Mock() + task.exception.side_effect = asyncio.CancelledError() + + flowalerts.handle_task_exception(task) + + flowalerts.print.assert_not_called() + + +def test_handle_task_exception_logs_regular_exceptions(): + """ + Verify non-shutdown task exceptions are still logged. + + :return: None. + """ + module_factory = ModuleFactory() + flowalerts = module_factory.create_flowalerts_obj() + exception = ValueError("boom") + task = Mock() + task.exception.return_value = exception + flowalerts.print_traceback_from_exception = Mock() + + flowalerts.handle_task_exception(task) + + flowalerts.print.assert_called_once_with( + "Unhandled exception in task: ValueError('boom') .. " + ) + flowalerts.print_traceback_from_exception.assert_called_once_with( + exception, task + ) From 47df6f9cb67fd1da889e3b97c0d1762ccd0fecbe Mon Sep 17 00:00:00 2001 From: alya Date: Sat, 6 Jun 2026 00:44:46 +0300 Subject: [PATCH 17/25] add immmune type PAMP/DAMP to slips evidence, STIX and IDMEF evidence --- modules/exporting_alerts/stix_exporter.py | 6 ++++++ modules/flow_alerts/set_evidence.py | 2 ++ slips/main.py | 1 + slips_files/core/evidence_handler_worker.py | 1 + slips_files/core/structures/evidence.py | 6 ++++++ 5 files changed, 16 insertions(+) diff --git a/modules/exporting_alerts/stix_exporter.py b/modules/exporting_alerts/stix_exporter.py index 875f0b5854..4a4605248f 100644 --- a/modules/exporting_alerts/stix_exporter.py +++ b/modules/exporting_alerts/stix_exporter.py @@ -649,6 +649,10 @@ def _build_custom_properties( if src_port: custom_properties["x_slips_src_port"] = src_port + immune_type = evidence.get("immune_type") + if immune_type: + custom_properties["x_slips_immune_type"] = src_port + return { key: value for key, value in custom_properties.items() @@ -656,6 +660,8 @@ def _build_custom_properties( } def _build_indicator(self, evidence: dict): + """this is where a single evidence is converted from slips format + to STIX format""" attacker = (evidence.get("attacker") or {}).get("value") if not attacker: attacker = (evidence.get("profile") or {}).get("ip") diff --git a/modules/flow_alerts/set_evidence.py b/modules/flow_alerts/set_evidence.py index 1a62708d90..bc17257ec1 100644 --- a/modules/flow_alerts/set_evidence.py +++ b/modules/flow_alerts/set_evidence.py @@ -16,6 +16,7 @@ EvidenceType, IoCType, Direction, + ImmuneType, ) ESTAB = "Established" @@ -620,6 +621,7 @@ def unknown_port(self, twid, flow) -> None: confidence=confidence, src_port=flow.sport, dst_port=flow.dport, + immune_type=ImmuneType.DAMP, ) self.db.set_evidence(evidence) diff --git a/slips/main.py b/slips/main.py index fae0def48a..2c315f98a5 100644 --- a/slips/main.py +++ b/slips/main.py @@ -428,6 +428,7 @@ def get_analyzed_flows_percentage(self) -> str: else: percentage = int(percentage) + percentage = min(100, percentage) # cap at 100% return f"Analyzed Flows: {green(percentage)}{green('%')}. " def is_total_flows_unknown(self) -> bool: diff --git a/slips_files/core/evidence_handler_worker.py b/slips_files/core/evidence_handler_worker.py index cf5cd51e7a..b2b7dac44e 100644 --- a/slips_files/core/evidence_handler_worker.py +++ b/slips_files/core/evidence_handler_worker.py @@ -129,6 +129,7 @@ def add_evidence_to_json_log_file( evidence.confidence ), "timewindow": evidence.timewindow.number, + "immune_type": evidence.immune_type, } ) } diff --git a/slips_files/core/structures/evidence.py b/slips_files/core/structures/evidence.py index 0b12a45e2e..6d1e9d6a78 100644 --- a/slips_files/core/structures/evidence.py +++ b/slips_files/core/structures/evidence.py @@ -116,6 +116,11 @@ def __str__(self): return self.name +class ImmuneType(Enum): + PAMP = auto() + DAMP = auto() + + class Direction(Enum): DST = auto() SRC = auto() @@ -282,6 +287,7 @@ class Evidence: timestamp: str = field( metadata={"validate": lambda x: validate_timestamp(x)} ) + immune_type: ImmuneType = field(default=None) interface: str = field(default="default") victim: Optional[Victim] = field(default=False) proto: Optional[Proto] = field(default=False) From c6d7c7f0dce12c6c7ad171d90e01867c2faa2ea5 Mon Sep 17 00:00:00 2001 From: alya Date: Sat, 6 Jun 2026 00:51:02 +0300 Subject: [PATCH 18/25] fix the links to idmefv2 docs/schema --- slips_files/common/idmefv2.py | 5 +++-- slips_files/core/structures/evidence.py | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/slips_files/common/idmefv2.py b/slips_files/common/idmefv2.py index f512b832da..d3af4b9712 100644 --- a/slips_files/common/idmefv2.py +++ b/slips_files/common/idmefv2.py @@ -42,7 +42,8 @@ class IDMEFv2: Class to convert Slips evidence and alerts to The Incident Detection Message Exchange Format version 2 (IDMEFv2 format). More Details about it here: - https://www.ietf.org/id/draft-lehmann-idmefv2-03.html#name-the-alert-class + https://datatracker.ietf.org/doc/draft-lehmann-idmefv2/#:~:text=Site%22%0A%20%20%20%20%20%20%20%7D%0A%20%20%20%20%20%5D%0A%20%20%20%7D%0A%0AAppendix%20C.-,JSON%20Validation%20Schema%20(Non%2Dnormative),-Listing%205%20contains + """ name = "idmefv2" @@ -187,7 +188,7 @@ def convert_to_idmef_event(self, evidence: Evidence) -> Message: The Incident Detection Message Exchange Format version 2 (IDMEFv2 format). More Details about it here: - https://www.ietf.org/id/draft-lehmann-idmefv2-03.html#name-the-alert-class + https://datatracker.ietf.org/doc/draft-lehmann-idmefv2/#:~:text=Site%22%0A%20%20%20%20%20%20%20%7D%0A%20%20%20%20%20%5D%0A%20%20%20%7D%0A%0AAppendix%20C.-,JSON%20Validation%20Schema%20(Non%2Dnormative),-Listing%205%20contains """ try: now = datetime.now(utils.local_tz).isoformat("T") diff --git a/slips_files/core/structures/evidence.py b/slips_files/core/structures/evidence.py index 6d1e9d6a78..beaf4701ac 100644 --- a/slips_files/core/structures/evidence.py +++ b/slips_files/core/structures/evidence.py @@ -249,7 +249,7 @@ def __repr__(self): class Method(Enum): """ Describes how was the evidence generated. these values are IDMEFv2 - https://www.ietf.org/id/draft-lehmann-idmefv2-03.html#section-5.3-4.20.1 + https://datatracker.ietf.org/doc/draft-lehmann-idmefv2/#:~:text=Site%22%0A%20%20%20%20%20%20%20%7D%0A%20%20%20%20%20%5D%0A%20%20%20%7D%0A%0AAppendix%20C.-,JSON%20Validation%20Schema%20(Non%2Dnormative),-Listing%205%20contains """ BIOMETRIC = "Biometric" From 5e3e4cf5b179ad9e7a7c458354b8475dce76aa7e Mon Sep 17 00:00:00 2001 From: alya Date: Sat, 6 Jun 2026 00:53:24 +0300 Subject: [PATCH 19/25] Dont set evidence for "unknown ports" when the proto isnt recognized as tcp or udp by Zeek --- modules/flow_alerts/conn.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/modules/flow_alerts/conn.py b/modules/flow_alerts/conn.py index 329874882a..3008b60267 100644 --- a/modules/flow_alerts/conn.py +++ b/modules/flow_alerts/conn.py @@ -192,6 +192,10 @@ def check_unknown_port(self, profileid, twid, flow): if not flow.dport: return + if "unknown_transport" in flow.proto: + # the protocol is unrecognized by zeek. not tcp or udp. + return + if self.db.is_a_port_scanner(flow.saddr, twid): # avoid setting unknown port evidence for each port # scanned from this attacker From 0fd169b672703f09cbd8db034772586aeed60399 Mon Sep 17 00:00:00 2001 From: alya Date: Sat, 6 Jun 2026 00:58:22 +0300 Subject: [PATCH 20/25] fix same user agent detected in "Multiple user agent" evidence --- modules/http_analyzer/http_analyzer.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/modules/http_analyzer/http_analyzer.py b/modules/http_analyzer/http_analyzer.py index 40bfceec31..eb60ea1b4d 100644 --- a/modules/http_analyzer/http_analyzer.py +++ b/modules/http_analyzer/http_analyzer.py @@ -372,6 +372,10 @@ def check_multiple_user_agents_in_a_row( return False ua: str = cached_ua.get("user_agent", "") + if ua == flow.user_agent: + # same UA is used again, nothing to alert about here + return False + self.set_evidence.multiple_user_agents_in_a_row(flow, ua, twid) return True From 257be1f58513db9b90817fb44e5c748df65866ac Mon Sep 17 00:00:00 2001 From: alya Date: Sat, 6 Jun 2026 02:03:59 +0300 Subject: [PATCH 21/25] ensure the detected gw ip belongs to the detected localnet before reporting it. --- slips_files/core/profiler_worker.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/slips_files/core/profiler_worker.py b/slips_files/core/profiler_worker.py index 9636766af7..1f68838303 100644 --- a/slips_files/core/profiler_worker.py +++ b/slips_files/core/profiler_worker.py @@ -394,6 +394,17 @@ def get_gw_ip_using_gw_mac(self, gw_mac) -> Optional[str]: # all of them are ipv6, return the first return gw_ips[0] + def gw_ip_belongs_to_localnet(self, gw_ip: str) -> bool: + """checks if the given detected gw_ip belongs to the detected local + network""" + for interface in utils.get_all_interfaces(self.args): + local_net = self.db.get_local_network(interface) + if not local_net: + continue + if gw_ip in ipaddress.ip_network(local_net): + return True + return False + def get_gateway_info(self, flow): """ Gets the IP and MAC of the gateway and stores them in the db @@ -429,7 +440,7 @@ def get_gateway_info(self, flow): # we need the mac to be set to be able to find the ip using it if not self.is_gw_info_detected("ip", flow.interface) and gw_mac_found: gw_ip: Optional[str] = self.get_gw_ip_using_gw_mac(flow.dmac) - if gw_ip: + if gw_ip and self.gw_ip_belongs_to_localnet(gw_ip): self.gw_ips[flow.interface] = gw_ip self.db.set_default_gateway("IP", gw_ip, flow.interface) self.print( From fe4192d9541d8ed92360598fbf4400515d5b8c56 Mon Sep 17 00:00:00 2001 From: alya Date: Sat, 6 Jun 2026 02:20:01 +0300 Subject: [PATCH 22/25] fix problem detecting if the gw ip belongs to localnetwork --- .../core/database/redis_db/profile_handler.py | 1 - slips_files/core/profiler_worker.py | 20 ++++++++-- .../slips_files/core/test_profiler_worker.py | 37 +++++++++++++++++++ 3 files changed, 54 insertions(+), 4 deletions(-) diff --git a/slips_files/core/database/redis_db/profile_handler.py b/slips_files/core/database/redis_db/profile_handler.py index f3267fc4b0..7c08268d52 100644 --- a/slips_files/core/database/redis_db/profile_handler.py +++ b/slips_files/core/database/redis_db/profile_handler.py @@ -1320,7 +1320,6 @@ def mark_profile_as_gateway(self, profileid): """ Used to mark this profile as dhcp server """ - self.set_profileid_field(profileid, self.constants.GATEWAY, "true") def set_ipv6_of_profile(self, profileid, ip: list): diff --git a/slips_files/core/profiler_worker.py b/slips_files/core/profiler_worker.py index 1f68838303..6ab86c2005 100644 --- a/slips_files/core/profiler_worker.py +++ b/slips_files/core/profiler_worker.py @@ -397,11 +397,21 @@ def get_gw_ip_using_gw_mac(self, gw_mac) -> Optional[str]: def gw_ip_belongs_to_localnet(self, gw_ip: str) -> bool: """checks if the given detected gw_ip belongs to the detected local network""" + try: + gw_ip_obj = ipaddress.ip_address(gw_ip) + except ValueError: + return False + for interface in utils.get_all_interfaces(self.args): local_net = self.db.get_local_network(interface) if not local_net: continue - if gw_ip in ipaddress.ip_network(local_net): + try: + local_net_obj = ipaddress.ip_network(local_net, strict=False) + except ValueError: + continue + + if gw_ip_obj in local_net_obj: return True return False @@ -423,7 +433,7 @@ def get_gateway_info(self, flow): if not gw_mac_found: # we didnt get the MAC of the GW of this flow's interface # ok consider the GW MAC = any dst MAC of a flow - # going from a private srcip -> a public ip + # going from a private srcip -> a public dstip if ( utils.is_private_ip(flow.saddr) and not utils.is_ignored_ip(flow.daddr) @@ -439,7 +449,11 @@ def get_gateway_info(self, flow): # we need the mac to be set to be able to find the ip using it if not self.is_gw_info_detected("ip", flow.interface) and gw_mac_found: - gw_ip: Optional[str] = self.get_gw_ip_using_gw_mac(flow.dmac) + gw_mac: Optional[str] = self.gw_macs.get(flow.interface) + if not gw_mac: + return + + gw_ip: Optional[str] = self.get_gw_ip_using_gw_mac(gw_mac) if gw_ip and self.gw_ip_belongs_to_localnet(gw_ip): self.gw_ips[flow.interface] = gw_ip self.db.set_default_gateway("IP", gw_ip, flow.interface) diff --git a/tests/unit/slips_files/core/test_profiler_worker.py b/tests/unit/slips_files/core/test_profiler_worker.py index 14f96f8fec..6861463eb2 100644 --- a/tests/unit/slips_files/core/test_profiler_worker.py +++ b/tests/unit/slips_files/core/test_profiler_worker.py @@ -395,6 +395,7 @@ def test_get_gateway_info_sets_mac_and_ip( profiler = ModuleFactory().create_profiler_worker_obj() profiler.is_gw_info_detected = Mock(side_effect=[False, False]) profiler.get_gw_ip_using_gw_mac = Mock(return_value="8.8.8.1") + profiler.gw_ip_belongs_to_localnet = Mock(return_value=True) mock_is_private_ip.return_value = True mock_is_ignored_ip.return_value = False flow = make_conn(dmac="00:11:22:33:44:55") @@ -428,6 +429,42 @@ def test_get_gateway_info_does_not_set_gateway_for_non_private_source(_mock): profiler.db.set_default_gateway.assert_not_called() +def test_get_gateway_info_uses_detected_gateway_mac_for_ip_lookup(): + profiler = ModuleFactory().create_profiler_worker_obj() + profiler.gw_macs = {"eth0": "00:11:22:33:44:55"} + profiler.is_gw_info_detected = Mock(side_effect=[True, False]) + profiler.get_gw_ip_using_gw_mac = Mock(return_value="192.168.1.1") + profiler.gw_ip_belongs_to_localnet = Mock(return_value=True) + + profiler.get_gateway_info(make_conn(dmac="66:77:88:99:aa:bb")) + + profiler.get_gw_ip_using_gw_mac.assert_called_once_with( + "00:11:22:33:44:55" + ) + profiler.db.set_default_gateway.assert_called_once_with( + "IP", "192.168.1.1", "eth0" + ) + + +@pytest.mark.parametrize( + "gw_ip, local_net, expected", + [ + ("192.168.1.1", "192.168.1.0/24", True), + ("10.0.0.1", "192.168.1.0/24", False), + ("not-an-ip", "192.168.1.0/24", False), + ], +) +@patch("slips_files.core.profiler_worker.utils.get_all_interfaces") +def test_gw_ip_belongs_to_localnet_handles_string_ips( + mock_get_all_interfaces, gw_ip, local_net, expected +): + profiler = ModuleFactory().create_profiler_worker_obj() + mock_get_all_interfaces.return_value = ["eth0"] + profiler.db.get_local_network.return_value = local_net + + assert profiler.gw_ip_belongs_to_localnet(gw_ip) is expected + + def test_get_gw_ip_using_gw_mac_prefers_ipv4(): profiler = ModuleFactory().create_profiler_worker_obj() profiler.db.get_ip_of_mac.return_value = json.dumps( From cb5b638f3b0f93a677865a99c0e78f6ec76648b3 Mon Sep 17 00:00:00 2001 From: alya Date: Sat, 6 Jun 2026 02:43:13 +0300 Subject: [PATCH 23/25] fix hanging on the pre_main of ip_info forever --- modules/ip_info/ip_info.py | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/modules/ip_info/ip_info.py b/modules/ip_info/ip_info.py index a2dfe59073..d7e5509f94 100644 --- a/modules/ip_info/ip_info.py +++ b/modules/ip_info/ip_info.py @@ -88,7 +88,7 @@ def subscribe_to_channels(self): "check_jarm_hash": self.c4, } - async def open_dbs(self): + def open_dbs(self) -> None: """Function to open the different offline databases used in this module. ASN, Country etc..""" # Open the maxminddb ASN offline db @@ -116,7 +116,18 @@ async def open_dbs(self): "https://dev.maxmind.com/geoip/geolite2-free-geolocation-data?lang=en. " "Please note it must be the MaxMind DB version." ) - self.create_task(self.read_mac_db) + self._start_mac_db_reader() + + def _start_mac_db_reader(self) -> None: + """ + Schedule the MAC vendor database reader on the active event loop. + """ + try: + asyncio.get_running_loop() + except RuntimeError: + return + + self.reading_mac_db_task = self.create_task(self.read_mac_db) async def read_mac_db(self): """ @@ -496,7 +507,6 @@ def get_gateway_ip_if_interface(self) -> Dict[str, str] | None: return interfaces: List[str] = utils.get_all_interfaces(self.args) - gw_ips = {} for interface in interfaces: try: @@ -518,6 +528,9 @@ def get_own_mac() -> str: def _get_wifi_interface_if_ap(self) -> str | None: ap_interfaces: str = self.db.get_wifi_interface() + if not ap_interfaces: + return None + try: # we're now sure that we're running in AP mode wifi_interface = ap_interfaces["wifi_interface"] @@ -733,18 +746,12 @@ def register_private_dns_server(self, flow: Any) -> bool: ) return True - def wait_for_dbs(self): + def wait_for_dbs(self) -> None: """ wait for update manager to finish updating the mac db and open the rest of dbs before starting this module """ - # this is the loop that controls tasks running on open_dbs - loop = asyncio.new_event_loop() - asyncio.set_event_loop(loop) - # run open_dbs in the background so we don't have - # to wait for update manager to finish updating the mac db to start - # this module - loop.run_until_complete(self.open_dbs()) + self.open_dbs() def set_evidence_malicious_jarm_hash( self, @@ -808,7 +815,7 @@ def set_evidence_malicious_jarm_hash( self.db.set_evidence(evidence) - def pre_main(self): + def pre_main(self) -> None: utils.drop_root_privs_permanently() self.wait_for_dbs() # the following method only works when running on an interface From cefb4b7e18a5223a8365055f05ec67f822978faa Mon Sep 17 00:00:00 2001 From: alya Date: Mon, 22 Jun 2026 22:04:59 +0300 Subject: [PATCH 24/25] update unit tests --- .../input_profilers/zeek_to_slips_maps.py | 2 + tests/unit/modules/ip_info/test_ip_info.py | 22 ++++ .../common/abstracts/test_iasync_module.py | 102 ++++++++++++++---- .../core/input/test_zeek_input_profiler.py | 100 ++++++++++++++++- 4 files changed, 206 insertions(+), 20 deletions(-) diff --git a/slips_files/core/input_profilers/zeek_to_slips_maps.py b/slips_files/core/input_profilers/zeek_to_slips_maps.py index da5b534337..3914b9dd25 100644 --- a/slips_files/core/input_profilers/zeek_to_slips_maps.py +++ b/slips_files/core/input_profilers/zeek_to_slips_maps.py @@ -14,6 +14,8 @@ "history": "history", "orig_pkts": "spkts", "resp_pkts": "dpkts", + "orig_l2_addr": "smac", + "resp_l2_addr": "dmac", "label": "ground_truth_label", "detailedlabel": "detailed_ground_truth_label", } diff --git a/tests/unit/modules/ip_info/test_ip_info.py b/tests/unit/modules/ip_info/test_ip_info.py index d3eae3ae7b..cbf9a1d10a 100644 --- a/tests/unit/modules/ip_info/test_ip_info.py +++ b/tests/unit/modules/ip_info/test_ip_info.py @@ -26,6 +26,28 @@ ) +def test_start_mac_db_reader_returns_without_running_loop() -> None: + module_factory = ModuleFactory() + ip_info = module_factory.create_ip_info_obj() + ip_info.create_task = Mock() + + ip_info._start_mac_db_reader() + + ip_info.create_task.assert_not_called() + + +async def test_start_mac_db_reader_schedules_reader_in_running_loop() -> None: + module_factory = ModuleFactory() + ip_info = module_factory.create_ip_info_obj() + created_task = Mock() + ip_info.create_task = Mock(return_value=created_task) + + ip_info._start_mac_db_reader() + + ip_info.create_task.assert_called_once_with(ip_info.read_mac_db) + assert ip_info.reading_mac_db_task == created_task + + @pytest.mark.parametrize( "ip_address, expected_geocountry", [ # Testcase 1: Valid IP address diff --git a/tests/unit/slips_files/common/abstracts/test_iasync_module.py b/tests/unit/slips_files/common/abstracts/test_iasync_module.py index 74f732c966..baafaee2cf 100644 --- a/tests/unit/slips_files/common/abstracts/test_iasync_module.py +++ b/tests/unit/slips_files/common/abstracts/test_iasync_module.py @@ -12,13 +12,9 @@ "exception", [KeyboardInterrupt(), SystemExit(), asyncio.CancelledError()], ) -def test_handle_task_exception_ignores_shutdown_exceptions(exception): - """ - Verify shutdown exceptions are not logged as task failures. - - :param exception: Shutdown exception raised or returned by a task. - :return: None. - """ +def test_handle_task_exception_ignores_shutdown_exceptions( + exception: BaseException, +) -> None: module_factory = ModuleFactory() flowalerts = module_factory.create_flowalerts_obj() task = Mock() @@ -29,12 +25,7 @@ def test_handle_task_exception_ignores_shutdown_exceptions(exception): flowalerts.print.assert_not_called() -def test_handle_task_exception_ignores_cancelled_tasks(): - """ - Verify cancelled tasks are not logged as task failures. - - :return: None. - """ +def test_handle_task_exception_ignores_cancelled_tasks() -> None: module_factory = ModuleFactory() flowalerts = module_factory.create_flowalerts_obj() task = Mock() @@ -45,12 +36,7 @@ def test_handle_task_exception_ignores_cancelled_tasks(): flowalerts.print.assert_not_called() -def test_handle_task_exception_logs_regular_exceptions(): - """ - Verify non-shutdown task exceptions are still logged. - - :return: None. - """ +def test_handle_task_exception_logs_regular_exceptions() -> None: module_factory = ModuleFactory() flowalerts = module_factory.create_flowalerts_obj() exception = ValueError("boom") @@ -66,3 +52,81 @@ def test_handle_task_exception_logs_regular_exceptions(): flowalerts.print_traceback_from_exception.assert_called_once_with( exception, task ) + + +@pytest.mark.parametrize( + ("exception", "expected"), + [ + (KeyboardInterrupt(), True), + (SystemExit(), True), + (asyncio.CancelledError(), True), + (ValueError("boom"), False), + (None, False), + ], +) +def test_is_shutdown_exception( + exception: BaseException | None, expected: bool +) -> None: + module_factory = ModuleFactory() + flowalerts = module_factory.create_flowalerts_obj() + + assert flowalerts.is_shutdown_exception(exception) is expected + + +def test_handle_loop_exception_ignores_shutdown_future() -> None: + module_factory = ModuleFactory() + flowalerts = module_factory.create_flowalerts_obj() + future = Mock() + future.result.side_effect = asyncio.CancelledError() + flowalerts.print_traceback = Mock() + + flowalerts.handle_loop_exception(Mock(), {"future": future}) + + flowalerts.print.assert_not_called() + flowalerts.print_traceback.assert_not_called() + + +def test_handle_loop_exception_logs_future_error() -> None: + module_factory = ModuleFactory() + flowalerts = module_factory.create_flowalerts_obj() + future = Mock() + future.result.side_effect = ValueError("boom") + flowalerts.print_traceback = Mock() + + flowalerts.handle_loop_exception(Mock(), {"future": future}) + + flowalerts.print_traceback.assert_called_once_with() + + +@pytest.mark.parametrize( + "exception", + [KeyboardInterrupt(), SystemExit(), asyncio.CancelledError()], +) +def test_handle_loop_exception_ignores_shutdown_exception( + exception: BaseException, +) -> None: + module_factory = ModuleFactory() + flowalerts = module_factory.create_flowalerts_obj() + + flowalerts.handle_loop_exception(Mock(), {"exception": exception}) + + flowalerts.print.assert_not_called() + + +def test_handle_loop_exception_logs_regular_exception() -> None: + module_factory = ModuleFactory() + flowalerts = module_factory.create_flowalerts_obj() + exception = ValueError("boom") + + flowalerts.handle_loop_exception(Mock(), {"exception": exception}) + + flowalerts.print.assert_called_once_with("Unhandled loop exception: boom") + + +def test_handle_loop_exception_logs_message() -> None: + module_factory = ModuleFactory() + flowalerts = module_factory.create_flowalerts_obj() + + flowalerts.handle_loop_exception(Mock(), {"message": "boom"}) + + flowalerts.print.assert_called_once_with("Unhandled loop error: boom") diff --git a/tests/unit/slips_files/core/input/test_zeek_input_profiler.py b/tests/unit/slips_files/core/input/test_zeek_input_profiler.py index 29f6d7fe80..5f4bbbdc48 100644 --- a/tests/unit/slips_files/core/input/test_zeek_input_profiler.py +++ b/tests/unit/slips_files/core/input/test_zeek_input_profiler.py @@ -3,7 +3,9 @@ from unittest.mock import Mock -from slips_files.core.input_profilers.zeek import ZeekJSON +import pytest + +from slips_files.core.input_profilers.zeek import ZeekJSON, ZeekTabs from tests.module_factory import ModuleFactory @@ -99,3 +101,99 @@ def test_zeek_json_maps_login_log_fields(): assert flow.confused is False assert flow.saddr == "147.32.80.40" assert flow.daddr == "147.32.80.37" + + +@pytest.mark.parametrize( + "src_mac,dst_mac", + [("00:0c:29:66:c7:82", "00:90:0b:7a:15:eb")], +) +def test_zeek_json_maps_conn_l2_addresses_to_mac_fields( + src_mac: str, dst_mac: str +) -> None: + """ + Test conn.log JSON l2 address fields are converted to MAC fields. + + :param src_mac: Source layer-2 address from Zeek conn.log. + :param dst_mac: Destination layer-2 address from Zeek conn.log. + :return: None. + """ + module_factory = ModuleFactory() + parser = ZeekJSON(module_factory.logger) + + flow, err = parser.process_line( + { + "type": "conn.log", + "interface": "default", + "data": { + "ts": 279.103822, + "uid": "CNybJS33LDUfyyg1Pi", + "id.orig_h": "10.0.2.15", + "id.orig_p": 44927, + "id.resp_h": "1.1.1.1", + "id.resp_p": 80, + "proto": "tcp", + "service": "http", + "duration": 0.5273809432983398, + "orig_bytes": 656, + "resp_bytes": 12310, + "conn_state": "SF", + "history": "ShADadFf", + "orig_pkts": 7, + "resp_pkts": 14, + "orig_l2_addr": src_mac, + "resp_l2_addr": dst_mac, + }, + } + ) + + assert err == "" + assert flow.smac == src_mac + assert flow.dmac == dst_mac + + +@pytest.mark.parametrize( + "src_mac,dst_mac", + [("08:00:27:ef:ee:34", "52:54:00:12:35:02")], +) +def test_zeek_tabs_maps_conn_l2_addresses_to_mac_fields( + src_mac: str, dst_mac: str +) -> None: + """ + Test conn.log tab l2 address fields are converted to MAC fields. + + :param src_mac: Source layer-2 address from Zeek conn.log. + :param dst_mac: Destination layer-2 address from Zeek conn.log. + :return: None. + """ + module_factory = ModuleFactory() + db = module_factory.logger + db.channels.NEW_ZEEK_FIELDS_LINE = "new_zeek_fields_line" + parser = ZeekTabs(db) + fields_line = ( + "#fields\tts\tuid\tid.orig_h\tid.orig_p\tid.resp_h\tid.resp_p\t" + "proto\tservice\tduration\torig_bytes\tresp_bytes\tconn_state\t" + "history\torig_pkts\tresp_pkts\torig_l2_addr\tresp_l2_addr" + ) + + flow, err = parser.process_line( + {"type": "conn.log", "interface": "default", "data": fields_line} + ) + assert flow is False + assert err == "Field line processed" + + flow, err = parser.process_line( + { + "type": "conn.log", + "interface": "default", + "data": ( + "904728.025376\tCIhV323VBG6udE1Ho3\t10.0.2.19\t" + "1701\t78.6.164.6\t2928\tudp\t-\t0.11151099996641278\t" + "196\t118\tSF\tDd\t1\t1\t" + f"{src_mac}\t{dst_mac}" + ), + } + ) + + assert err == "" + assert flow.smac == src_mac + assert flow.dmac == dst_mac From b7789fcae70d382bf28e477871646016138cee68 Mon Sep 17 00:00:00 2001 From: alya Date: Mon, 22 Jun 2026 22:10:32 +0300 Subject: [PATCH 25/25] dont use DAMP as the unknown port immune type --- modules/flow_alerts/set_evidence.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/modules/flow_alerts/set_evidence.py b/modules/flow_alerts/set_evidence.py index bc17257ec1..1a62708d90 100644 --- a/modules/flow_alerts/set_evidence.py +++ b/modules/flow_alerts/set_evidence.py @@ -16,7 +16,6 @@ EvidenceType, IoCType, Direction, - ImmuneType, ) ESTAB = "Established" @@ -621,7 +620,6 @@ def unknown_port(self, twid, flow) -> None: confidence=confidence, src_port=flow.sport, dst_port=flow.dport, - immune_type=ImmuneType.DAMP, ) self.db.set_evidence(evidence)