Merge pull request #1928 from stratosphereips/alya/improve_integration_tests

Improve integration tests

Author: AlyaGomaa

.github/workflows/integration-tests.yml

@@ -53,11 +53,18 @@ jobs:
         run: |
           python3 -m pytest tests/${{ matrix.test_file }} -p no:warnings -vv -s -n 3
 
+      - name: Build Artifact Name
+        # otherwise we get numeric names for the artifacts and we dont know which is which
+        id: artifact-name
+        run: |
+          sanitized_test_file="${{ matrix.test_file }}"
+          sanitized_test_file=$(printf '%s\n' "$sanitized_test_file" | tr '/' '_')
+          echo "name=${sanitized_test_file}-integration-output" >> "$GITHUB_OUTPUT"
+
       - name: Upload Artifacts
         if: always()
         uses: actions/upload-artifact@v6
         with:
-          # Replaces slashes with underscores for valid artifact naming
-          name: ${{ github.run_id }}-${{ strategy.job-index }}-integration-output
+          name: ${{ steps.artifact-name.outputs.name }}
           path: |
-            output/integration
+            output/integration_tests

config/TI_feeds.csv

@@ -11,7 +11,7 @@ https://mcfp.felk.cvut.cz/publicDatasets/CTU-AIPP-BlackList/Todays-Blacklists/AI
 https://mcfp.felk.cvut.cz/publicDatasets/CTU-AIPP-BlackList/Todays-Blacklists/AIP_historical_blacklist_prioritized_by_newest_attackers.csv,medium, ['phishing','honeypot']
 https://raw.githubusercontent.com/stratosphereips/Civilsphere/main/threatintel/strangereallintel-cyberthreatintel.csv,medium, ['phishing']
 https://raw.githubusercontent.com/AssoEchap/stalkerware-indicators/master/generated/network.csv,medium, ['stalkerware']
-https://raw.githubusercontent.com/stratosphereips/Civilsphere/main/threatintel/adserversandtrackers.csv,medium, ['adtrackers']
+https://raw.githubusercontent.com/stratosphereips/Civilsphere/main/threatintel/adserversandtrackers.csv,info, ['adtrackers']
 https://raw.githubusercontent.com/stratosphereips/Civilsphere/main/threatintel/civilsphereindicators.csv,medium, ['apt']
 https://raw.githubusercontent.com/botherder/targetedthreats/master/targetedthreats.csv,medium, ['apt']
 https://osint.digitalside.it/Threat-Intel/lists/latestdomains.txt,medium, ['honeypot']
@@ -26,10 +26,10 @@ https://lists.blocklist.de/lists/mail.txt,medium, ['honeypot']
 https://lists.blocklist.de/lists/bruteforcelogin.txt,medium, ['honeypot']
 https://feodotracker.abuse.ch/downloads/ipblocklist.csv,medium, ['honeypot']
 https://reputation.alienvault.com/reputation.generic,medium, ['honeypot']
-https://raw.githubusercontent.com/anudeepND/blacklist/master/adservers.txt,medium, ['adtrackers']
+https://raw.githubusercontent.com/anudeepND/blacklist/master/adservers.txt,info, ['adtrackers']
 # bigdargon: Hosts block ads of Vietnamese
-https://raw.githubusercontent.com/bigdargon/hostsVN/master/option/domain.txt,medium, ['adtrackers']
-https://raw.githubusercontent.com/SweetSophia/mifitxiaomipiholelist/master/mifitblocklist.txt,medium, ['xiaomi-trackers']
+https://raw.githubusercontent.com/bigdargon/hostsVN/master/option/domain.txt,info, ['adtrackers']
+https://raw.githubusercontent.com/SweetSophia/mifitxiaomipiholelist/master/mifitblocklist.txt,info, ['xiaomi-trackers']
 https://raw.githubusercontent.com/CriticalPathSecurity/Zeek-Intelligence-Feeds/master/abuse-ch-ipblocklist.intel,medium, ['honeypot']
 https://raw.githubusercontent.com/CriticalPathSecurity/Zeek-Intelligence-Feeds/master/alienvault.intel,medium, ['honeypot']
 https://raw.githubusercontent.com/CriticalPathSecurity/Zeek-Intelligence-Feeds/master/cobaltstrike_ips.intel,medium, ['honeypot']

dataset/test1-normal.nfdump dataset/test1-malicious.nfdumpdataset/test1-normal.nfdump renamed to dataset/test1-malicious.nfdump

@@ -25,7 +25,11 @@ To be able to use the fides module, you should use ```--cap-add=NET_ADMIN```
 If you plan on using the Fides Module, please be aware that it is used only if Slips is running on an interface OR on a growing Zeek directory. The `--use_fides=True` is ignored when Slips is run on a file.
 
 ## Configuration
-The evaluation model used, the evaluation thresholds, and other configurations are located in ```fides.conf.yml``` file
+The evaluation model used, the evaluation thresholds, and other configurations are located in ```modules/fides/config/fides.conf.yml```.
+
+If you need a Slips run to use a different Fides configuration file, set
+```global_p2p.fides_conf``` in Slips config to the relative path
+of that alternate YAML file.
 
 **Possible threat intelligence evaluation models**
 

docs/fides.md

@@ -1,6 +1,7 @@
 # SPDX-FileCopyrightText: 2021 Sebastian Garcia <sebastian.garcia@agents.fel.cvut.cz>
 # SPDX-License-Identifier: GPL-2.0-only
 import logging
+import shutil
 import subprocess
 import time
 from threading import Lock
@@ -30,6 +31,8 @@ class ARPPoisoner(IModule):
     authors = ["Alya Gomaa"]
 
     def init(self):
+        self.arp_scan_path = shutil.which("arp-scan")
+        self.arp_scan_bin_available = self.arp_scan_path is not None
         self._time_since_last_repoison = {}
         self._time_since_last_internet_cut = {}
         self.log_file_path = self.get_module_specific_output_path(
@@ -56,13 +59,32 @@ def init(self):
         self.ip_interface_map = {}
 
     def subscribe_to_channels(self):
+        if not self.arp_scan_bin_available:
+            self.channels = {}
+            return
         self.c1 = self.db.subscribe("new_blocking")
         self.c2 = self.db.subscribe("tw_closed")
         self.channels = {
             "new_blocking": self.c1,
             "tw_closed": self.c2,
         }
 
+    def pre_main(self) -> bool:
+        """
+        Stop the module before entering the main loop when arp-scan is
+        unavailable.
+
+        :return: True when the module should shut down, otherwise False.
+        """
+        if self.arp_scan_bin_available:
+            return False
+
+        self.print(
+            "The arp-scan tool is not installed. ARP poisoner module is "
+            "stopping.",
+        )
+        return True
+
     def log(self, text):
         """Logs the given text to the blocking log file"""
         with self.blocking_logfile_lock:

modules/arp_poisoner/arp_poisoner.py

@@ -53,8 +53,11 @@ def init(self):
 
         # load trust model configuration
         current_dir = Path(__file__).resolve().parent
-        config_path = current_dir / "config" / "fides.conf.yml"
-        self.__trust_model_config = load_configuration(config_path.__str__())
+        default_config_path = current_dir / "config" / "fides.conf.yml"
+        config_path = self.conf.read_configuration(
+            "global_p2p", "fides_conf", str(default_config_path)
+        )
+        self.__trust_model_config = load_configuration(config_path)
 
         # prepare variables for global protocols
         self.__bridge: NetworkBridge

modules/fides/fides.py

@@ -1,6 +1,6 @@
 from typing import Dict, List, Callable, Optional, Union
 
-
+from slips_files.common.slips_utils import utils
 from ..messaging.dacite import from_dict
 
 from ..messaging.model import (
@@ -28,7 +28,7 @@ class MessageHandler:
     # def print(self, *args, **kwargs):
     #    return self.printer.print(*args, **kwargs)
 
-    version = 1
+    version = utils.get_current_version()
 
     def __init__(
         self,

modules/fides/messaging/message_handler.py

@@ -13,7 +13,7 @@
 @dataclass
 class NetworkMessage:
     type: str
-    version: int
+    version: str
     data: Any
 
 

modules/fides/messaging/model.py

@@ -2,6 +2,7 @@
 from dataclasses import asdict
 from typing import Dict, List
 
+from slips_files.common.slips_utils import utils
 from .dacite import from_dict
 
 from .message_handler import MessageHandler
@@ -24,7 +25,7 @@ class NetworkBridge:
     execute "listen" method.
     """
 
-    version = 1
+    version = utils.get_current_version()
 
     def __init__(self, queue: Queue):
         self.__queue = queue
@@ -36,14 +37,15 @@ def listen(self, handler: MessageHandler, block: bool = False):
         """
 
         def message_received(message: str):
+            """this is the callback that executes every new msg"""
             try:
-                # with open("fides_nb.txt", "a") as f:
-                #     f.write(message)
+
                 logger.debug("New message received! Trying to parse.")
                 parsed = json.loads(message)
                 network_message = from_dict(
                     data_class=NetworkMessage, data=parsed
                 )
+
                 logger.debug("Message parsed. Executing handler.")
                 handler.on_message(network_message)
             except Exception as e:

modules/fides/messaging/network_bridge.py

@@ -3,8 +3,11 @@
 
 from ..model.aliases import PeerId, OrganisationId
 from ..model.peer import PeerInfo
-from ..model.recommendation_history import RecommendationHistory
-from ..model.service_history import ServiceHistory
+from ..model.recommendation_history import (
+    RecommendationHistory,
+    RecommendationHistoryRecord,
+)
+from ..model.service_history import ServiceHistory, ServiceHistoryRecord
 
 
 @dataclass
@@ -121,6 +124,7 @@ def to_dict(self, remove_histories: bool = False):
     # Method to create an object from a dictionary
     @classmethod
     def from_dict(cls, data):
+        """Create a PeerTrustData instance from a dictionary payload."""
         return cls(
             info=PeerInfo.from_dict(
                 data["info"]
@@ -135,14 +139,13 @@ def from_dict(cls, data):
                 "initial_reputation_provided_by_count"
             ],
             service_history=[
-                ServiceHistory.from_dict(sh) for sh in data["service_history"]
+                ServiceHistoryRecord.from_dict(sh)
+                for sh in data["service_history"]
             ],
-            # Assuming ServiceHistory has from_dict
             recommendation_history=[
-                RecommendationHistory.from_dict(rh)
+                RecommendationHistoryRecord.from_dict(rh)
                 for rh in data["recommendation_history"]
             ],
-            # Assuming RecommendationHistory has from_dict
         )