Merge pull request #1410 from stratosphereips/alya/support_reading_labeled_logs

Support reading labeled logs

Author: AlyaGomaa

.github/workflows/unit-tests.yml

@@ -26,7 +26,7 @@ jobs:
     strategy:
       matrix:
         test_file:
-          - test_inputProc.py
+          - test_input.py
           - test_main.py
           - test_conn.py
           - test_downloaded_file.py

slips/main.py

@@ -286,22 +286,18 @@ def get_input_file_type(self, given_path):
         elif "CSV" in cmd_result and os.path.isfile(given_path):
             input_type = "binetflow"
         elif "directory" in cmd_result and os.path.isdir(given_path):
-            from slips_files.core.input import SUPPORTED_LOGFILES
-
             for log_file in os.listdir(given_path):
                 # if there is at least 1 supported log file inside the
                 # given directory, start slips normally
                 # otherwise, stop slips
-                if log_file.replace(".log", "") in SUPPORTED_LOGFILES:
+                if not utils.is_ignored_zeek_log_file(log_file):
                     input_type = "zeek_folder"
                     break
             else:
-                # zeek dir filled with unsupported logs
-                # or .labeled logs that slips can't read.
                 print(
                     f"Log files in {given_path} are not supported \n"
                     f"Make sure all log files inside the given "
-                    f"directory end with .log .. Stopping."
+                    f"directory end with .log or .log.labeled .. Stopping."
                 )
                 sys.exit(-1)
         else:

slips_files/common/slips_utils.py

@@ -28,6 +28,8 @@
 from dataclasses import is_dataclass, asdict
 from enum import Enum
 
+from slips_files.core.supported_logfiles import SUPPORTED_LOGFILES
+
 IS_IN_A_DOCKER_CONTAINER = os.environ.get("IS_IN_A_DOCKER_CONTAINER", False)
 
 
@@ -289,6 +291,23 @@ def drop_root_privs(self):
         os.setresuid(sudo_uid, sudo_uid, -1)
         return
 
+    def is_ignored_zeek_log_file(self, filepath: str) -> bool:
+        """
+        Returns true if the given file ends with .log or .log.labeled and
+        is in SUPPORTED_LOGFILES list
+        :param filepath: a zeek log file
+        """
+        if not (
+            filepath.endswith(".log") or filepath.endswith(".log.labeled")
+        ):
+            return True
+
+        filename = os.path.basename(filepath)
+        # remove all extensions from filename
+        while "." in filename:
+            filename = filename.rsplit(".", 1)[0]
+        return filename not in SUPPORTED_LOGFILES
+
     def convert_format(self, ts, required_format: str):
         """
         Detects and converts the given ts to the given format

slips_files/core/input.py

@@ -26,7 +26,6 @@
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 # Contact: eldraco@gmail.com, sebastian.garcia@agents.fel.cvut.cz, stratosphere@aic.fel.cvut.cz
-from pathlib import Path
 from re import split
 
 from watchdog.observers import Observer
@@ -38,23 +37,7 @@
 from slips_files.common.slips_utils import utils
 import multiprocessing
 from slips_files.core.helpers.filemonitor import FileEventHandler
-
-SUPPORTED_LOGFILES = (
-    "conn",
-    "dns",
-    "http",
-    "ssl",
-    "ssh",
-    "dhcp",
-    "ftp",
-    "smtp",
-    "tunnel",
-    "notice",
-    "files",
-    "arp",
-    "software",
-    "weird",
-)
+from slips_files.core.supported_logfiles import SUPPORTED_LOGFILES
 
 
 class Input(ICore):
@@ -198,14 +181,6 @@ def check_if_time_to_del_rotated_files(self):
                     pass
             self.to_be_deleted = []
 
-    def is_ignored_file(self, filepath: str) -> bool:
-        """
-        Ignore zeek log files that we don't use
-        :param filepath: full path to a zeek log file
-        """
-        filename_without_ext = Path(filepath).stem
-        return filename_without_ext not in SUPPORTED_LOGFILES
-
     def get_file_handle(self, filename):
         # Update which files we know about
         try:
@@ -373,7 +348,7 @@ def read_zeek_files(self) -> int:
             # Go to all the files generated by Zeek and read 1
             # line from each of them
             for filename in self.zeek_files:
-                if self.is_ignored_file(filename):
+                if utils.is_ignored_zeek_log_file(filename):
                     continue
 
                 # reads 1 line from the given file and cache it
@@ -464,7 +439,7 @@ def read_zeek_folder(self):
             full_path = os.path.join(self.given_path, file)
 
             # exclude ignored files from the total flows to be processed
-            if self.is_ignored_file(full_path):
+            if utils.is_ignored_zeek_log_file(full_path):
                 continue
 
             if not growing_zeek_dir:
@@ -602,14 +577,15 @@ def handle_zeek_log_file(self):
          and conn.log flows given to slips through CYST unix socket.
         """
         if (
-            not self.given_path.endswith(".log")
-            or self.is_ignored_file(self.given_path)
-        ) and "cyst" not in self.given_path.lower():
+            utils.is_ignored_zeek_log_file(self.given_path)
+            and "cyst" not in self.given_path.lower()
+        ):
             # unsupported file
             return False
 
         if os.path.exists(self.given_path):
-            # in case of CYST flows, the given path is 'cyst' and there's no way to get the total flows
+            # in case of CYST flows, the given path is 'cyst' and there's no
+            # way to get the total flows
             self.is_zeek_tabs = self.is_zeek_tabs_file(self.given_path)
             total_flows = self.get_flows_number(self.given_path)
             self.db.set_input_metadata({"total_flows": total_flows})

slips_files/core/supported_logfiles.py

@@ -0,0 +1,16 @@
+SUPPORTED_LOGFILES = (
+    "conn",
+    "dns",
+    "http",
+    "ssl",
+    "ssh",
+    "dhcp",
+    "ftp",
+    "smtp",
+    "tunnel",
+    "notice",
+    "files",
+    "arp",
+    "software",
+    "weird",
+)

tests/test_inputProc.py tests/test_input.pytests/test_inputProc.py renamed to tests/test_input.py

@@ -333,49 +333,6 @@ def test_give_profiler(line, input_type, expected_line, expected_input_type):
     assert line_sent["input_type"] == expected_input_type
 
 
-@pytest.mark.parametrize(
-    "filepath, expected_result",
-    [  # Testcase 1: Supported file
-        ("path/to/conn.log", False),
-        # Testcase 2: Supported file
-        ("path/to/dns.log", False),
-        # Testcase 3: Supported file
-        ("path/to/http.log", False),
-        # Testcase 4: Supported file
-        ("path/to/ssl.log", False),
-        # Testcase 5: Supported file
-        ("path/to/ssh.log", False),
-        # Testcase 6: Supported file
-        ("path/to/dhcp.log", False),
-        # Testcase 7: Supported file
-        ("path/to/ftp.log", False),
-        # Testcase 8: Supported file
-        ("path/to/smtp.log", False),
-        # Testcase 9: Supported file
-        ("path/to/tunnel.log", False),
-        # Testcase 10: Supported file
-        ("path/to/notice.log", False),
-        # Testcase 11: Supported file
-        ("path/to/files.log", False),
-        # Testcase 12: Supported file
-        ("path/to/arp.log", False),
-        # Testcase 13: Supported file
-        ("path/to/software.log", False),
-        # Testcase 14: Supported file
-        ("path/to/weird.log", False),
-        # Testcase 15: Unsupported file
-        ("path/to/unsupported.log", True),
-    ],
-)
-def test_is_ignored_file(filepath, expected_result):
-    """
-    Test that the is_ignored_file method correctly
-    identifies ignored Zeek log files.
-    """
-    input_process = ModuleFactory().create_input_obj("", "zeek_log_file")
-    assert input_process.is_ignored_file(filepath) == expected_result
-
-
 def test_get_file_handle_existing_file():
     """
     Test that the get_file_handle method correctly

tests/test_slips_utils.py

@@ -27,6 +27,37 @@ def test_get_sha256_hash_from_nonexistent_file():
         utils.get_sha256_hash_of_file_contents("nonexistent_file.txt")
 
 
+@pytest.mark.parametrize(
+    "filepath, expected_result",
+    [  # Testcase 1: Supported file
+        ("path/to/conn.log", False),
+        ("path/to/dns.log", False),
+        ("path/to/http.log", False),
+        ("path/to/ssl.log", False),
+        ("path/to/ssh.log", False),
+        ("path/to/dhcp.log", False),
+        ("path/to/ftp.log", False),
+        ("path/to/smtp.log", False),
+        ("path/to/tunnel.log", False),
+        ("path/to/notice.log", False),
+        ("path/to/files.log", False),
+        ("path/to/arp.log", False),
+        ("path/to/software.log", False),
+        ("path/to/software.log.labeled", False),
+        ("path/to/weird.log", False),
+        ("path/to/software.log.labeled.something", True),
+        ("path/to/unsupported.log", True),
+    ],
+)
+def test_is_ignored_zeek_log_file(filepath, expected_result):
+    """
+    Test that the is_ignored_file method correctly
+    identifies ignored Zeek log files.
+    """
+    utils = ModuleFactory().create_utils_obj()
+    assert utils.is_ignored_zeek_log_file(filepath) == expected_result
+
+
 def test_get_sha256_hash_permission_error():
     utils = ModuleFactory().create_utils_obj()
     with patch("builtins.open", side_effect=PermissionError):