Merge pull request #1833 from stratosphereips/develop

Slips v1.1.18

Author: AlyaGomaa

.github/actions/list-tests-dir/action.yml

@@ -0,0 +1,25 @@
+name: "List Test Files"
+description: "Build JSON matrix of test files"
+inputs:
+  test_dir:
+    description: "Directory to search for test_*.py files"
+    required: true
+  output_prefix:
+    description: "Optional prefix to prepend to each file entry in the output array"
+    required: false
+    default: ""
+outputs:
+  test_files:
+    description: "JSON array of integration test files (relative to tests/)"
+    value: ${{ steps.set-matrix.outputs.files }}
+
+runs:
+  using: "composite"
+  steps:
+    - id: set-matrix
+      shell: bash
+      run: |
+        # Find files in test_dir/ and optionally prefix entries for consumers.
+        PREFIX="${{ inputs.output_prefix }}"
+        FILES=$(find "${{ inputs.test_dir }}" -name "test_*.py" -printf "%P\n" | sed "s|^|${PREFIX}|" | jq -R -s -c 'split("\n")[:-1]')
+        echo "files=$FILES" >> "$GITHUB_OUTPUT"

.github/workflows/integration-tests.yml

@@ -7,7 +7,20 @@ on:
       - 'develop'
 
 jobs:
+  list-integration-tests:
+    runs-on: ubuntu-latest
+    outputs:
+      test_files: ${{ steps.list.outputs.test_files }}
+    steps:
+      - uses: actions/checkout@v5
+      - id: list
+        uses: ./.github/actions/list-tests-dir
+        with:
+          test_dir: tests/integration
+          output_prefix: integration/
+
   integration-tests:
+    needs: list-integration-tests
     runs-on: ubuntu-22.04
     timeout-minutes: 1800
 
@@ -23,33 +36,28 @@ jobs:
       TF_ENABLE_ONEDNN_OPTS: 0
 
     strategy:
+      fail-fast: false
       matrix:
-        test_file:
-          - test_config_files.py
-          - test_portscans.py
-          - test_dataset.py
-          - test_pcap_dataset.py
-          - test_zeek_dataset.py
-          - test_fides.py
-          - test_iris.py
+        test_file: ${{ fromJson(needs.list-integration-tests.outputs.test_files) }}
 
     steps:
-    - uses: actions/checkout@v5
-      with:
-        ref: ${{ github.ref }}
-        fetch-depth: ''
-
-    - name: Start Redis
-      uses: ./.github/actions/start-redis
-
-    - name: Run Integration Tests for ${{ matrix.test_file }}
-      run: |
-        python3 -m pytest tests/integration_tests/${{ matrix.test_file }} -p no:warnings -vv -s -n 3
-
-    - name: Upload Artifacts
-      if: always()
-      uses: actions/upload-artifact@v5
-      with:
-        name: ${{ matrix.test_file }}-integration-tests-output
-        path: |
-          output/integration_tests
+      - uses: actions/checkout@v5
+        with:
+          ref: ${{ github.ref }}
+          fetch-depth: 0
+
+      - name: Start Redis
+        uses: ./.github/actions/start-redis
+
+      - name: Run Integration Tests for ${{ matrix.test_file }}
+        run: |
+          python3 -m pytest tests/${{ matrix.test_file }} -p no:warnings -vv -s -n 3
+
+      - name: Upload Artifacts
+        if: always()
+        uses: actions/upload-artifact@v5
+        with:
+          # Replaces slashes with underscores for valid artifact naming
+          name: ${{ github.run_id }}-${{ strategy.job-index }}-integration-output
+          path: |
+            output/integration

.github/workflows/publish-slips-images.yml

@@ -43,6 +43,8 @@ jobs:
     needs: setup
     runs-on: ubuntu-22.04
     strategy:
+      # prevents the whole job from being canceled if one entry fails
+      fail-fast: false
       matrix:
         image_type:
           - name: slips

.github/workflows/unit-tests.yml

@@ -7,110 +7,57 @@ on:
       - 'develop'
 
 jobs:
+  # This job finds all test files and creates a JSON array for the matrix
+  list-tests:
+    runs-on: ubuntu-latest
+    timeout-minutes: 120
+    outputs:
+      test_files: ${{ steps.list.outputs.test_files }}
+    steps:
+      - uses: actions/checkout@v5
+      - id: list
+        uses: ./.github/actions/list-tests-dir
+        with:
+          test_dir: tests/unit
 
   unit-tests:
+    needs: list-tests
     runs-on: ubuntu-22.04
     timeout-minutes: 120
 
+    # Do not stop other tests if one file fails
+    strategy:
+      fail-fast: false
+      matrix:
+        test_file: ${{ fromJson(needs.list-tests.outputs.test_files) }}
+
     container:
       image: stratosphereips/slips_dependencies
       # TensorFlow + multiprocessing + pytest -n can hit /dev/shm limits.
       options: --shm-size=2g
 
-
     env:
       TF_CPP_MIN_LOG_LEVEL: 3
       TF_ENABLE_ONEDNN_OPTS: 0
 
-    strategy:
-      matrix:
-        test_file:
-          - test_input.py
-          - test_main.py
-          - test_conn.py
-          - test_downloaded_file.py
-          - test_ssl.py
-          - test_tunnel.py
-          - test_ssh.py
-          - test_dns.py
-          - test_notice.py
-          - test_software.py
-          - test_smtp.py
-          - test_whitelist.py
-          - test_arp.py
-          - test_arp_poisoner.py
-          - test_arp_filter.py
-          - test_blocking.py
-          - test_unblocker.py
-          - test_flow_handler.py
-          - test_horizontal_portscans.py
-          - test_http_analyzer.py
-          - test_vertical_portscans.py
-          - test_network_discovery.py
-          - test_virustotal.py
-          - test_update_file_manager.py
-          - test_threat_intelligence.py
-          - test_slips_utils.py
-          - test_slips.py
-          - test_profiler.py
-          - test_profiler_worker.py
-          - test_profilers_manager.py
-          - test_leak_detector.py
-          - test_ip_info.py
-          - test_evidence.py
-          - test_asn_info.py
-          - test_urlhaus.py
-          - test_markov_chain.py
-          - test_daemon.py
-          - test_go_director.py
-          - test_notify.py
-          - test_checker.py
-          - test_base_model.py
-          - test_set_evidence.py
-          - test_trustdb.py
-          - test_cesnet.py
-          - test_output.py
-          - test_riskiq.py
-          - test_spamhaus.py
-          - test_scan_detections_db.py
-          - test_circllu.py
-          - test_evidence_handler.py
-          - test_evidence_logger.py
-          - test_evidence_formatter.py
-          - test_alert_handler.py
-          - test_redis_manager.py
-          - test_ioc_handler.py
-          - test_timeline.py
-          - test_database.py
-          - test_symbols_handler.py
-          - test_profile_handler.py
-          - test_process_manager.py
-          - test_metadata_manager.py
-          - test_host_ip_manager.py
-          - test_rnn_cc_detection.py
-          - test_idea_format.py
-          - test_fides_sqlite_db.py
-          - test_fides_module.py
-          - test_fides_queues.py
-          - test_fides_bridge.py
-
     steps:
-    - uses: actions/checkout@v5
-      with:
-        ref: ${{ github.ref }}
-        fetch-depth: 0
-
-    - name: Start Redis
-      uses: ./.github/actions/start-redis
-
-    - name: Run Unit Tests for ${{ matrix.test_file }}
-      run: |
-        python3 -m pytest tests/${{ matrix.test_file }} -p no:warnings -vv -s -n 5
-
-    - name: Upload Artifacts
-      if: always()
-      uses: actions/upload-artifact@v5
-      with:
-        name: test_slips_locally-unit-tests-output
-        path: |
-          output/unit_tests
+      - uses: actions/checkout@v5
+        with:
+          ref: ${{ github.ref }}
+          fetch-depth: 0
+
+      - name: Start Redis
+        uses: ./.github/actions/start-redis
+
+      - name: Run Unit Tests for ${{ matrix.test_file }}
+        run: |
+          # The path is reconstructed here using the matrix variable
+          python3 -m pytest tests/unit/${{ matrix.test_file }} -p no:warnings -vv -s -n 5
+
+      - name: Upload Artifacts
+        if: always()
+        uses: actions/upload-artifact@v5
+        with:
+          name: test_slips-output-${{ strategy.job-index }}
+          path: |
+            output/unit_tests

.gitignore

@@ -177,4 +177,4 @@ appendonly.aof
 /slipsOut/metadata/slips.yaml
 /slipsOut/metadata/whitelist.conf
 /p2p_db.sqlite
-
+old-pipeline/

.gitmodules

@@ -18,3 +18,4 @@
 [submodule "SlipsWeb"]
 	path = SlipsWeb
 	url = https://github.com/stratosphereips/SlipsWeb.git
+    branch = master

.pre-commit-config.yaml

@@ -22,7 +22,6 @@ repos:
 -     repo: https://github.com/astral-sh/ruff-pre-commit
       rev: v0.9.6
       hooks:
-        # Run the linter.
         - id: ruff
           args: [ --fix ]
           # excludes formatting slips_files/common/imports.py

.secrets.baseline

@@ -149,7 +149,7 @@
         "filename": "config/slips.yaml",
         "hashed_secret": "4cac50cee3ad8e462728e711eac3e670753d5016",
         "is_verified": false,
-        "line_number": 222
+        "line_number": 268
       }
     ],
     "dataset/test14-malicious-zeek-dir/http.log": [
@@ -7185,5 +7185,5 @@
       }
     ]
   },
-  "generated_at": "2025-12-11T01:08:53Z"
+  "generated_at": "2026-03-02T22:46:58Z"
 }

CHANGELOG.md

@@ -1,3 +1,12 @@
+1.1.18 (Mar 3rd, 2026)
+
+* Add the HTTPS anomaly detection module with adaptive baselines, confidence scoring, and detailed evidence reasons.
+* Enable ADWIN drift detection by default for HTTPS anomaly detection, with separate hourly and flow drift paths.
+* Add a local HTML report generator for HTTPS anomaly detection logs, with interactive charts and anomaly summaries.
+* Improve performance under high-throughput traffic and reduced OOM risk.
+* Improve Redis memory hygiene with tighter TTLs, capped zsets, periodic cleanups.
+* Cap non-HTTP port 80 checks in the HTTP analyzer to reduce resource spikes.
+
 1.1.17 (Jan 30th, 2025)
 
 * Improve horizontal, vertical, and ICMP portscan detection logic and speed.

README.md

@@ -1,5 +1,5 @@
 <h1 align="center">
-Slips v1.1.16
+Slips v1.1.18
 </h1>
 
 
@@ -184,6 +184,7 @@ Slips key features are:
 * **Targeted Attacks and Command & Control Detection**: It places a strong emphasis on identifying targeted attacks and command and control channels in network traffic.
 * **Traffic Analysis Flexibility**: Slips can analyze network traffic in real-time, PCAP files, and network flows from popular tools like Suricata, Zeek/Bro, and Argus.
 * **Threat Intelligence Updates**: Slips continuously updates threat intelligence files and databases, providing relevant detections as updates occur.
+* **HTTPS Anomaly Detection**: Adaptive TLS/HTTPS anomaly detection with drift handling and a local HTML report generator for deep dives.
 * **Integration with External Platforms**: Modules in Slips can look up IP addresses on external platforms such as VirusTotal and RiskIQ.
 * **Graphical User Interface**: Slips provides a console graphical user interface (Kalipso) and a web interface for displaying detection with graphs and tables.
 * **Peer-to-Peer (P2P) Module**: Slips includes a complex automatic system to find other peers in the network and share IoC data automatically in a balanced, trusted manner. The P2P module can be enabled as needed.