Merge branch 'blasthttp-integration-clean' into add-virtualhost-module

Author: liquidsec

bbot/core/shared_deps.py

@@ -1,15 +1,3 @@
-DEP_FFUF = [
-    {
-        "name": "Download ffuf",
-        "unarchive": {
-            "src": "https://github.com/ffuf/ffuf/releases/download/v#{BBOT_DEPS_FFUF_VERSION}/ffuf_#{BBOT_DEPS_FFUF_VERSION}_#{BBOT_OS}_#{BBOT_CPU_ARCH_GOLANG}.tar.gz",
-            "include": "ffuf",
-            "dest": "#{BBOT_TOOLS}",
-            "remote_src": True,
-        },
-    }
-]
-
 DEP_DOCKER = [
     {
         "name": "Check if Docker is already installed",
@@ -173,31 +161,6 @@
     },
 ]
 
-DEP_CURL = [
-    {
-        "name": "Download static curl binary (v8.11.0)",
-        "get_url": {
-            "url": "https://github.com/moparisthebest/static-curl/releases/download/v8.11.0/curl-amd64",
-            "dest": "#{BBOT_TOOLS}/curl",
-            "mode": "0755",
-            "force": True,
-        },
-    },
-    {
-        "name": "Ensure curl binary is executable",
-        "file": {
-            "path": "#{BBOT_TOOLS}/curl",
-            "mode": "0755",
-        },
-    },
-    {
-        "name": "Verify curl binary works",
-        "command": "#{BBOT_TOOLS}/curl --version",
-        "register": "curl_version_output",
-        "changed_when": False,
-    },
-]
-
 DEP_MASSCAN = [
     {
         "name": "install os deps (Debian)",
@@ -269,7 +232,7 @@
     },
 ]
 
-# shared module dependencies -- ffuf, massdns, chromium, etc.
+# shared module dependencies -- massdns, chromium, etc.
 SHARED_DEPS = {}
 for var, val in list(locals().items()):
     if var.startswith("DEP_") and isinstance(val, list):

bbot/defaults.yml

@@ -132,8 +132,6 @@ engine:
 
 # Tool dependencies
 deps:
-  ffuf:
-    version: "2.1.0"
   # How to handle installation of module dependencies
   # Choices are:
   #  - abort_on_failure (default) - if a module dependency fails to install, abort the scan

bbot/modules/telerik.py

@@ -156,7 +156,7 @@ class telerik(BaseModule):
     options = {"exploit_RAU_crypto": False, "include_subdirs": False}
     options_desc = {
         "exploit_RAU_crypto": "Attempt to confirm any RAU AXD detections are vulnerable",
-        "include_subdirs": "Include subdirectories in the scan (off by default)",  # will create many finding events if used in conjunction with web spider or ffuf
+        "include_subdirs": "Include subdirectories in the scan (off by default)",  # will create many finding events if used in conjunction with web spider or web_brute
     }
 
     in_scope_only = True

bbot/presets/web/dirbust-heavy.yml

@@ -7,16 +7,16 @@ flags:
   - iis-shortnames
 
 modules:
-  - ffuf
+  - web_brute
   - wayback
 
 config:
   modules:
     iis_shortnames:
-      # we exploit the shortnames vulnerability to produce URL_HINTs which are consumed by ffuf_shortnames
+      # we exploit the shortnames vulnerability to produce URL_HINTs which are consumed by web_brute_shortnames
       detect_only: False
-    ffuf:
-      depth: 3
+    web_brute:
+      max_depth: 3
       lines: 5000
       extensions:
         - php

bbot/presets/web/dirbust-light.yml

@@ -4,10 +4,10 @@ include:
   - iis-shortnames
 
 modules:
-  - ffuf
+  - web_brute
 
 config:
   modules:
-    ffuf:
+    web_brute:
       # wordlist size = 1000
       lines: 1000

bbot/presets/web/dotnet-audit.yml

@@ -7,19 +7,19 @@ include:
 modules:
   - http
   - badsecrets
-  - ffuf_shortnames
-  - ffuf
+  - web_brute_shortnames
+  - web_brute
   - telerik
   - ajaxpro
   - dotnetnuke
   - aspnet_bin_exposure
 
 config:
   modules:
-    ffuf:
+    web_brute:
       extensions: asp,aspx,ashx,asmx,ascx
-      extensions_ignore_case: True
-    ffuf_shortnames:
+      ignore_case: True
+    web_brute_shortnames:
       find_subwords: True
     telerik:
       exploit_RAU_crypto: True

bbot/test/test_step_1/test_scan.py

@@ -227,8 +227,8 @@ def mock_event(type, module, parent=None):
 
     http_mod = mock_module("http", ["URL", "HTTP_RESPONSE"])
     excavate_mod = mock_module("excavate", ["URL_UNVERIFIED", "WEB_PARAMETER"])
-    ffuf_mod = mock_module("ffuf_shortnames", ["URL_UNVERIFIED"])
-    ffuf2_mod = mock_module("ffuf", ["URL_UNVERIFIED"])
+    web_brute_shortnames_mod = mock_module("web_brute_shortnames", ["URL_UNVERIFIED"])
+    web_brute_mod = mock_module("web_brute", ["URL_UNVERIFIED"])
     speculate_mod = mock_module("speculate", ["DNS_NAME", "OPEN_TCP_PORT", "IP_ADDRESS", "FINDING", "ORG_STUB"])
     robots_mod = mock_module("robots", ["URL_UNVERIFIED"])
 
@@ -237,13 +237,13 @@ def mock_event(type, module, parent=None):
         parent = mock_event("URL_UNVERIFIED", excavate_mod)
         stats.event_produced(mock_event("URL", http_mod, parent=parent))
 
-    # 2) ffuf_shortnames discovers URL_UNVERIFIED, http verifies → ffuf_shortnames gets credit
+    # 2) web_brute_shortnames discovers URL_UNVERIFIED, http verifies → web_brute_shortnames gets credit
     for _ in range(3):
-        parent = mock_event("URL_UNVERIFIED", ffuf_mod)
+        parent = mock_event("URL_UNVERIFIED", web_brute_shortnames_mod)
         stats.event_produced(mock_event("URL", http_mod, parent=parent))
 
-    # 3) ffuf discovers URL_UNVERIFIED, http verifies → ffuf gets credit
-    parent = mock_event("URL_UNVERIFIED", ffuf2_mod)
+    # 3) web_brute discovers URL_UNVERIFIED, http verifies → web_brute gets credit
+    parent = mock_event("URL_UNVERIFIED", web_brute_mod)
     stats.event_produced(mock_event("URL", http_mod, parent=parent))
 
     # 4) speculate (internal module) creates URL_UNVERIFIED, http verifies → http keeps credit
@@ -267,8 +267,8 @@ def mock_event(type, module, parent=None):
 
     # verify per-module produced counts
     assert stats.module_stats["excavate"].produced == {"URL": 5}
-    assert stats.module_stats["ffuf_shortnames"].produced == {"URL": 3}
-    assert stats.module_stats["ffuf"].produced == {"URL": 1}
+    assert stats.module_stats["web_brute_shortnames"].produced == {"URL": 3}
+    assert stats.module_stats["web_brute"].produced == {"URL": 1}
     assert stats.module_stats["robots"].produced == {"URL": 2}
     # http gets credit for speculate's 4 URLs + 2 from OPEN_TCP_PORT = 6
     assert stats.module_stats["http"].produced == {"URL": 6}
@@ -286,9 +286,9 @@ def mock_event(type, module, parent=None):
     table_dict = {row[0]: row[1] for row in rows}
     assert table_dict["http"] == "6 (6 URL)"
     assert table_dict["excavate"] == "5 (5 URL)"
-    assert table_dict["ffuf_shortnames"] == "3 (3 URL)"
+    assert table_dict["web_brute_shortnames"] == "3 (3 URL)"
     assert table_dict["robots"] == "2 (2 URL)"
-    assert table_dict["ffuf"] == "1 (1 URL)"
+    assert table_dict["web_brute"] == "1 (1 URL)"
     assert table_dict["CNAME"] == "1 (1 DNS_NAME)"
     assert table_dict["cloudcheck"] == "1 (1 STORAGE_BUCKET)"
     assert "speculate" not in table_dict

bbot/test/test_step_2/module_tests/test_module_github_org.py

@@ -2,7 +2,9 @@
 
 
 class TestGithub_Org(ModuleTestBase):
-    config_overrides = {"modules": {"github_org": {"api_key": "asdf"}, "github": {"api_key": "asdf"}, "git_clone": {"api_key": ""}}}
+    config_overrides = {
+        "modules": {"github_org": {"api_key": "asdf"}, "github": {"api_key": "asdf"}, "git_clone": {"api_key": ""}}
+    }
     modules_overrides = ["github_org", "speculate"]
 
     async def setup_before_prep(self, module_test):

bbot/test/test_step_2/module_tests/test_module_rabbitmq.py

@@ -1,6 +1,5 @@
 import json
 import asyncio
-from contextlib import suppress
 
 from .base import ModuleTestBase
 

bbot/test/test_step_2/module_tests/test_module_subdomainradar.py

@@ -204,6 +204,7 @@ async def setup_after_prep(self, module_test):
                 "time_to_finish": 41,
             },
         )
+
     def check(self, module_test, events):
         assert any(e.data == "www.blacklanternsecurity.com" for e in events), "Failed to detect subdomain #1"
         assert any(e.data == "asdf.blacklanternsecurity.com" for e in events), "Failed to detect subdomain #2"