Replace httpx/curl HTTP infrastructure with blasthttp

- Add blasthttp (>=0.1.3) as HTTP engine, remove httpx subprocess dependency
- Remove HTTPEngine subprocess, all HTTP now in-process via shared blasthttp client
- Remove curl helper, use request() with resolve_ip and request_target
- Remove obsolete ffuf module (replaced by web_brute)
- Remove obsolete httpx module (replaced by http)
- Add native http module using blasthttp batch API
- Add native web_brute module using blasthttp batch API
- Add web_brute_shortnames module
- Add generic_ssrf module
- Rewrite sslcert to use blasthttp cert_info
- Add blasthttp mock infrastructure for tests
- Add resolve_ip passthrough in test conftest for localhost
- Add rate limit tests
- Add 5-minute default timeout for downloads
- Rename output http module to webhook
- Fix elastic output module import
- Update all module tests for blasthttp mock API

Author: liquidsec

.gitignore

@@ -3,3 +3,4 @@ __pycache__/
 /data/
 /neo4j/
 .DS_Store
+pytest_debug.log

AGENTS.md

@@ -146,6 +146,7 @@ Key helpers:
 | Helper | What it does |
 |--------|-------------|
 | `self.helpers.request(url)` | Make an HTTP request (with retries, SSL handling, etc.) |
+| `self.helpers.blasthttp` | Shared blasthttp client (rate-limited via `web.http_rate_limit` config) |
 | `self.helpers.resolve(host)` | DNS resolution |
 | `self.helpers.is_ip(s)` | Check if string is an IP |
 | `self.helpers.is_dns_name(s)` | Check if string is a hostname |
@@ -219,7 +220,7 @@ from .base import ModuleTestBase
 
 class TestMyModule(ModuleTestBase):
     async def setup_after_prep(self, module_test):
-        module_test.httpx_mock.add_response(
+        module_test.blasthttp_mock.add_response(
             url="https://api.example.com/lookup?domain=blacklanternsecurity.com",
             json={"emails": ["info@blacklanternsecurity.com"]},
         )
@@ -370,7 +371,7 @@ Whether to process seed events (the initial targets provided to the scan).
 Whether to accept "special" URLs (e.g. JavaScript files) that are not normally distributed to web modules.
 
 ```python
-# httpx.py - needs to process all URLs including special ones
+# http.py - needs to process all URLs including special ones
 accept_url_special = True
 ```
 
@@ -535,7 +536,7 @@ _preserve_graph = True
 Exclude this module from scan statistics. Used by output and report modules.
 
 ##### `_disable_auto_module_deps` (bool) -- default: `False`
-Prevent BBOT from automatically enabling dependency modules. For example, if your module watches `URL` events, BBOT normally auto-enables `httpx`. Set this to `True` to prevent that.
+Prevent BBOT from automatically enabling dependency modules. For example, if your module watches `URL` events, BBOT normally auto-enables `http`. Set this to `True` to prevent that.
 
 ---
 
@@ -875,7 +876,7 @@ class TestMyModule(ModuleTestBase):
     targets = ["http://127.0.0.1:8888"]
 
     # Optional: override which modules are enabled
-    modules_overrides = ["httpx", "my_module"]
+    modules_overrides = ["http", "my_module"]
 
     # Optional: override config
     config_overrides = {"modules": {"my_module": {"some_option": True}}}
@@ -887,7 +888,7 @@ class TestMyModule(ModuleTestBase):
     async def setup_after_prep(self, module_test):
         """Called AFTER the scan is prepared. Modify modules, add mocks here."""
         # Mock an HTTP response
-        module_test.httpx_mock.add_response(
+        module_test.blasthttp_mock.add_response(
             url="https://api.example.com/lookup?domain=blacklanternsecurity.com",
             json={"results": ["sub.blacklanternsecurity.com"]},
         )
@@ -920,7 +921,7 @@ The test lifecycle runs:
 
 ### Test Utilities
 
-- **`module_test.httpx_mock`** - mock HTTP responses (from pytest-httpx)
+- **`module_test.blasthttp_mock`** - mock HTTP responses 
 - **`module_test.httpserver`** - real HTTP server on port 8888
 - **`module_test.httpserver_ssl`** - real HTTPS server on port 9999
 - **`module_test.mock_dns(data)`** - mock DNS responses
@@ -933,7 +934,7 @@ Real example -- `test_module_robots.py`:
 ```python
 class TestRobots(ModuleTestBase):
     targets = ["http://127.0.0.1:8888"]
-    modules_overrides = ["httpx", "robots"]
+    modules_overrides = ["http", "robots"]
     config_overrides = {"modules": {"robots": {"include_sitemap": True}}}
 
     async def setup_after_prep(self, module_test):

bbot/core/event/base.py

@@ -101,8 +101,8 @@ class BaseEvent:
             "parent": "OPEN_TCP_PORT:cf7e6a937b161217eaed99f0c566eae045d094c7",
             "tags": ["in-scope", "distance-0", "dir", "status-301"],
             "http_title": "301 Moved Permanently",
-            "module": "httpx",
-            "module_sequence": "httpx"
+            "module": "http",
+            "module_sequence": "http"
         }
         ```
     """

bbot/core/helpers/command.py

@@ -1,6 +1,7 @@
 import os
 import asyncio
 import logging
+import contextlib
 import traceback
 from signal import SIGINT
 from subprocess import CompletedProcess, CalledProcessError, SubprocessError
@@ -157,7 +158,18 @@ async def run_live(self, *command, check=False, text=True, idle_timeout=None, **
                     command_str = " ".join(command)
                     log.warning(f"Stderr for run_live({command_str}):\n\t{stderr}")
         finally:
-            proc_tracker.remove(proc)
+            proc_tracker.discard(proc)
+            # Kill the subprocess if it's still running (e.g. generator was cancelled/closed)
+            if proc.returncode is None:
+                with contextlib.suppress(Exception):
+                    proc.terminate()
+                try:
+                    await asyncio.wait_for(proc.wait(), timeout=5)
+                except (asyncio.TimeoutError, Exception):
+                    with contextlib.suppress(Exception):
+                        proc.kill()
+                if input_task is not None:
+                    input_task.cancel()
 
 
 async def _spawn_proc(self, *command, **kwargs):
@@ -270,7 +282,7 @@ def _prepare_command_kwargs(self, command, kwargs):
         >>> _prepare_command_kwargs(['ls', '-l'], {'sudo': True})
         (['sudo', '-E', '-A', 'LD_LIBRARY_PATH=...', 'PATH=...', 'ls', '-l'], {'limit': 104857600, 'stdout': -1, 'stderr': -1, 'env': environ(...)})
     """
-    # limit = 100MB (this is needed for cases like httpx that are sending large JSON blobs over stdout)
+    # limit = 100MB (this is needed for cases that are sending large JSON blobs over stdout)
     if "limit" not in kwargs:
         kwargs["limit"] = 1024 * 1024 * 100
     if "stdout" not in kwargs:

bbot/core/helpers/diff.py

@@ -148,7 +148,7 @@ def compare_headers(self, headers_1, headers_2):
             for x in list(ddiff[k]):
                 try:
                     header_value = str(x).split("'")[1]
-                except KeyError:
+                except (KeyError, IndexError):
                     continue
                 differing_headers.append(header_value)
         return differing_headers

bbot/core/helpers/helper.py

@@ -86,12 +86,14 @@ def __init__(self, preset):
         self.process_pool = ProcessPoolExecutor(max_workers=num_processes)
 
         self._cloud = None
+        self._blasthttp_client = None
 
         self.re = RegexHelper(self)
         self.yara = YaraHelper(self)
         self.simhash = SimHashHelper()
         self._dns = None
         self._web = None
+        self._asn = None
         self._cloudcheck = None
         self._asn = None
         self.config_aware_validators = self.validators.Validators(self)
@@ -117,6 +119,17 @@ def asn(self):
             self._asn = ASNHelper(self)
         return self._asn
 
+    @property
+    def blasthttp(self):
+        if self._blasthttp_client is None:
+            import blasthttp as _blasthttp
+
+            self._blasthttp_client = _blasthttp.BlastHTTP()
+            rate_limit = self.web_config.get("http_rate_limit", 0)
+            if rate_limit:
+                self._blasthttp_client.set_rate_limit(rate_limit)
+        return self._blasthttp_client
+
     @property
     def cloudcheck(self):
         if self._cloudcheck is None:

bbot/core/helpers/misc.py

@@ -1661,7 +1661,7 @@ def rm_rf(f, ignore_errors=False):
         f (str or Path): The directory path to delete.
 
     Examples:
-        >>> rm_rf("/tmp/httpx98323849")
+        >>> rm_rf("/tmp/bbot98323849")
     """
     import shutil
 

bbot/core/helpers/names_generator.py

@@ -25,6 +25,7 @@
     "blazed",
     "bloodshot",
     "brown",
+    "cantankerous",
     "cheeky",
     "childish",
     "chiseled",
@@ -52,6 +53,7 @@
     "demented",
     "demonic",
     "demonstrative",
+    "derpy",
     "depraved",
     "depressed",
     "deranged",
@@ -103,6 +105,7 @@
     "glutinous",
     "golden",
     "gothic",
+    "greasy",
     "grievous",
     "gummy",
     "hallucinogenic",
@@ -137,11 +140,14 @@
     "intoxicated",
     "inventive",
     "irritable",
+    "janky",
+    "lackadaisical",
     "large",
     "liquid",
     "loveable",
     "lovely",
     "lucid",
+    "lumpy",
     "malevolent",
     "malfunctioning",
     "malicious",
@@ -221,6 +227,7 @@
     "sinful",
     "sinister",
     "slippery",
+    "sloppy",
     "sly",
     "sneaky",
     "soft",
@@ -311,6 +318,7 @@
     "amir",
     "amy",
     "andrea",
+    "andres",
     "andrew",
     "angela",
     "ann",
@@ -344,6 +352,7 @@
     "bradley",
     "brandon",
     "brandybuck",
+    "brendan",
     "brenda",
     "brian",
     "brianna",
@@ -399,6 +408,7 @@
     "diana",
     "diane",
     "dobby",
+    "dominic",
     "donald",
     "donna",
     "dooku",