Merge branch 'dev' into docs-update

Author: liquidsec

README.md

@@ -1,6 +1,6 @@
 [![bbot_banner](https://github.com/user-attachments/assets/f02804ce-9478-4f1e-ac4d-9cf5620a3214)](https://github.com/blacklanternsecurity/bbot)
 
-[![Python Version](https://img.shields.io/badge/python-3.9+-FF8400)](https://www.python.org) [![License](https://img.shields.io/badge/license-AGPLv3-FF8400.svg)](https://github.com/blacklanternsecurity/bbot/blob/dev/LICENSE) [![DEF CON Recon Village 2024](https://img.shields.io/badge/DEF%20CON%20Demo%20Labs-2023-FF8400.svg)](https://www.reconvillage.org/talks) [![PyPi Downloads](https://static.pepy.tech/personalized-badge/bbot?right_color=orange&left_color=grey)](https://pepy.tech/project/bbot) [![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff) [![Tests](https://github.com/blacklanternsecurity/bbot/actions/workflows/tests.yml/badge.svg?branch=stable)](https://github.com/blacklanternsecurity/bbot/actions?query=workflow%3A"tests") [![Codecov](https://codecov.io/gh/blacklanternsecurity/bbot/branch/dev/graph/badge.svg?token=IR5AZBDM5K)](https://codecov.io/gh/blacklanternsecurity/bbot) [![Discord](https://img.shields.io/discord/859164869970362439)](https://discord.com/invite/PZqkgxu5SA)
+[![Python Version](https://img.shields.io/badge/python-3.10+-FF8400)](https://www.python.org) [![License](https://img.shields.io/badge/license-AGPLv3-FF8400.svg)](https://github.com/blacklanternsecurity/bbot/blob/dev/LICENSE) [![DEF CON Recon Village 2024](https://img.shields.io/badge/DEF%20CON%20Demo%20Labs-2023-FF8400.svg)](https://www.reconvillage.org/talks) [![PyPi Downloads](https://static.pepy.tech/personalized-badge/bbot?right_color=orange&left_color=grey)](https://pepy.tech/project/bbot) [![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff) [![Tests](https://github.com/blacklanternsecurity/bbot/actions/workflows/tests.yml/badge.svg?branch=stable)](https://github.com/blacklanternsecurity/bbot/actions?query=workflow%3A"tests") [![Codecov](https://codecov.io/gh/blacklanternsecurity/bbot/branch/dev/graph/badge.svg?token=IR5AZBDM5K)](https://codecov.io/gh/blacklanternsecurity/bbot) [![Discord](https://img.shields.io/discord/859164869970362439)](https://discord.com/invite/PZqkgxu5SA)
 
 ### **BEE·bot** is a multipurpose scanner inspired by [Spiderfoot](https://github.com/smicallef/spiderfoot), built to automate your **Recon**, **Bug Bounties**, and **ASM**!
 

bbot/core/event/base.py

@@ -2005,6 +2005,31 @@ def _data_human(self):
         return tech
 
 
+class VIRTUAL_HOST(DictHostEvent):
+    class _data_validator(BaseModel):
+        host: str
+        virtual_host: str
+        url: Optional[str] = None
+        description: Optional[str] = None
+        ip: Optional[str] = None
+        _validate_url = field_validator("url")(validators.validate_url)
+        _validate_host = field_validator("host")(validators.validate_host)
+
+    def _data_id(self):
+        virtual_host = self.data.get("virtual_host", "")
+        return f"{self.host}:{virtual_host}"
+
+    def _pretty_string(self):
+        return self.data.get("virtual_host", "")
+
+    def _data_human(self):
+        virtual_host = self.data.get("virtual_host", "")
+        url = self.data.get("url", "")
+        if url:
+            return f"{virtual_host} ({url})"
+        return virtual_host
+
+
 class PROTOCOL(DictHostEvent):
     class _data_validator(BaseModel):
         host: str

bbot/core/helpers/git.py

@@ -1,12 +1,27 @@
 from pathlib import Path
 
 
+_SAFE_GIT_CONFIG = """\
+[core]
+\trepositoryformatversion = 0
+\tfilemode = true
+\tbare = false
+\tlogallrefupdates = true
+\tfsmonitor = false
+\tsymlinks = false
+\tsshCommand = echo
+[transfer]
+\tfsckObjects = true
+"""
+
+
 def sanitize_git_repo(repo_folder: Path):
-    # sanitizing the git config is infeasible since there are too many different ways to do evil things
-    # instead, we move it out of .git and into the repo folder, so we don't miss any secrets etc. inside
+    # replace the git config with a safe one that neutralizes dangerous directives
+    # the original is preserved in the repo folder so secret-scanning tools can still inspect it
     config_file = repo_folder / ".git" / "config"
     if config_file.exists():
         config_file.rename(repo_folder / "git_config_original")
+    config_file.write_text(_SAFE_GIT_CONFIG)
     # leave .git/index in place -- it's binary metadata (filename-to-SHA mappings),
     # not a security risk, and removing it breaks tools that need to clone the repo
     # move the hooks folder

bbot/core/helpers/simhash.py

@@ -11,10 +11,11 @@ def __init__(self, bits=64):
     @staticmethod
     def compute_simhash(text, bits=64, truncate=True, normalization_filter=None):
         """
-        Static method for computing SimHash that can be used with multiprocessing.
+        Static method for computing a SimHash fingerprint.
 
-        This method is designed to be used with run_in_executor_mp() for CPU-intensive
-        SimHash computations across multiple processes.
+        Designed to be called via run_in_executor_cpu(): the work is short and the
+        input is truncated to ~3KB inside the helper, so a thread pool avoids the
+        pickle/spawn overhead of a process pool.
 
         Args:
             text (str): The text to hash

bbot/core/modules.py

@@ -30,6 +30,12 @@
 
 log = logging.getLogger("bbot.module_loader")
 
+
+class _SafeUnpickler(pickle.Unpickler):
+    def find_class(self, module, name):
+        raise pickle.UnpicklingError(f"Forbidden class: {module}.{name}")
+
+
 bbot_code_dir = Path(__file__).parent.parent
 
 
@@ -450,7 +456,7 @@ def preload_cache(self):
             if self.preload_cache_file.is_file():
                 with suppress(Exception):
                     with open(self.preload_cache_file, "rb") as f:
-                        self._preload_cache = pickle.load(f)
+                        self._preload_cache = _SafeUnpickler(f).load()
         return self._preload_cache
 
     @preload_cache.setter

bbot/errors.py

@@ -38,10 +38,6 @@ class WordlistError(BBOTError):
     pass
 
 
-class CurlError(BBOTError):
-    pass
-
-
 class PresetNotFoundError(BBOTError):
     pass
 

bbot/modules/apkpure.py

@@ -49,6 +49,9 @@ async def handle_event(self, event):
 
     async def download_apk(self, app_id):
         path = None
+        if "/" in app_id or "\\" in app_id or ".." in app_id:
+            self.warning(f"Unsafe app_id, skipping: {app_id}")
+            return path
         url = f"https://d.apkpure.com/b/XAPK/{app_id}?version=latest"
         self.helpers.mkdir(self.output_dir / app_id)
         response = await self.helpers.request(url, allow_redirects=True)

bbot/modules/aspnet_bin_exposure.py

@@ -12,13 +12,19 @@ class aspnet_bin_exposure(BaseModule):
     }
 
     in_scope_only = True
+    _module_threads = 2
+
     test_dlls = [
         "Telerik.Web.UI.dll",
         "Newtonsoft.Json.dll",
         "System.Net.Http.dll",
         "EntityFramework.dll",
         "AjaxControlToolkit.dll",
     ]
+    _techniques = [
+        "b/(S(X))in/###DLL_PLACEHOLDER###/(S(X))/",
+        "(S(X))/b/(S(X))in/###DLL_PLACEHOLDER###",
+    ]
 
     @staticmethod
     def normalize_url(url):
@@ -27,54 +33,52 @@ def normalize_url(url):
     def _incoming_dedup_hash(self, event):
         return hash(self.normalize_url(event.url))
 
+    @staticmethod
+    def _is_dll_download(response):
+        return (
+            response is not None
+            and response.status_code == 200
+            and "content-type" in response.headers
+            and "application/x-msdownload" in response.headers["content-type"]
+        )
+
     async def handle_event(self, event):
         normalized_url = self.normalize_url(event.url)
+        kwargs = {"method": "GET", "allow_redirects": False, "timeout": 10}
+
+        probes = []
         for test_dll in self.test_dlls:
-            for technique in ["b/(S(X))in/###DLL_PLACEHOLDER###/(S(X))/", "(S(X))/b/(S(X))in/###DLL_PLACEHOLDER###"]:
+            for technique in self._techniques:
                 test_url = f"{normalized_url}{technique.replace('###DLL_PLACEHOLDER###', test_dll)}"
-                self.debug(f"Sending test URL: [{test_url}]")
-                kwargs = {"method": "GET", "allow_redirects": False, "timeout": 10}
-                test_result = await self.helpers.request(test_url, **kwargs)
-                if test_result:
-                    if test_result.status_code == 200 and (
-                        "content-type" in test_result.headers
-                        and "application/x-msdownload" in test_result.headers["content-type"]
-                    ):
-                        self.debug(
-                            f"Got positive result for probe with test url: [{test_url}]. Status Code: [{test_result.status_code}] Content Length: [{len(test_result.content)}]"
-                        )
+                probes.append((test_url, kwargs, technique))
+
+        async for test_url, test_result, technique in self.helpers.request_batch_stream(probes, threads=10):
+            if not self._is_dll_download(test_result):
+                continue
+
+            self.debug(
+                f"Got positive result for probe with test url: [{test_url}]. Status Code: [{test_result.status_code}] Content Length: [{len(test_result.content)}]"
+            )
 
-                        if test_result.status_code == 200 and (
-                            "content-type" in test_result.headers
-                            and "application/x-msdownload" in test_result.headers["content-type"]
-                        ):
-                            confirm_url = (
-                                f"{normalized_url}{technique.replace('###DLL_PLACEHOLDER###', 'oopsnotarealdll.dll')}"
-                            )
-                            confirm_result = await self.helpers.request(confirm_url, **kwargs)
+            confirm_url = f"{normalized_url}{technique.replace('###DLL_PLACEHOLDER###', 'oopsnotarealdll.dll')}"
+            confirm_result = await self.helpers.request(confirm_url, **kwargs)
 
-                            if confirm_result and (
-                                confirm_result.status_code != 200
-                                or not (
-                                    "content-type" in confirm_result.headers
-                                    and "application/x-msdownload" in confirm_result.headers["content-type"]
-                                )
-                            ):
-                                description = f"IIS Bin Directory DLL Exposure. Detection Url: [{test_url}]"
-                                await self.emit_event(
-                                    {
-                                        "name": "IIS Bin Directory DLL Exposure",
-                                        "severity": "HIGH",
-                                        "confidence": "HIGH",
-                                        "host": str(event.host),
-                                        "url": normalized_url,
-                                        "description": description,
-                                    },
-                                    "FINDING",
-                                    event,
-                                    context="{module} detected IIS Bin Directory DLL Exposure vulnerability",
-                                )
-                                return True
+            if confirm_result and not self._is_dll_download(confirm_result):
+                description = f"IIS Bin Directory DLL Exposure. Detection Url: [{test_url}]"
+                await self.emit_event(
+                    {
+                        "name": "IIS Bin Directory DLL Exposure",
+                        "severity": "HIGH",
+                        "confidence": "HIGH",
+                        "host": str(event.host),
+                        "url": normalized_url,
+                        "description": description,
+                    },
+                    "FINDING",
+                    event,
+                    context="{module} detected IIS Bin Directory DLL Exposure vulnerability",
+                )
+                return True
 
     async def filter_event(self, event):
         if "dir" in event.tags:

bbot/modules/docker_pull.py

@@ -83,14 +83,13 @@ async def docker_api_request(self, url: str):
             response = await self.helpers.request(url, headers=self.headers, follow_redirects=True)
             if response is not None and response.status_code != 401:
                 return response
-            try:
-                www_authenticate_headers = response.headers.get("www-authenticate", "")
-                realm = www_authenticate_headers.split('realm="')[1].split('"')[0]
-                service = www_authenticate_headers.split('service="')[1].split('"')[0]
-                scope = www_authenticate_headers.split('scope="')[1].split('"')[0]
-            except (KeyError, IndexError):
+            www_auth = response.headers.get("www-authenticate", "")
+            realm, service, scope = self._parse_www_authenticate(www_auth)
+            if not all([realm, service, scope]):
                 self.log.warning(f"Could not obtain realm, service or scope from {url}")
                 break
+            if not self._validate_realm(url, realm):
+                break
             auth_url = f"{realm}?service={service}&scope={scope}"
             auth_response = await self.helpers.request(auth_url)
             if not auth_response:
@@ -101,6 +100,29 @@ async def docker_api_request(self, url: str):
             self.headers.update({"Authorization": f"Bearer {token}"})
         return None
 
+    @staticmethod
+    def _parse_www_authenticate(header):
+        value = header
+        if value.lower().startswith("bearer "):
+            value = value[7:]
+        params = {}
+        for part in value.split(","):
+            part = part.strip()
+            if "=" in part:
+                key, _, val = part.partition("=")
+                params[key.strip()] = val.strip('"')
+        return params.get("realm", ""), params.get("service", ""), params.get("scope", "")
+
+    def _validate_realm(self, registry_url, realm):
+        registry_host = self.helpers.urlparse(registry_url).hostname or ""
+        realm_host = self.helpers.urlparse(realm).hostname or ""
+        _, registry_domain = self.helpers.split_domain(registry_host)
+        _, realm_domain = self.helpers.split_domain(realm_host)
+        if not realm_domain or realm_domain != registry_domain:
+            self.warning(f"Auth realm TLD ({realm_domain}) does not match registry TLD ({registry_domain}), skipping")
+            return False
+        return True
+
     async def get_tags(self, registry, repository):
         url = f"{registry}/v2/{repository}/tags/list"
         r = await self.docker_api_request(url)

bbot/modules/generic_ssrf.py

@@ -123,8 +123,7 @@ async def test(self, event):
         post_data_list = [(subdomain_tag, post_data), (subdomain_tag_lower, post_data_lower)]
 
         for tag, pd in post_data_list:
-            # Send raw body (not URL-encoded) so payload URLs like http://... reach the
-            # server literally — matching old curl -d behavior.
+            # Send raw body (not URL-encoded) so payload URLs like http://... reach the server literally.
             raw_body = "&".join(f"{k}={v}" for k, v in pd.items())
             r = await self.generic_ssrf.helpers.request(url=test_url, method="POST", body=raw_body)
             if r: