Merge pull request #2266 from blacklanternsecurity/dev

Dev -> Stable 2.4.0

Author: TheTechromancer

.github/ISSUE_TEMPLATE/bug_report.md

@@ -7,24 +7,18 @@ assignees: ""
 ---
 
 **Describe the bug**
-What happened?
-
-**Expected behavior**
-What was supposed to happen?
+What happened vs what was expected?
 
 **BBOT Command**
 Example: `bbot -m httpx -t evilcorp.com`
 
 **OS, BBOT Installation Method + Version**
 Example: `OS: Arch Linux, Installation method: pip, BBOT version: 1.0.3.545`
-Note: You can get the bbot version with `bbot --version`
-Note: Windows is **not** supported. We have successfully used BBOT on Docker Desktop in the past, however Windows is highly problematic so if you choose this path you are on your own.
+Note: You can get the BBOT version with `bbot --version`
+Note: BBOT is designed from the ground up to run on Linux. Windows and MacOS are not officially supported. If you are using one of these platforms, it's recommended to use Docker.
 
 **BBOT Config**
-Attach your BBOT config (`bbot --current-config`).
-
-**Logs**
-If possible, produce the bug while `--debug` is enabled, and attach the relevant parts of `~/.bbot/logs/bbot.debug.log`
+Attach your full BBOT preset (to show it, add `--current-preset` to your BBOT command).
 
-**Screenshots**
-If applicable, add screenshots to help explain your problem.
+**Logs/Screenshots**
+If possible, produce the bug while `--debug` is enabled, and attach the relevant parts of the output.

.github/workflows/distro_tests.yml

@@ -53,6 +53,8 @@ jobs:
             python3.11 -m pipx ensurepath
             pipx install poetry
           "
+      - name: Set OS Environment Variable
+        run: echo "OS_NAME=${{ matrix.os }}" | sed 's|[:/]|_|g' >> $GITHUB_ENV
       - name: Run tests
         run: |
           export PATH="$HOME/.local/bin:$PATH"
@@ -66,5 +68,5 @@ jobs:
         if: always()
         uses: actions/upload-artifact@v4
         with:
-          name: pytest-debug-logs-${{ matrix.os }}
-          path: pytest_debug_${{ matrix.os }}.log
+          name: pytest-debug-logs-${{ env.OS_NAME }}
+          path: pytest_debug.log

.github/workflows/tests.yml

@@ -1,4 +1,4 @@
-name: Tests (Python Versions)
+name: Tests
 on:
   push:
     branches:
@@ -24,6 +24,8 @@ jobs:
         uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
+      - name: Set Python Version Environment Variable
+        run: echo "PYTHON_VERSION=${{ matrix.python-version }}" | sed 's|[:/]|_|g' >> $GITHUB_ENV
       - name: Install dependencies
         run: |
           pip install poetry
@@ -39,8 +41,8 @@ jobs:
         if: always()
         uses: actions/upload-artifact@v4
         with:
-          name: pytest-debug-logs-${{ matrix.python-version }}
-          path: pytest_debug_${{ matrix.python-version }}.log
+          name: pytest-debug-logs-${{ env.PYTHON_VERSION }}
+          path: pytest_debug.log
       - name: Upload Code Coverage
         uses: codecov/codecov-action@v5
         with:

bbot/cli.py

@@ -161,8 +161,11 @@ async def _main():
             all_modules = list(preset.module_loader.preloaded())
             scan.helpers.depsinstaller.force_deps = True
             succeeded, failed = await scan.helpers.depsinstaller.install(*all_modules)
-            log.info("Finished installing module dependencies")
-            return False if failed else True
+            if failed:
+                log.hugewarning(f"Failed to install dependencies for the following modules: {', '.join(failed)}")
+                return False
+            log.hugesuccess(f"Successfully installed dependencies for the following modules: {', '.join(succeeded)}")
+            return True
 
         scan_name = str(scan.name)
 
@@ -180,13 +183,14 @@ async def _main():
 
             if sys.stdin.isatty():
                 # warn if any targets belong directly to a cloud provider
-                for event in scan.target.seeds.events:
-                    if event.type == "DNS_NAME":
-                        cloudcheck_result = scan.helpers.cloudcheck(event.host)
-                        if cloudcheck_result:
-                            scan.hugewarning(
-                                f'YOUR TARGET CONTAINS A CLOUD DOMAIN: "{event.host}". You\'re in for a wild ride!'
-                            )
+                if not scan.preset.strict_scope:
+                    for event in scan.target.seeds.events:
+                        if event.type == "DNS_NAME":
+                            cloudcheck_result = scan.helpers.cloudcheck(event.host)
+                            if cloudcheck_result:
+                                scan.hugewarning(
+                                    f'YOUR TARGET CONTAINS A CLOUD DOMAIN: "{event.host}". You\'re in for a wild ride!'
+                                )
 
                 if not options.yes:
                     log.hugesuccess(f"Scan ready. Press enter to execute {scan.name}")

bbot/core/helpers/depsinstaller/installer.py

@@ -96,7 +96,7 @@ def __init__(self, parent_helper):
         self.ensure_root_lock = Lock()
 
     async def install(self, *modules):
-        self.install_core_deps()
+        await self.install_core_deps()
         succeeded = []
         failed = []
         try:
@@ -386,20 +386,34 @@ def ensure_root(self, message=""):
                 else:
                     log.warning("Incorrect password")
 
-    def install_core_deps(self):
+    async def install_core_deps(self):
         to_install = set()
         to_install_friendly = set()
         playbook = []
         self._install_sudo_askpass()
         # ensure tldextract data is cached
         self.parent_helper.tldextract("evilcorp.co.uk")
+        # install any missing commands
         for command, package_name_or_playbook in self.CORE_DEPS.items():
             if not self.parent_helper.which(command):
                 to_install_friendly.add(command)
                 if isinstance(package_name_or_playbook, str):
                     to_install.add(package_name_or_playbook)
                 else:
                     playbook.extend(package_name_or_playbook)
+        # install ansible community.general collection
+        if not self.setup_status.get("ansible:community.general", False):
+            log.info("Installing Ansible Community General Collection")
+            try:
+                command = ["ansible-galaxy", "collection", "install", "community.general"]
+                await self.parent_helper.run(command, check=True)
+                self.setup_status["ansible:community.general"] = True
+                log.info("Successfully installed Ansible Community General Collection")
+            except CalledProcessError as err:
+                log.warning(
+                    f"Failed to install Ansible Community.General Collection (return code {err.returncode}): {err.stderr}"
+                )
+        # construct ansible playbook
         if to_install:
             playbook.append(
                 {
@@ -408,6 +422,7 @@ def install_core_deps(self):
                     "become": True,
                 }
             )
+        # run playbook
         if playbook:
             log.info(f"Installing core BBOT dependencies: {','.join(sorted(to_install_friendly))}")
             self.ensure_root()

bbot/core/helpers/regex.py

@@ -31,6 +31,10 @@ async def search(self, compiled_regex, *args, **kwargs):
         self.ensure_compiled_regex(compiled_regex)
         return await self.parent_helper.run_in_executor(compiled_regex.search, *args, **kwargs)
 
+    async def sub(self, compiled_regex, *args, **kwargs):
+        self.ensure_compiled_regex(compiled_regex)
+        return await self.parent_helper.run_in_executor(compiled_regex.sub, *args, **kwargs)
+
     async def findall(self, compiled_regex, *args, **kwargs):
         self.ensure_compiled_regex(compiled_regex)
         return await self.parent_helper.run_in_executor(compiled_regex.findall, *args, **kwargs)

bbot/core/helpers/validators.py

@@ -176,6 +176,7 @@ def clean_url(url: str, url_querystring_remove=True):
         scheme = parsed.scheme
     except ValueError:
         scheme = "https"
+    port = None
     with suppress(Exception):
         port = parsed.port
     if port is None:

bbot/core/helpers/web/client.py

@@ -83,6 +83,7 @@ def build_request(self, *args, **kwargs):
         # TODO: re-enable this
         if self._target.in_scope(str(request.url)):
             for hk, hv in self._web_config.get("http_headers", {}).items():
+                hv = str(hv)
                 # don't clobber headers
                 if hk not in request.headers:
                     request.headers[hk] = hv

bbot/core/helpers/web/web.py

@@ -232,7 +232,7 @@ async def download(self, url, **kwargs):
         if success:
             return filename
 
-    async def wordlist(self, path, lines=None, **kwargs):
+    async def wordlist(self, path, lines=None, zip=False, zip_filename=None, **kwargs):
         """
         Asynchronous function for retrieving wordlists, either from a local path or a URL.
         Allows for optional line-based truncation and caching. Returns the full path of the wordlist
@@ -242,6 +242,9 @@ async def wordlist(self, path, lines=None, **kwargs):
             path (str): The local or remote path of the wordlist.
             lines (int, optional): Number of lines to read from the wordlist.
                 If specified, will return a truncated wordlist with this many lines.
+            zip (bool, optional): Whether to unzip the file after downloading. Defaults to False.
+            zip_filename (str, optional): The name of the file to extract from the ZIP archive.
+                Required if zip is True.
             cache_hrs (float, optional): Number of hours to cache the downloaded wordlist.
                 Defaults to 720 hours (30 days) for remote wordlists.
             **kwargs: Additional keyword arguments to pass to the 'download' function for remote wordlists.
@@ -259,6 +262,8 @@ async def wordlist(self, path, lines=None, **kwargs):
             Fetching and truncating to the first 100 lines
             >>> wordlist_path = await self.helpers.wordlist("/root/rockyou.txt", lines=100)
         """
+        import zipfile
+
         if not path:
             raise WordlistError(f"Invalid wordlist: {path}")
         if "cache_hrs" not in kwargs:
@@ -272,6 +277,18 @@ async def wordlist(self, path, lines=None, **kwargs):
             if not filename.is_file():
                 raise WordlistError(f"Unable to find wordlist at {path}")
 
+        if zip:
+            if not zip_filename:
+                raise WordlistError("zip_filename must be specified when zip is True")
+            try:
+                with zipfile.ZipFile(filename, "r") as zip_ref:
+                    if zip_filename not in zip_ref.namelist():
+                        raise WordlistError(f"File {zip_filename} not found in the zip archive {filename}")
+                    zip_ref.extract(zip_filename, filename.parent)
+                    filename = filename.parent / zip_filename
+            except Exception as e:
+                raise WordlistError(f"Error unzipping file {filename}: {e}")
+
         if lines is None:
             return filename
         else:
@@ -336,6 +353,7 @@ async def curl(self, *args, **kwargs):
         ignore_bbot_global_settings = kwargs.get("ignore_bbot_global_settings", False)
 
         if ignore_bbot_global_settings:
+            http_timeout = 20  # setting 20 as a worse-case setting
             log.debug("ignore_bbot_global_settings enabled. Global settings will not be applied")
         else:
             http_timeout = self.parent_helper.web_config.get("http_timeout", 20)
@@ -349,12 +367,12 @@ async def curl(self, *args, **kwargs):
                 for hk, hv in self.web_config.get("http_headers", {}).items():
                     headers[hk] = hv
 
-            # add the timeout
-            if "timeout" not in kwargs:
-                timeout = http_timeout
+        # add the timeout
+        if "timeout" not in kwargs:
+            timeout = http_timeout
 
-            curl_command.append("-m")
-            curl_command.append(str(timeout))
+        curl_command.append("-m")
+        curl_command.append(str(timeout))
 
         for k, v in headers.items():
             if isinstance(v, list):
@@ -400,7 +418,7 @@ async def curl(self, *args, **kwargs):
         if raw_body:
             curl_command.append("-d")
             curl_command.append(raw_body)
-
+        log.verbose(f"Running curl command: {curl_command}")
         output = (await self.parent_helper.run(curl_command)).stdout
         return output
 

bbot/modules/badsecrets.py

@@ -17,7 +17,7 @@ class badsecrets(BaseModule):
     options_desc = {
         "custom_secrets": "Include custom secrets loaded from a local file",
     }
-    deps_pip = ["badsecrets~=0.6.21"]
+    deps_pip = ["badsecrets~=0.9.29"]
 
     async def setup(self):
         self.custom_secrets = None