Merge pull request #2339 from blacklanternsecurity/dev

Dev -> Stable 2.4.1

Author: TheTechromancer

.github/workflows/codeql.yml

@@ -0,0 +1,98 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+#
+# ******** NOTE ********
+# We have attempted to detect the languages in your repository. Please check
+# the `language` matrix defined below to confirm you have the correct set of
+# supported CodeQL languages.
+#
+name: "CodeQL Advanced"
+
+on:
+  push:
+    branches: [ "stable", "dev" ]
+  pull_request:
+    branches: [ "stable", "dev" ]
+  schedule:
+    - cron: '32 3 * * 5'
+
+jobs:
+  analyze:
+    name: Analyze (${{ matrix.language }})
+    # Runner size impacts CodeQL analysis time. To learn more, please see:
+    #   - https://gh.io/recommended-hardware-resources-for-running-codeql
+    #   - https://gh.io/supported-runners-and-hardware-resources
+    #   - https://gh.io/using-larger-runners (GitHub.com only)
+    # Consider using larger runners or machines with greater resources for possible analysis time improvements.
+    runs-on: ${{ (matrix.language == 'swift' && 'macos-latest') || 'ubuntu-latest' }}
+    permissions:
+      # required for all workflows
+      security-events: write
+
+      # required to fetch internal or private CodeQL packs
+      packages: read
+
+      # only required for workflows in private repositories
+      actions: read
+      contents: read
+
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+        - language: python
+          build-mode: none
+        # CodeQL supports the following values keywords for 'language': 'c-cpp', 'csharp', 'go', 'java-kotlin', 'javascript-typescript', 'python', 'ruby', 'swift'
+        # Use `c-cpp` to analyze code written in C, C++ or both
+        # Use 'java-kotlin' to analyze code written in Java, Kotlin or both
+        # Use 'javascript-typescript' to analyze code written in JavaScript, TypeScript or both
+        # To learn more about changing the languages that are analyzed or customizing the build mode for your analysis,
+        # see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/customizing-your-advanced-setup-for-code-scanning.
+        # If you are analyzing a compiled language, you can modify the 'build-mode' for that language to customize how
+        # your codebase is analyzed, see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v4
+
+    # Add any setup steps before running the `github/codeql-action/init` action.
+    # This includes steps like installing compilers or runtimes (`actions/setup-node`
+    # or others). This is typically only required for manual builds.
+    # - name: Setup runtime (example)
+    #   uses: actions/setup-example@v1
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v3
+      with:
+        languages: ${{ matrix.language }}
+        build-mode: ${{ matrix.build-mode }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file.
+        # Prefix the list here with "+" to use these queries and those in the config file.
+
+        # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
+        # queries: security-extended,security-and-quality
+
+    # If the analyze step fails for one of the languages you are analyzing with
+    # "We were unable to automatically build your code", modify the matrix above
+    # to set the build mode to "manual" for that language. Then modify this step
+    # to build your code.
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
+    - if: matrix.build-mode == 'manual'
+      shell: bash
+      run: |
+        echo 'If you are using a "manual" build mode for one or more of the' \
+          'languages you are analyzing, replace this with the commands to build' \
+          'your code, for example:'
+        echo '  make bootstrap'
+        echo '  make release'
+        exit 1
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v3
+      with:
+        category: "/language:${{matrix.language}}"

.github/workflows/distro_tests.yml

@@ -63,7 +63,7 @@ jobs:
           export BBOT_DISTRO_TESTS=true
           poetry env use python3.11
           poetry install
-          poetry run pytest --reruns 2 -o timeout_func_only=true --timeout 1200 --disable-warnings --log-cli-level=INFO .
+          poetry run pytest --reruns 2 --exitfirst -o timeout_func_only=true --timeout 1200 --disable-warnings --log-cli-level=INFO .
       - name: Upload Debug Logs
         if: always()
         uses: actions/upload-artifact@v4

.github/workflows/tests.yml

@@ -36,7 +36,7 @@ jobs:
           poetry run ruff format --check
       - name: Run tests
         run: |
-          poetry run pytest -vv --exitfirst --reruns 2 -o timeout_func_only=true --timeout 1200 --disable-warnings --log-cli-level=INFO --cov-config=bbot/test/coverage.cfg --cov-report xml:cov.xml --cov=bbot .
+          poetry run pytest -vv --reruns 2 -o timeout_func_only=true --timeout 1200 --disable-warnings --log-cli-level=INFO --cov-config=bbot/test/coverage.cfg --cov-report xml:cov.xml --cov=bbot .
       - name: Upload Debug Logs
         if: always()
         uses: actions/upload-artifact@v4

bbot/cli.py

@@ -1,5 +1,6 @@
 #!/usr/bin/env python3
 
+import io
 import sys
 import logging
 import multiprocessing
@@ -78,7 +79,7 @@ async def _main():
             return
 
         # if we're listing modules or their options
-        if options.list_modules or options.list_output_modules or options.list_module_options:
+        if options.list_modules or options.list_output_modules or options.list_module_options or options.module_help:
             # if no modules or flags are specified, enable everything
             if not (options.modules or options.output_modules or options.flags):
                 for module, preloaded in preset.module_loader.preloaded().items():
@@ -119,6 +120,20 @@ async def _main():
                     print(row)
                 return
 
+            # --module-help
+            if options.module_help:
+                module_name = options.module_help
+                all_modules = list(preset.module_loader.preloaded())
+                if module_name not in all_modules:
+                    log.hugewarning(f'Module "{module_name}" not found')
+                    return
+
+                # Load the module class
+                loaded_modules = preset.module_loader.load_modules([module_name])
+                module_name, module_class = next(iter(loaded_modules.items()))
+                print(module_class.help_text())
+                return
+
         # --list-flags
         if options.list_flags:
             flags = preset.flags if preset.flags else None
@@ -184,7 +199,7 @@ async def _main():
             if sys.stdin.isatty():
                 # warn if any targets belong directly to a cloud provider
                 if not scan.preset.strict_scope:
-                    for event in scan.target.seeds.events:
+                    for event in scan.target.seeds.event_seeds:
                         if event.type == "DNS_NAME":
                             cloudcheck_result = scan.helpers.cloudcheck(event.host)
                             if cloudcheck_result:
@@ -226,7 +241,12 @@ def handle_keyboard_input(keyboard_input):
 
                 # set stdout and stderr to blocking mode
                 # this is needed to prevent BlockingIOErrors in logging etc.
-                fds = [sys.stdout.fileno(), sys.stderr.fileno()]
+                fds = []
+                for stream in [sys.stdout, sys.stderr]:
+                    try:
+                        fds.append(stream.fileno())
+                    except io.UnsupportedOperation:
+                        log.debug(f"Can't get fileno for {stream}")
                 for fd in fds:
                     flags = fcntl.fcntl(fd, fcntl.F_GETFL)
                     fcntl.fcntl(fd, fcntl.F_SETFL, flags & ~os.O_NONBLOCK)

bbot/core/config/logger.py

@@ -9,9 +9,8 @@
 from contextlib import suppress
 
 from ..helpers.misc import mkdir, error_and_exit
-from ...logger import colorize, loglevel_mapping
 from ..multiprocess import SHARED_INTERPRETER_STATE
-
+from ...logger import colorize, loglevel_mapping, GzipRotatingFileHandler
 
 debug_format = logging.Formatter("%(asctime)s [%(levelname)s] %(name)s %(filename)s:%(lineno)s %(message)s")
 
@@ -61,7 +60,6 @@ def __init__(self, core):
         self._loggers = None
         self._log_handlers = None
         self._log_level = None
-        self.root_logger = logging.getLogger()
         self.core_logger = logging.getLogger("bbot")
         self.core = core
 
@@ -84,14 +82,6 @@ def cleanup_logging(self):
         with suppress(Exception):
             self.queue_handler.close()
 
-        # Clean root logger
-        root_logger = logging.getLogger()
-        for handler in list(root_logger.handlers):
-            with suppress(Exception):
-                root_logger.removeHandler(handler)
-            with suppress(Exception):
-                handler.close()
-
         # Clean all other loggers
         for logger in logging.Logger.manager.loggerDict.values():
             if hasattr(logger, "handlers"):  # Logger, not PlaceHolder
@@ -112,8 +102,7 @@ def setup_queue_handler(self, logging_queue=None, log_level=logging.DEBUG):
             self.queue = logging_queue
         self.queue_handler = logging.handlers.QueueHandler(logging_queue)
 
-        self.root_logger.addHandler(self.queue_handler)
-
+        self.core_logger.addHandler(self.queue_handler)
         self.core_logger.setLevel(log_level)
         # disable asyncio logging for child processes
         if not SHARED_INTERPRETER_STATE.is_main_process:
@@ -217,13 +206,11 @@ def log_handlers(self):
                 error_and_exit(f"Failure creating or error writing to BBOT logs directory ({log_dir})")
 
             # Main log file
-            main_handler = logging.handlers.TimedRotatingFileHandler(
-                f"{log_dir}/bbot.log", when="d", interval=1, backupCount=14
-            )
+            main_handler = GzipRotatingFileHandler(f"{log_dir}/bbot.log", maxBytes=1024 * 1024 * 100, backupCount=100)
 
             # Separate log file for debugging
-            debug_handler = logging.handlers.TimedRotatingFileHandler(
-                f"{log_dir}/bbot.debug.log", when="d", interval=1, backupCount=14
+            debug_handler = GzipRotatingFileHandler(
+                f"{log_dir}/bbot.debug.log", maxBytes=1024 * 1024 * 100, backupCount=100
             )
 
             # Log to stderr