fix conflict

github-actions · github-actions · commit ea98f26cff13 · 2025-04-24T13:36:46.000-04:00
diff --git a/bbot/__init__.py b/bbot/__init__.py
@@ -2,3 +2,5 @@
 __version__ = "v0.0.0"
 
 from .scanner import Scanner, Preset
+
+__all__ = ["Scanner", "Preset"]
diff --git a/bbot/core/event/__init__.py b/bbot/core/event/__init__.py
@@ -1 +1,3 @@
 from .base import make_event, is_event, event_from_json
+
+__all__ = ["make_event", "is_event", "event_from_json"]
diff --git a/bbot/core/helpers/__init__.py b/bbot/core/helpers/__init__.py
@@ -1,4 +1,4 @@
 from .url import *
 from .misc import *
-from . import regexes
-from . import validators
+from . import regexes as regexes
+from . import validators as validators
diff --git a/bbot/core/helpers/depsinstaller/__init__.py b/bbot/core/helpers/depsinstaller/__init__.py
@@ -1 +1,3 @@
 from .installer import DepsInstaller
+
+__all__ = ["DepsInstaller"]
diff --git a/bbot/core/helpers/dns/__init__.py b/bbot/core/helpers/dns/__init__.py
@@ -1 +1 @@
-from .dns import DNSHelper
+from .dns import DNSHelper  # noqa
diff --git a/bbot/core/helpers/misc.py b/bbot/core/helpers/misc.py
@@ -50,7 +50,7 @@ def is_domain(d):
     if is_ip(d):
         return False
     extracted = tldextract(d)
-    if extracted.registered_domain:
+    if extracted.top_domain_under_public_suffix:
         if not extracted.subdomain:
             return True
     else:
@@ -85,7 +85,7 @@ def is_subdomain(d):
     if is_ip(d):
         return False
     extracted = tldextract(d)
-    if extracted.registered_domain:
+    if extracted.top_domain_under_public_suffix:
         if extracted.subdomain:
             return True
     else:
@@ -486,7 +486,7 @@ def split_domain(hostname):
         return ("", hostname)
     parsed = tldextract(hostname)
     subdomain = parsed.subdomain
-    domain = parsed.registered_domain
+    domain = parsed.top_domain_under_public_suffix
     if not domain:
         split = hostname.split(".")
         subdomain = ".".join(split[:-2])
diff --git a/bbot/core/helpers/web/__init__.py b/bbot/core/helpers/web/__init__.py
@@ -1 +1 @@
-from .web import WebHelper
+from .web import WebHelper  # noqa
diff --git a/bbot/core/helpers/web/client.py b/bbot/core/helpers/web/client.py
@@ -70,7 +70,7 @@ def __init__(self, *args, **kwargs):
         kwargs["headers"] = headers
         # proxy
         proxies = self._web_config.get("http_proxy", None)
-        kwargs["proxies"] = proxies
+        kwargs["proxy"] = proxies
 
         log.verbose(f"Creating httpx.AsyncClient({args}, {kwargs})")
         super().__init__(*args, **kwargs)
diff --git a/bbot/core/helpers/web/web.py b/bbot/core/helpers/web/web.py
@@ -95,7 +95,7 @@ async def request(self, *args, **kwargs):
             files (dict, optional): Dictionary of 'name': file-like-objects for multipart encoding upload.
             auth (tuple, optional): Auth tuple to enable Basic/Digest/Custom HTTP auth.
             timeout (float, optional): The maximum time to wait for the request to complete.
-            proxies (dict, optional): Dictionary mapping protocol schemes to proxy URLs.
+            proxy (str, optional): HTTP proxy URL.
             allow_redirects (bool, optional): Enables or disables redirection. Defaults to None.
             stream (bool, optional): Enables or disables response streaming.
             raise_error (bool, optional): Whether to raise exceptions for HTTP connect, timeout errors. Defaults to False.
diff --git a/bbot/defaults.yml b/bbot/defaults.yml
@@ -100,6 +100,10 @@ web:
   http_retries: 1
   # HTTP retries (for httpx)
   httpx_retries: 1
+  # Default sleep interval when rate limited by 429 (and retry-after isn't provided)
+  429_sleep_interval: 30
+  # Maximum sleep interval when rate limited by 429 (and an excessive retry-after is provided)
+  429_max_sleep_interval: 60
   # Enable/disable debug messages for web requests/responses
   debug: false
   # Maximum number of HTTP redirects to follow
diff --git a/bbot/modules/baddns.py b/bbot/modules/baddns.py
@@ -22,7 +22,7 @@ class baddns(BaseModule):
         "enabled_submodules": "A list of submodules to enable. Empty list (default) enables CNAME, TXT and MX Only",
     }
     module_threads = 8
-    deps_pip = ["baddns~=1.4.13"]
+    deps_pip = ["baddns~=1.9.130"]
 
     def select_modules(self):
         selected_submodules = []
diff --git a/bbot/modules/baddns_direct.py b/bbot/modules/baddns_direct.py
@@ -19,7 +19,7 @@ class baddns_direct(BaseModule):
         "custom_nameservers": "Force BadDNS to use a list of custom nameservers",
     }
     module_threads = 8
-    deps_pip = ["baddns~=1.4.13"]
+    deps_pip = ["baddns~=1.9.130"]
 
     scope_distance_modifier = 1
 
diff --git a/bbot/modules/baddns_zone.py b/bbot/modules/baddns_zone.py
@@ -16,7 +16,7 @@ class baddns_zone(baddns_module):
         "only_high_confidence": "Do not emit low-confidence or generic detections",
     }
     module_threads = 8
-    deps_pip = ["baddns~=1.4.13"]
+    deps_pip = ["baddns~=1.9.130"]
 
     def set_modules(self):
         self.enabled_submodules = ["NSEC", "zonetransfer"]
diff --git a/bbot/modules/base.py b/bbot/modules/base.py
@@ -106,8 +106,6 @@ class BaseModule:
 
     # disable the module after this many failed attempts in a row
     _api_failure_abort_threshold = 3
-    # sleep for this many seconds after being rate limited
-    _429_sleep_interval = 30
 
     default_discovery_context = "{module} discovered {event.type}: {event.data}"
 
@@ -165,6 +163,10 @@ def __init__(self, scan):
         # used for optional "per host" tracking
         self._per_host_tracker = set()
 
+        # 429 rate limit handling
+        self._429_sleep_interval = self.scan.web_config.get("429_sleep_interval", 30)
+        self._429_max_sleep_interval = self.scan.web_config.get("429_max_sleep_interval", 60)
+
     async def setup(self):
         """
         Performs one-time setup tasks for the module.
@@ -1172,7 +1174,14 @@ async def api_request(self, *args, **kwargs):
                     retry_after = self._get_retry_after(r)
                     if retry_after or status_code == 429:
                         sleep_interval = int(retry_after) if retry_after is not None else self._429_sleep_interval
-                        self.info(f"Retrying {new_url} in {sleep_interval:,} seconds (HTTP status: {status_code})")
+                        if retry_after and retry_after > self._429_max_sleep_interval:
+                            self.verbose(
+                                f"Got an excessive retry-after header of {retry_after} from {new_url}, using {self._429_max_sleep_interval} instead"
+                            )
+                            sleep_interval = self._429_max_sleep_interval
+                        self.verbose(
+                            f"Sleeping for {sleep_interval:,} seconds due to rate limit (HTTP status: {status_code})"
+                        )
                         await asyncio.sleep(sleep_interval)
                     elif self._api_keys:
                         # if request failed, cycle API keys and try again
diff --git a/bbot/modules/certspotter.py b/bbot/modules/certspotter.py
@@ -15,7 +15,7 @@ class certspotter(subdomain_enum):
 
     def request_url(self, query):
         url = f"{self.base_url}/issuances?domain={self.helpers.quote(query)}&include_subdomains=true&expand=dns_names"
-        return self.api_request(url, timeout=self.http_timeout + 30)
+        return self.api_request(url)
 
     async def parse_results(self, r, query):
         results = set()
diff --git a/bbot/modules/internal/speculate.py b/bbot/modules/internal/speculate.py
@@ -176,8 +176,8 @@ async def handle_event(self, event):
         org_stubs = set()
         if event.type == "DNS_NAME" and event.scope_distance == 0:
             tldextracted = self.helpers.tldextract(event.data)
-            registered_domain = getattr(tldextracted, "registered_domain", "")
-            if registered_domain:
+            top_domain_under_public_suffix = getattr(tldextracted, "top_domain_under_public_suffix", "")
+            if top_domain_under_public_suffix:
                 tld_stub = getattr(tldextracted, "domain", "")
                 if tld_stub:
                     decoded_tld_stub = self.helpers.smart_decode_punycode(tld_stub)
diff --git a/bbot/modules/telerik.py b/bbot/modules/telerik.py
@@ -1,5 +1,4 @@
 from sys import executable
-from urllib.parse import urlparse
 
 from bbot.modules.base import BaseModule
 
diff --git a/bbot/scanner/__init__.py b/bbot/scanner/__init__.py
@@ -1,2 +1,4 @@
 from .preset import Preset
 from .scanner import Scanner
+
+__all__ = ["Preset", "Scanner"]
diff --git a/bbot/scanner/preset/__init__.py b/bbot/scanner/preset/__init__.py
@@ -1 +1,3 @@
 from .preset import Preset
+
+__all__ = ["Preset"]
diff --git a/bbot/scanner/target.py b/bbot/scanner/target.py
@@ -28,6 +28,8 @@ class BaseTarget(RadixTarget):
     accept_target_types = ["TARGET"]
 
     def __init__(self, *targets, **kwargs):
+        # ignore blank targets (sometimes happens as a symptom of .splitlines())
+        targets = [stripped for t in targets if (stripped := (t.strip() if isinstance(t, str) else t))]
         self.event_seeds = set()
         super().__init__(*targets, **kwargs)
 
diff --git a/bbot/test/bbot_fixtures.py b/bbot/test/bbot_fixtures.py
@@ -1,11 +1,9 @@
 import os  # noqa
-import sys
-import zlib
+import sys  # noqa
 import pytest
 import shutil  # noqa
 import asyncio  # noqa
 import logging
-import subprocess
 import tldextract
 import pytest_httpserver
 from pathlib import Path
@@ -16,8 +14,8 @@
 from bbot.errors import *  # noqa: F401
 from bbot.core import CORE
 from bbot.scanner import Preset
+from bbot.core.helpers.misc import mkdir, rand_string
 from bbot.core.helpers.async_helpers import get_event_loop
-from bbot.core.helpers.misc import mkdir, rand_string, get_python_constraints
 
 
 log = logging.getLogger("bbot.test.fixtures")
diff --git a/bbot/test/conftest.py b/bbot/test/conftest.py
@@ -1,5 +1,6 @@
 import os
 import ssl
+import time
 import shutil
 import pytest
 import asyncio
@@ -8,8 +9,6 @@
 from contextlib import suppress
 from omegaconf import OmegaConf
 from pytest_httpserver import HTTPServer
-import time
-import queue
 
 from bbot.core import CORE
 from bbot.core.helpers.misc import execute_sync_or_async
diff --git a/bbot/test/test_step_1/test_presets.py b/bbot/test/test_step_1/test_presets.py
@@ -1079,8 +1079,6 @@ async def test_preset_output_dir():
 
 # regression test for https://github.com/blacklanternsecurity/bbot/issues/2337
 def test_preset_serialization():
-    from ipaddress import ip_address, ip_network
-
     preset = Preset("192.168.1.1")
     preset = preset.bake()
 
diff --git a/bbot/test/test_step_2/module_tests/test_module_censys.py b/bbot/test/test_step_2/module_tests/test_module_censys.py
@@ -7,7 +7,7 @@ class TestCensys(ModuleTestBase):
     async def setup_before_prep(self, module_test):
         module_test.httpx_mock.add_response(
             url="https://search.censys.io/api/v1/account",
-            # match_headers={"Authorization": "Basic YXBpX2lkOmFwaV9zZWNyZXQ="},
+            match_headers={"Authorization": "Basic YXBpX2lkOmFwaV9zZWNyZXQ="},
             json={
                 "email": "info@blacklanternsecurity.com",
                 "login": "nope",
@@ -18,8 +18,9 @@ async def setup_before_prep(self, module_test):
         )
         module_test.httpx_mock.add_response(
             url="https://search.censys.io/api/v2/certificates/search",
-            # match_headers={"Authorization": "Basic YXBpX2lkOmFwaV9zZWNyZXQ="},
-            match_content=b'{"q": "names: blacklanternsecurity.com", "per_page": 100}',
+            match_headers={"Authorization": "Basic YXBpX2lkOmFwaV9zZWNyZXQ="},
+            method="POST",
+            match_json={"q": "names: blacklanternsecurity.com", "per_page": 100},
             json={
                 "code": 200,
                 "status": "OK",
@@ -47,8 +48,9 @@ async def setup_before_prep(self, module_test):
         )
         module_test.httpx_mock.add_response(
             url="https://search.censys.io/api/v2/certificates/search",
-            # match_headers={"Authorization": "Basic YXBpX2lkOmFwaV9zZWNyZXQ="},
-            match_content=b'{"q": "names: blacklanternsecurity.com", "per_page": 100, "cursor": "NextToken"}',
+            match_headers={"Authorization": "Basic YXBpX2lkOmFwaV9zZWNyZXQ="},
+            method="POST",
+            match_json={"q": "names: blacklanternsecurity.com", "per_page": 100, "cursor": "NextToken"},
             json={
                 "code": 200,
                 "status": "OK",
diff --git a/bbot/test/test_step_2/module_tests/test_module_dotnetnuke.py b/bbot/test/test_step_2/module_tests/test_module_dotnetnuke.py
@@ -1,4 +1,3 @@
-import asyncio
 import re
 from .base import ModuleTestBase
 from werkzeug.wrappers import Response
diff --git a/bbot/test/test_step_2/module_tests/test_module_host_header.py b/bbot/test/test_step_2/module_tests/test_module_host_header.py
@@ -1,4 +1,3 @@
-import asyncio
 import re
 from werkzeug.wrappers import Response
 
diff --git a/bbot/test/test_step_2/module_tests/test_module_ipstack.py b/bbot/test/test_step_2/module_tests/test_module_ipstack.py
@@ -26,7 +26,7 @@ async def setup_before_prep(self, module_test):
                     "capital": "Washington D.C.",
                     "languages": [{"code": "en", "name": "English", "native": "English"}],
                     "country_flag": "https://assets.ipstack.com/flags/us.svg",
-                    "country_flag_emoji": "\ud83c\uddfa\ud83c\uddf8",
+                    "country_flag_emoji": "🇺🇸",
                     "country_flag_emoji_unicode": "U+1F1FA U+1F1F8",
                     "calling_code": "1",
                     "is_eu": False,
@@ -53,7 +53,7 @@ async def setup_before_prep(self, module_test):
                     "capital": "Washington D.C.",
                     "languages": [{"code": "en", "name": "English", "native": "English"}],
                     "country_flag": "https://assets.ipstack.com/flags/us.svg",
-                    "country_flag_emoji": "\ud83c\uddfa\ud83c\uddf8",
+                    "country_flag_emoji": "🇺🇸",
                     "country_flag_emoji_unicode": "U+1F1FA U+1F1F8",
                     "calling_code": "1",
                     "is_eu": False,
diff --git a/bbot/test/test_step_2/module_tests/test_module_postman.py b/bbot/test/test_step_2/module_tests/test_module_postman.py
@@ -45,7 +45,22 @@ async def setup_after_prep(self, module_test):
         )
         module_test.httpx_mock.add_response(
             url="https://www.postman.com/_api/ws/proxy",
-            match_content=b'{"service": "search", "method": "POST", "path": "/search-all", "body": {"queryIndices": ["collaboration.workspace"], "queryText": "blacklanternsecurity", "size": 25, "from": 0, "clientTraceId": "", "requestOrigin": "srp", "mergeEntities": "true", "nonNestedRequests": "true", "domain": "public"}}',
+            match_json={
+                "service": "search",
+                "method": "POST",
+                "path": "/search-all",
+                "body": {
+                    "queryIndices": ["collaboration.workspace"],
+                    "queryText": "blacklanternsecurity",
+                    "size": 25,
+                    "from": 0,
+                    "clientTraceId": "",
+                    "requestOrigin": "srp",
+                    "mergeEntities": "true",
+                    "nonNestedRequests": "true",
+                    "domain": "public",
+                },
+            },
             json={
                 "data": [
                     {
@@ -177,7 +192,11 @@ async def setup_after_prep(self, module_test):
         )
         module_test.httpx_mock.add_response(
             url="https://www.postman.com/_api/ws/proxy",
-            match_content=b'{"service": "workspaces", "method": "GET", "path": "/workspaces?handle=blacklanternsecurity&slug=bbot-public"}',
+            match_json={
+                "service": "workspaces",
+                "method": "GET",
+                "path": "/workspaces?handle=blacklanternsecurity&slug=bbot-public",
+            },
             json={
                 "meta": {"model": "workspace", "action": "find", "nextCursor": ""},
                 "data": [
@@ -207,7 +226,11 @@ async def setup_after_prep(self, module_test):
         )
         module_test.httpx_mock.add_response(
             url="https://www.postman.com/_api/ws/proxy",
-            match_content=b'{"service": "workspaces", "method": "GET", "path": "/workspaces?handle=testteam&slug=testing-bbot-api"}',
+            match_json={
+                "service": "workspaces",
+                "method": "GET",
+                "path": "/workspaces?handle=testteam&slug=testing-bbot-api",
+            },
             json={
                 "meta": {"model": "workspace", "action": "find", "nextCursor": ""},
                 "data": [
diff --git a/bbot/test/test_step_2/module_tests/test_module_postman_download.py b/bbot/test/test_step_2/module_tests/test_module_postman_download.py
@@ -45,7 +45,22 @@ async def setup_after_prep(self, module_test):
         )
         module_test.httpx_mock.add_response(
             url="https://www.postman.com/_api/ws/proxy",
-            match_content=b'{"service": "search", "method": "POST", "path": "/search-all", "body": {"queryIndices": ["collaboration.workspace"], "queryText": "blacklanternsecurity", "size": 25, "from": 0, "clientTraceId": "", "requestOrigin": "srp", "mergeEntities": "true", "nonNestedRequests": "true", "domain": "public"}}',
+            match_json={
+                "service": "search",
+                "method": "POST",
+                "path": "/search-all",
+                "body": {
+                    "queryIndices": ["collaboration.workspace"],
+                    "queryText": "blacklanternsecurity",
+                    "size": 25,
+                    "from": 0,
+                    "clientTraceId": "",
+                    "requestOrigin": "srp",
+                    "mergeEntities": "true",
+                    "nonNestedRequests": "true",
+                    "domain": "public",
+                },
+            },
             json={
                 "data": [
                     {
@@ -126,7 +141,11 @@ async def setup_after_prep(self, module_test):
         )
         module_test.httpx_mock.add_response(
             url="https://www.postman.com/_api/ws/proxy",
-            match_content=b'{"service": "workspaces", "method": "GET", "path": "/workspaces?handle=blacklanternsecurity&slug=bbot-public"}',
+            match_json={
+                "service": "workspaces",
+                "method": "GET",
+                "path": "/workspaces?handle=blacklanternsecurity&slug=bbot-public",
+            },
             json={
                 "meta": {"model": "workspace", "action": "find", "nextCursor": ""},
                 "data": [
diff --git a/bbot/test/test_step_2/module_tests/test_module_telerik.py b/bbot/test/test_step_2/module_tests/test_module_telerik.py
@@ -1,5 +1,5 @@
 import re
-from .base import ModuleTestBase, tempwordlist
+from .base import ModuleTestBase
 
 
 class TestTelerik(ModuleTestBase):
diff --git a/bbot/test/test_step_2/module_tests/test_module_trufflehog.py b/bbot/test/test_step_2/module_tests/test_module_trufflehog.py
diff --git a/bbot/test/test_step_2/module_tests/test_module_websocket.py b/bbot/test/test_step_2/module_tests/test_module_websocket.py
diff --git a/docs/modules/nuclei.md b/docs/modules/nuclei.md
diff --git a/docs/scanning/configuration.md b/docs/scanning/configuration.md
diff --git a/poetry.lock b/poetry.lock
diff --git a/pyproject.toml b/pyproject.toml

Original file line number	Diff line number	Diff line change
`@@ -1 +1,3 @@`
`1`	`1`	`from .base import make_event, is_event, event_from_json`
	`2`	`+`
	`3`	`+__all__ = ["make_event", "is_event", "event_from_json"]`
Original file line number	Diff line number	Diff line change
`@@ -1 +1,3 @@`
`1`	`1`	`from .installer import DepsInstaller`
	`2`	`+`
	`3`	`+__all__ = ["DepsInstaller"]`
Original file line number	Diff line number	Diff line change
`@@ -1 +1 @@`
`1`		`-from .dns import DNSHelper`
	`1`	`+from .dns import DNSHelper # noqa`
Original file line number	Diff line number	Diff line change
`@@ -1 +1 @@`
`1`		`-from .web import WebHelper`
	`1`	`+from .web import WebHelper # noqa`