Merge pull request #3067 from blacklanternsecurity/paramminer-wordlist-cleanup

Paramminer Cleanup / Update

Author: liquidsec

bbot/defaults.yml

@@ -263,8 +263,25 @@ parameter_blacklist:
   - ASP.NET_SessionId
   - .AspNetCore.Session
   - PHPSESSID
+  - sessionid
+  - csrftoken
   - __cf_bm
+  - cf_clearance
+  - _abck
+  - bm_sz
+  - ak_bmsc
   - f5_cspm
+  - _ga
+  - _gid
+  - _gat
+  - _gcl_au
+  - _fbp
+  - _fbc
+  - __utma
+  - __utmb
+  - __utmc
+  - __utmz
+  - _hjid
 
 parameter_blacklist_prefixes:
   - TS01
@@ -277,6 +294,9 @@ parameter_blacklist_prefixes:
   - ApplicationGatewayAffinity
   - JSESSIONID
   - ARRAffinity
+  - _hjSession
+  - _gat_
+  - intercom-
 
 # Don't output these types of events (they are still distributed to modules)
 omit_event_types:

bbot/modules/paramminer_cookies.py

@@ -20,17 +20,15 @@ class paramminer_cookies(paramminer_headers):
         "skip_boring_words": True,
     }
     options_desc = {
-        "wordlist": "Define the wordlist to be used to derive headers",
+        "wordlist": "Define the wordlist to be used to derive cookies",
         "recycle_words": "Attempt to use words found during the scan on all other endpoints",
         "skip_boring_words": "Remove commonly uninteresting words from the wordlist",
     }
-    options_desc = {"wordlist": "Define the wordlist to be used to derive cookies"}
     scanned_hosts = []
-    boring_words = set()
     _module_threads = 12
     in_scope_only = True
     compare_mode = "cookie"
-    default_wordlist = "paramminer_parameters.txt"
+    default_wordlist = "paramminer_cookies.txt"
 
     async def check_batch(self, compare_helper, url, cookie_list):
         cookies = {p: self.rand_string(14) for p in cookie_list}

bbot/modules/paramminer_getparams.py

@@ -1,4 +1,8 @@
-from .paramminer_headers import paramminer_headers
+import itertools
+import string
+from urllib.parse import urlparse
+
+from .paramminer_headers import paramminer_headers, _mutate_case
 
 
 class paramminer_getparams(paramminer_headers):
@@ -19,17 +23,54 @@ class paramminer_getparams(paramminer_headers):
         "wordlist": "",  # default is defined within setup function
         "recycle_words": False,
         "skip_boring_words": True,
+        "mutate_case": False,
+        "brute_short": False,
     }
     options_desc = {
         "wordlist": "Define the wordlist to be used to derive headers",
         "recycle_words": "Attempt to use words found during the scan on all other endpoints",
         "skip_boring_words": "Remove commonly uninteresting words from the wordlist",
+        "mutate_case": (
+            "Also test case-mutated variants of each entry "
+            "(camelCase for snake_case/kebab-case, Title case for single words). "
+            "Skipped on URLs with case-insensitive backend extensions like .aspx/.cfm."
+        ),
+        "brute_short": (
+            "Generate every 1-, 2-, and 3-letter [a-z] combination and add to the wordlist. "
+            "Costs ~18,278 extra requests per host — opt-in for thorough scans."
+        ),
     }
     boring_words = {"utm_source", "utm_campaign", "utm_medium", "utm_term", "utm_content"}
     in_scope_only = True
     compare_mode = "getparam"
     default_wordlist = "paramminer_parameters.txt"
 
+    async def setup(self):
+        result = await super().setup()
+        if self.config.get("brute_short", False):
+            chars = string.ascii_lowercase
+            extra = set()
+            for length in (1, 2, 3):
+                extra |= {"".join(c) for c in itertools.product(chars, repeat=length)}
+            # respect global blacklist + boring words on generated combos
+            extra -= self.boring_words
+            extra -= self.global_blacklist
+            if self.global_blacklist_prefixes:
+                extra = {w for w in extra if not w.startswith(self.global_blacklist_prefixes)}
+            self.wl |= extra
+            self.debug(f"brute_short: added {len(extra)} 1-3 letter combinations")
+        return result
+
+    def _mutate_for_url(self, url, words):
+        if not self.config.get("mutate_case", False):
+            return words
+        path = urlparse(url).path.lower()
+        for ext in self.case_insensitive_extensions:
+            if path.endswith(ext):
+                return words
+        mutations = {m for m in (_mutate_case(w) for w in words) if m}
+        return words | mutations
+
     async def check_batch(self, compare_helper, url, getparam_list):
         test_getparams = {p: self.rand_string(14) for p in getparam_list}
         return await compare_helper.compare(

bbot/modules/paramminer_headers.py

@@ -3,6 +3,28 @@
 from bbot.errors import HttpCompareError
 from bbot.modules.base import BaseModule
 
+_case_split = re.compile(r"[-_]+")
+
+
+def _mutate_case(word):
+    """
+    Multi-word (snake_case/kebab-case) → camelCase: ``user_id`` → ``userId``.
+    Single word → Title case: ``admin`` → ``Admin``.
+    Returns None if no useful mutation exists.
+    """
+    parts = _case_split.split(word)
+    if len(parts) >= 2:
+        head = parts[0]
+        tail = "".join(p[:1].upper() + p[1:] for p in parts[1:] if p)
+        if not tail:
+            return None
+        result = head + tail
+    else:
+        if not word or not word[0].islower():
+            return None
+        result = word[:1].upper() + word[1:]
+    return result if result != word else None
+
 
 class paramminer_headers(BaseModule):
     """
@@ -27,6 +49,21 @@ class paramminer_headers(BaseModule):
         "recycle_words": "Attempt to use words found during the scan on all other endpoints",
         "skip_boring_words": "Remove commonly uninteresting words from the wordlist",
     }
+    # URLs ending with these extensions are known to be case-insensitive — skip case mutation.
+    # (Used by paramminer_getparams and paramminer_cookies; HTTP headers are inherently
+    # case-insensitive per RFC 7230 so this isn't relevant to paramminer_headers itself.)
+    case_insensitive_extensions = {
+        ".aspx",
+        ".ashx",
+        ".ascx",
+        ".asmx",
+        ".axd",
+        ".cshtml",
+        ".vbhtml",
+        ".razor",
+        ".cfm",
+        ".cfc",
+    }
     scanned_hosts = []
     boring_words = {
         "accept",
@@ -95,17 +132,32 @@ async def setup(self):
         self.event_dict = {}
         self.already_checked = set()
 
+        # global parameter blacklist (shared with excavate) — known framework/CDN/tracker names
+        self.global_blacklist = {p.lower() for p in self.scan.config.get("parameter_blacklist", [])}
+        self.global_blacklist_prefixes = tuple(
+            p.lower() for p in self.scan.config.get("parameter_blacklist_prefixes", [])
+        )
+
         self.wl = {
             h.strip().lower() for h in self.helpers.read_file(self.wordlist_file) if len(h) > 0 and "%" not in h
         }
 
         # check against the boring list (if the option is set)
         if self.config.get("skip_boring_words", True):
             self.wl -= self.boring_words
+            self.wl -= self.global_blacklist
+            if self.global_blacklist_prefixes:
+                self.wl = {w for w in self.wl if not w.startswith(self.global_blacklist_prefixes)}
+
         self.extracted_words_master = set()
 
         return True
 
+    def _mutate_for_url(self, url, words):
+        """Hook for subclasses to expand a word set with URL-aware mutations
+        (e.g. paramminer_getparams adds case mutations on case-sensitive backends)."""
+        return words
+
     def rand_string(self, *args, **kwargs):
         return self.helpers.rand_string(*args, **kwargs)
 
@@ -166,8 +218,14 @@ async def handle_event(self, event):
         if event.type == "WEB_PARAMETER":
             parameter_name = event.data.get("name")
             if self.recycle_words or (event.data.get("type") == "SPECULATIVE"):
-                if self.config.get("skip_boring_words", True) and parameter_name in self.boring_words:
-                    return
+                if self.config.get("skip_boring_words", True):
+                    if parameter_name in self.boring_words:
+                        return
+                    lower_name = parameter_name.lower()
+                    if lower_name in self.global_blacklist:
+                        return
+                    if self.global_blacklist_prefixes and lower_name.startswith(self.global_blacklist_prefixes):
+                        return
                 if parameter_name not in self.wl:  # Ensure it's not already in the wordlist
                     self.debug(f"Adding {parameter_name} to wordlist")
                     self.extracted_words_master.add(parameter_name)
@@ -194,7 +252,7 @@ async def handle_event(self, event):
                 return
 
             try:
-                results = await self.do_mining(self.wl, url, batch_size, compare_helper)
+                results = await self.do_mining(self._mutate_for_url(url, self.wl), url, batch_size, compare_helper)
             except HttpCompareError as e:
                 self.debug(f"Encountered HttpCompareError: [{e}] for URL [{event.url}]")
             await self.process_results(event, results)
@@ -253,7 +311,9 @@ async def finish(self):
                 self.debug(f"Error initializing compare helper: {e}")
                 continue
             words_to_process = {
-                i for i in self.extracted_words_master.copy() if hash(i + url) not in self.already_checked
+                i
+                for i in self._mutate_for_url(url, self.extracted_words_master)
+                if hash(i + url) not in self.already_checked
             }
             try:
                 results = await self.do_mining(words_to_process, url, batch_size, compare_helper)

bbot/presets/web/lightfuzz-max.yml

@@ -1,7 +1,8 @@
-description: "Maximum fuzzing: everything in lightfuzz-heavy, plus WAF targets are no longer skipped, each unique parameter-value pair is fuzzed individually (no collapsing), common headers like X-Forwarded-For are fuzzed even if not observed, and potential parameters are speculated from JSON/XML response bodies. Significantly increases scan time."
+description: "Maximum fuzzing: everything in lightfuzz-heavy, plus the heavy paramminer variant (1-3 letter brute-force on GET params, case mutation on case-sensitive backends, recycle_words on all paramminer modules), WAF targets are no longer skipped, each unique parameter-value pair is fuzzed individually (no collapsing), common headers like X-Forwarded-For are fuzzed even if not observed, and potential parameters are speculated from JSON/XML response bodies. Significantly increases scan time."
 
 include:
   - lightfuzz-heavy
+  - paramminer-heavy
 
 config:
   url_querystring_collapse: False # in cases where the same parameter is observed multiple times, fuzz them individually instead of collapsing them into a single parameter

bbot/presets/web/paramminer-heavy.yml

@@ -0,0 +1,15 @@
+description: "Aggressive paramminer brute-force: enables 1-3 letter combination brute-force on GET parameters and case mutation (camelCase / Title-case variants) on case-sensitive backends. Significantly increases scan time."
+
+include:
+  - paramminer
+
+config:
+  modules:
+    paramminer_getparams:
+      brute_short: True
+      mutate_case: True
+      recycle_words: True
+    paramminer_headers:
+      recycle_words: True
+    paramminer_cookies:
+      recycle_words: True