restore cloudcheck tag propagation to URL/OPEN_TCP_PORT children

bbot/core/event/base.py

@@ -790,7 +790,6 @@ def _minimize(self):
             # release container slots; lazy-init pattern means None == empty
             self._dns_children = None
             self._raw_dns_records = None
-            self._resolved_hosts = None
 
     def clone(self):
         # Create a shallow copy of the event first

bbot/modules/virtualhost.py

@@ -1019,10 +1019,10 @@ async def finish(self):
 
     async def filter_event(self, event):
         if (
-            "cdn-cloudflare" in event.tags
-            or "cdn-imperva" in event.tags
-            or "cdn-akamai" in event.tags
-            or "cdn-cloudfront" in event.tags
+            "cloudflare" in event.tags
+            or "imperva" in event.tags
+            or "akamai" in event.tags
+            or "cloudfront" in event.tags
         ):
             self.debug(f"Not processing URL {event.url} because it's behind a WAF or CDN, and that's pointless")
             return False

bbot/modules/waf_bypass.py

@@ -26,11 +26,16 @@ class Config(BaseModuleConfig):
         similarity_threshold: float = Field(0.90, description="Similarity threshold for content matching")
         search_ip_neighbors: bool = Field(True, description="Also check IP neighbors of the target domain")
         neighbor_cidr: int = Field(
-            24,
+            28,
             ge=24,
             le=31,
             description="CIDR mask (24-31) used for neighbor enumeration when search_ip_neighbors is true",
         )
+        max_concurrent_checks: int = Field(
+            100,
+            ge=1,
+            description="Maximum number of concurrent bypass-attempt HTTP checks in finish()",
+        )
 
     meta = {
         "description": "Detects potential WAF bypasses",
@@ -77,15 +82,15 @@ async def handle_event(self, event):
                         self.cloud_ips.add(ip_str)
                         self.debug(f"Added cloud-ip {ip_str} to cloud_ips")
                 else:
-                    self.warning(f"DNS resolution for {domain} returned non-IP result: {ip_str}")
+                    self.verbose(f"DNS resolution for {domain} returned non-IP result: {ip_str}")
         else:
             self.warning(f"DNS resolution failed for {domain}")
 
         # Detect WAF/CDN protection based on tags
         provider_name = None
-        if "cdn-cloudflare" in event.tags or "waf-cloudflare" in event.tags:
+        if "cloudflare" in event.tags:
             provider_name = "CloudFlare"
-        elif "cdn-imperva" in event.tags:
+        elif "imperva" in event.tags:
             provider_name = "Imperva"
 
         is_protected = provider_name is not None
@@ -189,8 +194,10 @@ async def finish(self):
                         continue
 
                     if ip not in waf_ips:  # And IP isn't a known WAF IP
-                        ip_bypass_candidates[ip] = domain
-                        self.debug(f"Added potential bypass IP {ip} from domain {domain}")
+                        # first attribution wins so logs are stable across scans
+                        if ip not in ip_bypass_candidates:
+                            ip_bypass_candidates[ip] = f"{domain} (direct)"
+                            self.debug(f"Added potential bypass IP {ip} from domain {domain}")
 
                         # if we have IP neighbors searching enabled, and the IP isn't a cloud IP, we can add the IP neighbors to our list of potential bypasses
                         if self.search_ip_neighbors and ip not in self.cloud_ips:
@@ -221,7 +228,9 @@ async def finish(self):
                                         self.debug(
                                             f"Added Neighbor IP ({ip} -> {neighbor_ip_str}) as potential bypass IP derived from {domain}"
                                         )
-                                        ip_bypass_candidates[neighbor_ip_str] = domain
+                                        ip_bypass_candidates[neighbor_ip_str] = (
+                                            f"{domain} (neighbor of {ip}/{self.neighbor_cidr})"
+                                        )
                     else:
                         self.debug(f"IP {ip} is in WAF IPS so we don't check as potential bypass")
 
@@ -245,7 +254,9 @@ async def finish(self):
         )
 
         self.debug(f"about to start {len(coros)} coroutines")
-        async for completed in self.helpers.as_completed(coros):
+        async for completed in self.helpers.as_completed(
+            coros, max_concurrent=int(self.config.get("max_concurrent_checks") or 100)
+        ):
             result = await completed
             if result:
                 confirmed_bypasses.append(result)

bbot/presets/waf-bypass.yml

@@ -16,4 +16,4 @@ config:
     waf_bypass:
       similarity_threshold: 0.90
       search_ip_neighbors: true
-      neighbor_cidr: 24 
+      neighbor_cidr: 28

bbot/test/test_step_2/module_tests/test_module_cloudcheck.py

@@ -123,6 +123,59 @@ async def setup_after_prep(self, module_test):
         assert "google" in child_disjoint.tags
         assert "cloud" in child_disjoint.tags
 
+        # ── _minimize() must not wipe resolved_hosts ───────────────────
+        # Real scan flow: a DNS_NAME gets processed by dnsresolve + cloudcheck,
+        # then Event._minimize() decrements its consumer count. Later, child
+        # events (URL, OPEN_TCP_PORT) that share the DNS_NAME's host reach
+        # dnsresolve and copy the parent's resolved_hosts (line 156). If
+        # _minimize() wiped resolved_hosts, that copy carries nothing and
+        # cloudcheck's per-host filter has no IPs to match against, so the
+        # cloud tag never inherits.
+        minimized = scan.make_event("minimize.evilcorp.com", parent=scan.root_event)
+        minimized.resolved_hosts = ("asdf.amazonaws.com", "10.0.0.1")
+        await module.handle_event(minimized)
+        # drive _module_consumers below 0 so the wipe block runs
+        minimized._minimize()
+        minimized._minimize()
+        assert minimized.resolved_hosts, (
+            "_minimize() must preserve resolved_hosts — dnsresolve line 156 copies it to child "
+            "URL/OPEN_TCP_PORT events, and cloudcheck's per-host inheritance needs those IPs"
+        )
+        # cloud host_metadata already survives minimization; confirm
+        assert "asdf.amazonaws.com" in minimized.host_metadata
+
+        # ── URL/OPEN_TCP_PORT propagation end-to-end ──────────────────
+        # Together with the _minimize() preservation above, this exercises
+        # the real flow: parent DNS_NAME cloud-tagged, then minimized, then a
+        # URL child gets its resolved_hosts populated (mirroring dnsresolve),
+        # and cloudcheck's existing subset-gate + per-host filter should copy
+        # the cloud tag onto the child.
+        url_child = scan.make_event(
+            "http://minimize.evilcorp.com/",
+            "URL",
+            parent=minimized,
+            tags=["status-200", "in-scope"],
+        )
+        # simulate dnsresolve intercept: child inherits parent's resolved_hosts
+        url_child._resolved_hosts = minimized.resolved_hosts
+        await module.handle_event(url_child)
+        assert "amazon" in url_child.tags, (
+            "URL child of amazon-tagged DNS_NAME must inherit the cloud tag "
+            "once its resolved_hosts carries the parent's IPs"
+        )
+        assert "cloud" in url_child.tags
+        assert "asdf.amazonaws.com" in url_child.host_metadata
+
+        port_child = scan.make_event(
+            "minimize.evilcorp.com:443",
+            "OPEN_TCP_PORT",
+            parent=minimized,
+        )
+        port_child._resolved_hosts = minimized.resolved_hosts
+        await module.handle_event(port_child)
+        assert "amazon" in port_child.tags, "OPEN_TCP_PORT child must inherit cloud tag from parent"
+        assert "asdf.amazonaws.com" in port_child.host_metadata
+
         # ── YARA prefilter short-circuit ───────────────────────────────
         # When no host could possibly match any bucket regex (mismatched
         # suffix) the prefilter returns no matches and we skip the Python

bbot/test/test_step_2/module_tests/test_module_virtualhost.py

@@ -1070,3 +1070,36 @@ def check(self, module_test, events):
             f"SAN analyzer received {type(san_arg).__name__}, expected str. Value: {san_arg!r}"
         )
         assert san_arg.startswith("https://"), f"Expected HTTPS URL, got {san_arg!r}"
+
+
+class TestVirtualhostSkipsCdnWaf(VirtualhostTestBase):
+    """filter_event must reject URLs tagged with any of the flat cloud-provider tags
+    that cloudcheck emits (post-`Migrate cloudcheck to host_metadata` refactor)."""
+
+    targets = ["http://localhost:8888"]
+    modules_overrides = ["virtualhost"]
+
+    async def setup_after_prep(self, module_test):
+        pass
+
+    async def check(self, module_test, events):
+        vh_module = module_test.scan.modules["virtualhost"]
+
+        for tag in ("cloudflare", "imperva", "akamai", "cloudfront"):
+            url_event = module_test.scan.make_event(
+                "http://cdn-test.local:8888/",
+                "URL",
+                parent=module_test.scan.root_event,
+                tags=[tag, "in-scope", "status-200"],
+            )
+            result = await vh_module.filter_event(url_event)
+            assert result is False, f"virtualhost must skip URL tagged {tag!r} (got {result!r})"
+
+        untagged_event = module_test.scan.make_event(
+            "http://plain.local:8888/",
+            "URL",
+            parent=module_test.scan.root_event,
+            tags=["in-scope", "status-200"],
+        )
+        result = await vh_module.filter_event(untagged_event)
+        assert result is True, f"virtualhost must not skip an untagged URL (got {result!r})"

bbot/test/test_step_2/module_tests/test_module_waf_bypass.py

@@ -43,7 +43,7 @@ async def handle_event(self, event):
                 self.events_seen.append(event.data)
                 url = "http://protected.test:8888/"
                 url_event = self.scan.make_event(
-                    url, "URL", parent=self.scan.root_event, tags=["cdn-cloudflare", "in-scope", "status-200"]
+                    url, "URL", parent=self.scan.root_event, tags=["cloudflare", "in-scope", "status-200"]
                 )
                 if url_event is not None:
                     await self.emit_event(url_event)
@@ -135,3 +135,58 @@ def check(self, module_test, events):
             in e.data["description"]
         ]
         assert correct_description, "Incorrect description"
+
+
+class TestWAFBypassTagRecognition(ModuleTestBase):
+    """Regression: waf_bypass keys off the flat cloud-provider tags cloudcheck emits
+    post-`Migrate cloudcheck to host_metadata` refactor. Both `cloudflare` and `imperva`
+    tags must trigger protected-domain detection."""
+
+    targets = ["evilcorp.com"]
+    modules_overrides = ["waf_bypass"]
+
+    async def setup_after_prep(self, module_test):
+        pass
+
+    async def check(self, module_test, events):
+        waf_module = module_test.scan.modules["waf_bypass"]
+
+        async def fake_resolve(host):
+            return []
+
+        module_test.monkeypatch.setattr(waf_module.helpers.dns, "resolve", fake_resolve)
+
+        async def fake_get_url_content(url, ip=None):
+            return None
+
+        import types
+
+        module_test.monkeypatch.setattr(
+            waf_module, "get_url_content", types.MethodType(fake_get_url_content, waf_module), raising=True
+        )
+
+        for tag in ("cloudflare", "imperva"):
+            waf_module.protected_domains = {}
+            url_event = module_test.scan.make_event(
+                f"http://{tag}-protected.test/",
+                "URL",
+                parent=module_test.scan.root_event,
+                tags=[tag, "in-scope", "status-200"],
+            )
+            await waf_module.handle_event(url_event)
+            assert f"{tag}-protected.test" in waf_module.protected_domains, (
+                f"waf_bypass did not recognize {tag!r}-tagged URL as protected"
+            )
+
+        # sanity: an untagged URL is not treated as protected
+        waf_module.protected_domains = {}
+        untagged = module_test.scan.make_event(
+            "http://plain.test/",
+            "URL",
+            parent=module_test.scan.root_event,
+            tags=["in-scope", "status-200"],
+        )
+        await waf_module.handle_event(untagged)
+        assert "plain.test" not in waf_module.protected_domains, (
+            "waf_bypass should not flag untagged URLs as protected"
+        )