ensure 1 second delay between requests

Author: github-actions

bbot/modules/shodan_idb.py

@@ -1,3 +1,5 @@
+import time
+
 from bbot.modules.base import BaseModule
 
 
@@ -50,11 +52,16 @@ class shodan_idb(BaseModule):
     # we get lots of 404s, that's normal
     _api_failure_abort_threshold = 9999999999
 
-    # there aren't any rate limits to speak of, so our outgoing queue can be pretty big
-    _qsize = 500
+    # since there are rate limits, we set a lower qsize
+    # this way when our queue is full, we don't overwhelm the API
+    _qsize = 100
 
     base_url = "https://internetdb.shodan.io"
 
+    async def setup(self):
+        await super().setup()
+        self.last_request_time = 0
+
     def _incoming_dedup_hash(self, event):
         return hash(self.get_ip(event))
 
@@ -63,6 +70,16 @@ async def handle_event(self, event):
         if ip is None:
             return
         url = f"{self.base_url}/{ip}"
+
+        # Rate limiting: ensure at least 1 second between requests
+        current_time = time.time()
+        time_since_last = current_time - self.last_request_time
+        if time_since_last < 1:
+            await self.helpers.sleep(1 - time_since_last)
+
+        # Update the last request time
+        self.last_request_time = time.time()
+
         r = await self.api_request(url)
         if r is None:
             self.debug(f"No response for {event.data}")

bbot/test/test_step_2/module_tests/test_module_shodan_idb.py

@@ -58,3 +58,26 @@ def check(self, module_test, events):
                 if e.type == "TECHNOLOGY" and e.data["technology"] == "cpe:/a:microsoft:outlook_web_access:15.0.1367"
             ]
         )
+
+
+class TestShodan_IDB_RateLimit(ModuleTestBase):
+    config_overrides = {"dns": {"minimal": False}}
+    module_name = "shodan_idb"
+
+    async def setup_before_prep(self, module_test):
+        await module_test.mock_dns(
+            {
+                "blacklanternsecurity.com": {"A": ["1.2.3.4"]},
+                "autodiscover.blacklanternsecurity.com": {"A": ["2.3.4.5"]},
+                "mail.blacklanternsecurity.com": {"A": ["3.4.5.6"]},
+            }
+        )
+
+        module_test.httpx_mock.add_response(
+            url="https://internetdb.shodan.io/1.2.3.4",
+            status_code=429,
+            json={"error": "Rate limit exceeded"},
+        )
+
+    def check(self, module_test, events):
+        pass