Skip to content

Commit 9aefafc

Browse files
liquidsecliquidsec
committed
Fix tests for blasthttp migration: httpx_mock→blasthttp_mock, whitelist→targets, VULNERABILITY→FINDING, httpx→http module refs
1 parent 9b3b3f4 commit 9aefafc

4 files changed

Lines changed: 25 additions & 54 deletions

File tree

‎bbot/modules/templates/subdomain_enum.py‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -171,8 +171,6 @@ async def filter_event(self, event):
171171
# reject if it's a cloud resource and not in our target (unless it's a seed event)
172172
if is_cloud and not self.scan.in_target(event) and "seed" not in event.tags:
173173
return False, "Event is a cloud resource and not a direct target"
174-
# don't reject targets — if the user explicitly targeted a domain, always process it
175-
is_target = event in self.scan.target.whitelist
176174
# optionally reject events with wildcards / errors
177175
if self.reject_wildcards:
178176
if any(t in event.tags for t in ("a-error", "aaaa-error")):

‎bbot/test/test_step_1/test_helpers.py‎

Lines changed: 0 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -978,7 +978,6 @@ async def test_rm_temp_dir_at_exit(helpers):
978978
assert not temp_dir.exists()
979979

980980

981-
982981
# these must be top-level functions so they can be pickled for the subprocess
983982
def _hang_forever():
984983
import time
@@ -1173,29 +1172,3 @@ def test_clean_dns_record():
11731172
assert clean_dns_record("'d1jwhzvlef5tfb.example.com'") == "d1jwhzvlef5tfb.example.com"
11741173
# quotes + trailing dot
11751174
assert clean_dns_record('"d1jwhzvlef5tfb.example.com."') == "d1jwhzvlef5tfb.example.com"
1176-
1177-
1178-
# these must be top-level functions so they can be pickled for the subprocess
1179-
def _hang_forever():
1180-
import time
1181-
1182-
time.sleep(9999)
1183-
1184-
1185-
def _cpu_work(n):
1186-
return sum(range(n))
1187-
1188-
1189-
@pytest.mark.asyncio
1190-
async def test_run_in_executor_mp(helpers):
1191-
# normal tasks should complete fine
1192-
result = await helpers.run_in_executor_mp(_cpu_work, 100_000)
1193-
assert result == sum(range(100_000))
1194-
1195-
# a hanging task should raise TimeoutError
1196-
with pytest.raises(asyncio.TimeoutError):
1197-
await helpers.run_in_executor_mp(_hang_forever, _timeout=2)
1198-
1199-
# pool should still work after a timeout
1200-
result = await helpers.run_in_executor_mp(_cpu_work, 50_000, _timeout=30)
1201-
assert result == sum(range(50_000))

‎bbot/test/test_step_2/module_tests/test_module_excavate.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1301,7 +1301,7 @@ class TestExcavate_webparameter_ip_host(ModuleTestBase):
13011301
"""
13021302

13031303
targets = ["http://localhost:8888"]
1304-
modules_overrides = ["httpx", "excavate", "hunt"]
1304+
modules_overrides = ["http", "excavate", "hunt"]
13051305
config_overrides = {"interactsh_disable": True}
13061306

13071307
async def setup_after_prep(self, module_test):

‎bbot/test/test_step_2/module_tests/test_module_wayback.py‎

Lines changed: 24 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -21,8 +21,8 @@ def check(self, module_test, events):
2121

2222
class TestWaybackParameters(ModuleTestBase):
2323
module_name = "wayback"
24+
targets = ["blacklanternsecurity.com", "127.0.0.1"]
2425
modules_overrides = ["wayback", "hunt"]
25-
whitelist = ["blacklanternsecurity.com", "127.0.0.1"]
2626
config_overrides = {"modules": {"wayback": {"urls": True, "parameters": True}}}
2727

2828
async def setup_after_prep(self, module_test):
@@ -61,7 +61,7 @@ def check(self, module_test, events):
6161
class TestWaybackInterestingFiles(ModuleTestBase):
6262
module_name = "wayback"
6363
modules_overrides = ["wayback"]
64-
whitelist = ["blacklanternsecurity.com", "127.0.0.1"]
64+
targets =["blacklanternsecurity.com", "127.0.0.1"]
6565
config_overrides = {"modules": {"wayback": {"urls": True}}}
6666

6767
async def setup_after_prep(self, module_test):
@@ -89,7 +89,7 @@ def check(self, module_test, events):
8989
class TestWaybackArchive(ModuleTestBase):
9090
module_name = "wayback"
9191
modules_overrides = ["wayback", "badsecrets", "excavate"]
92-
whitelist = ["blacklanternsecurity.com", "127.0.0.1"]
92+
targets =["blacklanternsecurity.com", "127.0.0.1"]
9393
config_overrides = {"modules": {"wayback": {"urls": True, "archive": True}}}
9494

9595
sample_viewstate = """<html>
@@ -121,17 +121,17 @@ def check(self, module_test, events):
121121
# the dead URL (port 1) should NOT be verified as live
122122
assert not any(e.type == "URL" and "deadpage" in e.url for e in events)
123123
# badsecrets should have found the vulnerability in the archived viewstate
124-
assert any(e.type == "VULNERABILITY" and "Known Secret Found." in e.data["description"] for e in events), (
124+
assert any(e.type == "FINDING" and "Known Secret Found." in e.data["description"] for e in events), (
125125
"Failed to detect badsecrets vulnerability from archived content"
126126
)
127127
# the vulnerability should reference the original URL, with "from-wayback" tag for provenance
128128
for e in events:
129-
if e.type == "VULNERABILITY" and "Known Secret Found." in e.data["description"]:
129+
if e.type == "FINDING" and "Known Secret Found." in e.data["description"]:
130130
assert "127.0.0.1" in e.data["url"], (
131-
f"VULNERABILITY url should contain the original host, got: {e.data['url']}"
131+
f"FINDING url should contain the original host, got: {e.data['url']}"
132132
)
133133
assert "web.archive.org" not in e.data["url"], (
134-
f"VULNERABILITY url should NOT be an archive.org URL, got: {e.data['url']}"
134+
f"FINDING url should NOT be an archive.org URL, got: {e.data['url']}"
135135
)
136136
# web.archive.org should NOT appear as a DNS_NAME event
137137
assert not any(e.type == "DNS_NAME" and e.data == "web.archive.org" for e in events), (
@@ -144,7 +144,7 @@ class TestWaybackHttpHttpsDedup(ModuleTestBase):
144144

145145
module_name = "wayback"
146146
modules_overrides = ["wayback"]
147-
whitelist = ["blacklanternsecurity.com"]
147+
targets =["blacklanternsecurity.com"]
148148
config_overrides = {"modules": {"wayback": {"urls": True}}}
149149

150150
async def setup_after_prep(self, module_test):
@@ -171,7 +171,7 @@ class TestWaybackHttpOnlyKept(ModuleTestBase):
171171

172172
module_name = "wayback"
173173
modules_overrides = ["wayback"]
174-
whitelist = ["blacklanternsecurity.com"]
174+
targets =["blacklanternsecurity.com"]
175175
config_overrides = {"modules": {"wayback": {"urls": True}}}
176176

177177
async def setup_after_prep(self, module_test):
@@ -196,7 +196,7 @@ class TestWaybackCdnCgiBlacklist(ModuleTestBase):
196196

197197
module_name = "wayback"
198198
modules_overrides = ["wayback"]
199-
whitelist = ["blacklanternsecurity.com"]
199+
targets =["blacklanternsecurity.com"]
200200
config_overrides = {"modules": {"wayback": {"urls": True}}}
201201

202202
async def setup_after_prep(self, module_test):
@@ -225,7 +225,7 @@ class TestWaybackArchiveHostField(ModuleTestBase):
225225

226226
module_name = "wayback"
227227
modules_overrides = ["wayback", "excavate"]
228-
whitelist = ["blacklanternsecurity.com", "127.0.0.1"]
228+
targets =["blacklanternsecurity.com", "127.0.0.1"]
229229
config_overrides = {"modules": {"wayback": {"urls": True, "archive": True}}}
230230

231231
async def setup_after_prep(self, module_test):
@@ -268,7 +268,7 @@ class TestWaybackArchiveHuntFinding(ModuleTestBase):
268268

269269
module_name = "wayback"
270270
modules_overrides = ["wayback", "excavate", "hunt"]
271-
whitelist = ["blacklanternsecurity.com", "127.0.0.1"]
271+
targets =["blacklanternsecurity.com", "127.0.0.1"]
272272
config_overrides = {"modules": {"wayback": {"urls": True, "archive": True}}}
273273

274274
async def setup_after_prep(self, module_test):
@@ -340,8 +340,8 @@ class TestWaybackLightfuzzXSS(ModuleTestBase):
340340

341341
module_name = "wayback"
342342
targets = ["blacklanternsecurity.com"]
343-
modules_overrides = ["wayback", "httpx", "lightfuzz", "excavate"]
344-
whitelist = ["blacklanternsecurity.com", "127.0.0.1"]
343+
modules_overrides = ["wayback", "http", "lightfuzz", "excavate"]
344+
targets =["blacklanternsecurity.com", "127.0.0.1"]
345345
config_overrides = {
346346
"interactsh_disable": True,
347347
"modules": {
@@ -454,7 +454,7 @@ class TestWaybackArchiveBloomDedup(ModuleTestBase):
454454

455455
module_name = "wayback"
456456
modules_overrides = ["wayback"]
457-
whitelist = ["blacklanternsecurity.com", "127.0.0.1"]
457+
targets =["blacklanternsecurity.com", "127.0.0.1"]
458458
config_overrides = {"modules": {"wayback": {"urls": True, "archive": True}}}
459459

460460
async def setup_after_prep(self, module_test):
@@ -499,7 +499,7 @@ class TestWaybackArchiveRetry(ModuleTestBase):
499499

500500
module_name = "wayback"
501501
modules_overrides = ["wayback"]
502-
whitelist = ["blacklanternsecurity.com", "127.0.0.1"]
502+
targets =["blacklanternsecurity.com", "127.0.0.1"]
503503
config_overrides = {"modules": {"wayback": {"urls": True, "archive": True}}}
504504

505505
async def setup_after_prep(self, module_test):
@@ -531,15 +531,15 @@ class TestWaybackGarbageUrlFilter(ModuleTestBase):
531531

532532
module_name = "wayback"
533533
modules_overrides = ["wayback"]
534-
whitelist = ["blacklanternsecurity.com"]
534+
targets =["blacklanternsecurity.com"]
535535
config_overrides = {"modules": {"wayback": {"urls": True}}}
536536

537537
async def setup_after_prep(self, module_test):
538538
# build a crawler-trap URL with repeating path segments (like the real-world example)
539539
repeating = "/themes/sites/example.com".lstrip("/")
540540
garbage_path = "/get-materials/" + "/".join([repeating] * 20)
541541
garbage_url = f"https://blacklanternsecurity.com{garbage_path}"
542-
module_test.httpx_mock.add_response(
542+
module_test.blasthttp_mock.add_response(
543543
url="http://web.archive.org/cdx/search/cdx?url=blacklanternsecurity.com&matchType=domain&output=json&fl=original&collapse=original&limit=100000&filter=!statuscode:404&filter=!statuscode:301&filter=!statuscode:302&filter=!mimetype:image/.*&filter=!mimetype:text/css&filter=!mimetype:warc/revisit",
544544
json=[
545545
["original"],
@@ -564,13 +564,13 @@ class TestWaybackGarbageUrlLength(ModuleTestBase):
564564

565565
module_name = "wayback"
566566
modules_overrides = ["wayback"]
567-
whitelist = ["blacklanternsecurity.com"]
567+
targets =["blacklanternsecurity.com"]
568568
config_overrides = {"modules": {"wayback": {"urls": True}}}
569569

570570
async def setup_after_prep(self, module_test):
571571
# URL exceeding 2000 character limit
572572
long_url = "https://blacklanternsecurity.com/" + "a" * 2000
573-
module_test.httpx_mock.add_response(
573+
module_test.blasthttp_mock.add_response(
574574
url="http://web.archive.org/cdx/search/cdx?url=blacklanternsecurity.com&matchType=domain&output=json&fl=original&collapse=original&limit=100000&filter=!statuscode:404&filter=!statuscode:301&filter=!statuscode:302&filter=!mimetype:image/.*&filter=!mimetype:text/css&filter=!mimetype:warc/revisit",
575575
json=[
576576
["original"],
@@ -595,26 +595,26 @@ class TestWaybackArchive429Retry(ModuleTestBase):
595595

596596
module_name = "wayback"
597597
modules_overrides = ["wayback"]
598-
whitelist = ["blacklanternsecurity.com", "127.0.0.1"]
598+
targets =["blacklanternsecurity.com", "127.0.0.1"]
599599
config_overrides = {"modules": {"wayback": {"urls": True, "archive": True}}}
600600

601601
async def setup_after_prep(self, module_test):
602602
# speed up delays for testing
603603
module_test.scan.modules["wayback"]._archive_429_default_delay = 0.01
604604
module_test.scan.modules["wayback"]._archive_error_delay = 0.01
605605
module_test.scan.modules["wayback"]._archive_delay = 0
606-
module_test.httpx_mock.add_response(
606+
module_test.blasthttp_mock.add_response(
607607
url="http://web.archive.org/cdx/search/cdx?url=blacklanternsecurity.com&matchType=domain&output=json&fl=original&collapse=original&limit=100000&filter=!statuscode:404&filter=!statuscode:301&filter=!statuscode:302&filter=!mimetype:image/.*&filter=!mimetype:text/css&filter=!mimetype:warc/revisit",
608608
json=[["original"], ["http://127.0.0.1:1/rate-limited-page"]],
609609
)
610610
# first attempt: 429 rate limited
611-
module_test.httpx_mock.add_response(
611+
module_test.blasthttp_mock.add_response(
612612
url="http://web.archive.org/web/http://127.0.0.1:1/rate-limited-page",
613613
status_code=429,
614614
headers={"Retry-After": "1"},
615615
)
616616
# retry after backoff: 200
617-
module_test.httpx_mock.add_response(
617+
module_test.blasthttp_mock.add_response(
618618
url="http://web.archive.org/web/http://127.0.0.1:1/rate-limited-page",
619619
text="<html><body>content after rate limit</body></html>",
620620
headers={"Content-Type": "text/html"},

0 commit comments

Comments
 (0)