Skip to content

Commit a6cde82

Browse files
committed
fix: [crawler]invalid about:blank last_redirected_url
1 parent c0cb034 commit a6cde82

1 file changed

Lines changed: 4 additions & 0 deletions

File tree

bin/crawlers/Crawler.py

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -414,6 +414,10 @@ def save_capture_response(self, capture, task, parent_id, entries):
414414

415415
if 'last_redirected_url' in entries and entries.get('last_redirected_url'): # TODO ADD RELATIONSHIP REDIRECT
416416
last_url = entries['last_redirected_url']
417+
if last_url == 'about:blank':
418+
self.logger.warning(f'Error something fails, redirect: about:blank')
419+
return False
420+
417421
unpacked_last_url = crawlers.unpack_url(last_url)
418422
current_domain = unpacked_last_url['domain']
419423
# REDIRECTION TODO CHECK IF TYPE CHANGE

0 commit comments

Comments
 (0)