Merge branch 'master' into develop

Author: Terrtia

bin/crawlers/Crawler.py

@@ -344,7 +344,7 @@ def compute(self, capture):  # TODO ADD FUNCTION TO MANUALLY IMPORT ???
         self.root_item = None
 
         # Save Capture
-        saved = self.save_capture_response(parent_id, entries)
+        saved = self.save_capture_response(capture, task, parent_id, entries)
         if saved:
             if self.parent != 'lookup':
                 # Update domain first/last seen
@@ -395,16 +395,22 @@ def compute(self, capture):  # TODO ADD FUNCTION TO MANUALLY IMPORT ???
         task.remove()
         self.root_item = None
 
-    def save_capture_response(self, parent_id, entries):
+    def save_capture_response(self, capture, task, parent_id, entries):
         filter_page = False
         print(entries.keys())
         if 'error' in entries:
             # TODO IMPROVE ERROR MESSAGE
+            error_message = str(entries['error'])
             self.logger.warning(str(entries['error']))
-            print(entries.get('error'))
+            if error_message.startswith('Something went poorly'):
+                # Timeout, require restart of lacus
+                if 'Too many open files' in error_message:
+                    task.reset()
+                    capture.delete()
+                    self.logger.warning(f'Lacus Too many open files Error, {task.uuid} Send back in queue')
+                    time.sleep(60)
             if entries.get('html'):
                 print('retrieved content')
-                # print(entries.get('html'))
 
         if 'last_redirected_url' in entries and entries.get('last_redirected_url'):  # TODO ADD RELATIONSHIP REDIRECT
             last_url = entries['last_redirected_url']
@@ -551,7 +557,7 @@ def save_capture_response(self, parent_id, entries):
         entries_children = entries.get('children')
         if entries_children:
             for children in entries_children:
-                self.save_capture_response(parent_id, children)
+                self.save_capture_response(capture, task, parent_id, children)
         return True
 
 

bin/lib/crawlers.py

@@ -2367,6 +2367,18 @@ def recrawl_onion_domains(date_month=None, all_onions_up=False):  # TODO RENAME
     for onion in to_crawl:
         recrawl_domain(onion)
 
+def recrawl_onion_domains_down_this_month(date_month=None):
+    """Resend all onion domains marked as down this month to the crawler queue."""
+    onion_domains = set()
+    for date in Date.get_month_dates(date=date_month):
+        onion_domains.update(r_crawler.smembers(f'onion_down:{date}'))
+
+    for onion_domain in onion_domains:
+        recrawl_domain(onion_domain)
+
+    return len(onion_domains)
+
+
 ## -- CRAWLER TASK -- ##
 
 #### CRAWLER TASK API ####