chg: [interactive crawler] better error tracking and show error to user + forum account, only create cookiejar

Author: Terrtia

bin/crawlers/Crawler.py

@@ -207,25 +207,34 @@ def get_message(self):
                     capture_start = capture.get_start_time(r_str=False)
                     if capture_start == 0:
                         task = capture.get_task()
+                        error_message = f'Lacus returned an unknown capture state for task {task.uuid}'
+                        crawlers.set_interactive_session_error_by_capture(capture.uuid, error_message)
                         task.delete()
                         capture.delete()
                         self.logger.warning(f'capture UNKNOWN ERROR STATE, {task.uuid} Removed from queue')
                         return None
                     if int(time.time()) - capture_start > 600:  # TODO ADD in new crawler config
                         task = capture.get_task()
-                        task.reset()
-                        capture.delete()
-                        self.logger.warning(f'capture UNKNOWN Timeout, {task.uuid} Send back in queue')
+                        error_message = f'Lacus capture UNKNOWN timeout for task {task.uuid}'
+                        if crawlers.set_interactive_session_error_by_capture(capture.uuid, error_message):
+                            self.logger.warning(f'capture UNKNOWN Timeout, {task.uuid} Interactive session failed')
+                        else:
+                            task.reset()
+                            capture.delete()
+                            self.logger.warning(f'capture UNKNOWN Timeout, {task.uuid} Send back in queue')
                     else:
                         capture.update(status)
                 elif status == crawlers.CaptureStatus.QUEUED:
                     capture_start = capture.get_start_time(r_str=False)
                     if int(time.time()) - capture_start > 36000:  # TODO ADD in new crawler config
                         task = capture.get_task()
-                        task.reset()
-                        capture.delete()
-                        self.logger.warning(
-                            f'capture QUEUED Timeout, {task.uuid}, {task.get_url()} Send back in queue, start_time={capture_start}')
+                        error_message = f'Lacus capture QUEUED timeout for task {task.uuid}'
+                        if crawlers.set_interactive_session_error_by_capture(capture.uuid, error_message):
+                            self.logger.warning(f'capture QUEUED Timeout, {task.uuid}, {task.get_url()} Interactive session failed, start_time={capture_start}')
+                        else:
+                            task.reset()
+                            capture.delete()
+                            self.logger.warning(f'capture QUEUED Timeout, {task.uuid}, {task.get_url()} Send back in queue, start_time={capture_start}')
                     else:
                         capture.update(status)
                     print(capture.uuid, crawlers.CaptureStatus(status).name, int(time.time()))
@@ -235,9 +244,13 @@ def get_message(self):
                 # Invalid State
                 else:
                     task = capture.get_task()
-                    task.reset()
-                    capture.delete()
-                    self.logger.warning(f'ERROR INVALID CAPTURE STATUS {status}, {task.uuid} Send back in queue')
+                    error_message = f'Lacus returned invalid capture status {status} for task {task.uuid}'
+                    if crawlers.set_interactive_session_error_by_capture(capture.uuid, error_message):
+                        self.logger.warning(f'ERROR INVALID CAPTURE STATUS {status}, {task.uuid} Interactive session failed')
+                    else:
+                        task.reset()
+                        capture.delete()
+                        self.logger.warning(f'ERROR INVALID CAPTURE STATUS {status}, {task.uuid} Send back in queue')
 
             except ConnectionError:
                 self.logger.warning(f'Lacus ConnectionError, capture {capture.uuid}')
@@ -334,6 +347,20 @@ def compute(self, capture):  # TODO ADD FUNCTION TO MANUALLY IMPORT ???
         entries = self.lacus.get_capture(capture.uuid)
 
         print(entries.get('status'))
+        if task.is_cookiejar_only():
+            if entries.get('error'):
+                error_message = str(entries['error'])
+                self.logger.warning(error_message)
+                crawlers.set_interactive_session_error_by_capture(capture.uuid, error_message)
+            else:
+                cookiejar_saved = crawlers.finalize_interactive_cookiejar_session(capture.uuid, entries.get('storage', {}))
+                if cookiejar_saved is False:
+                    crawlers.release_interactive_session_by_capture(capture.uuid, status='error')
+                else:
+                    crawlers.release_interactive_session_by_capture(capture.uuid, status='completed')
+            task.remove()
+            self.root_item = None
+            return None
         self.har = task.get_har()
         self.screenshot = task.get_screenshot()
         # DEBUG
@@ -399,11 +426,15 @@ def compute(self, capture):  # TODO ADD FUNCTION TO MANUALLY IMPORT ???
 
     def save_capture_response(self, capture, task, parent_id, entries):
         filter_page = False
+        crawled_domain = None
+        crawled_url = None
         print(entries.keys())
         if 'error' in entries:
             # TODO IMPROVE ERROR MESSAGE
             error_message = str(entries['error'])
             self.logger.warning(str(entries['error']))
+            if crawlers.set_interactive_session_error_by_capture(capture.uuid, error_message):
+                return False
             if error_message.startswith('Something went poorly'):
                 # Timeout, require restart of lacus
                 if 'Too many open files' in error_message:
@@ -559,11 +590,16 @@ def save_capture_response(self, capture, task, parent_id, entries):
                         fav = Favicons.create(favicon)
                         fav.add(item.get_date(), item)
 
+                crawled_domain = self.domain.id
+                crawled_url = last_url
+
         # Next Children
         entries_children = entries.get('children')
         if entries_children:
             for children in entries_children:
                 self.save_capture_response(capture, task, parent_id, children)
+        if crawled_domain:
+            crawlers.set_interactive_session_crawled_domain_by_capture(capture.uuid, crawled_domain, url=crawled_url)
         return True
 
 

bin/lib/crawlers.py

@@ -2393,6 +2393,21 @@ def get_interactive_session_by_capture(capture_uuid):
             return session
     return None
 
+def set_interactive_session_error_by_capture(capture_uuid, error_message, status='error'):
+    session = get_interactive_session_by_capture(capture_uuid)
+    if session and session.exists():
+        session.set('error', str(error_message))
+        session.release(status=status)
+        return True
+    return False
+
+def set_interactive_session_crawled_domain_by_capture(capture_uuid, domain, url=None):
+    session = get_interactive_session_by_capture(capture_uuid)
+    if session and session.exists():
+        session.set('crawled_domain', domain)
+        if url:
+            session.set('crawled_url', url)
+
 def release_interactive_session_by_capture(capture_uuid, status='completed'):
     session = get_interactive_session_by_capture(capture_uuid)
     if session and session.exists():
@@ -2454,12 +2469,12 @@ def _remote_headed_response_to_meta(response):
 def finalize_interactive_cookiejar_session(capture_uuid, storage):
     session = get_interactive_session_by_capture(capture_uuid)
     if not session or session.get('save_cookiejar') != '1':
-        return
+        return None
     try:
         if not storage or not isinstance(storage, dict):
             print('storage error')
             session.set('error', 'No cookies or local storage returned by the interactive browser')
-            return
+            return False
         cookiejar_uuid = create_cookiejar(session.get('user_org'), session.get_user(), session.get('cookiejar_description'), 0, None)
         print(f'New cookiejar {cookiejar_uuid}')
         cookiejar = Cookiejar(cookiejar_uuid)
@@ -2476,8 +2491,10 @@ def finalize_interactive_cookiejar_session(capture_uuid, storage):
                 account.set_cookiejar_uuid(cookiejar_uuid)
                 account.set_status('waiting')
                 forum.refresh_account_availability(account_id)
+        return True
     except Exception as e:
         session.set('error', str(e))
+        return False
 
 def refresh_interactive_session_status(session):
     capture_uuid = session.get_capture_uuid()
@@ -2578,14 +2595,29 @@ def api_start_interactive_capture(data, user_org, user_id):
     if error:
         return error, code
     try:
-        task_uuid = create_task(task['url'], depth=0, har=task['har'], screenshot=task['screenshot'], proxy=task['proxy'], tags=task['tags'], parent='interactive', priority=90, external=True)
+        cookiejar_only = bool(data.get('cookiejar_only'))
+        if cookiejar_only:
+            har = False
+            screenshot = False
+            tags = []
+            with_favicon = False
+        else:
+            har = task['har']
+            screenshot = task['screenshot']
+            tags = task['tags']
+            with_favicon = True
+
+        task_uuid = create_task(task['url'], depth=0, har=har, screenshot=screenshot, proxy=task['proxy'], tags=tags, parent='interactive', priority=90, external=True)
         if not task_uuid:
             session.release(status='error')
             return {'error': 'Aborted by Crawler'}, 400
         session.set('task_uuid', task_uuid)
+        if cookiejar_only:
+            CrawlerTask(task_uuid).set_cookiejar_only()
+            session.set('cookiejar_only', '1')
         capture_uuid = session.uuid
         lacus = get_lacus()
-        returned_uuid = lacus.enqueue(url=task['url'], depth=0, proxy=task['proxy'], with_favicon=True, force=True, uuid=capture_uuid, remote_headfull=True, java_script_enabled=task['javascript'], general_timeout_in_sec=int(data.get('general_timeout_in_sec') or 90))
+        returned_uuid = lacus.enqueue(url=task['url'], depth=0, proxy=task['proxy'], with_favicon=with_favicon, force=True, uuid=capture_uuid, remote_headfull=True, java_script_enabled=task['javascript'], general_timeout_in_sec=int(data.get('general_timeout_in_sec') or 90))
         capture_uuid = returned_uuid or capture_uuid
         session.set('capture_uuid', capture_uuid)
         r_cache.hset('crawler:interactive:captures', capture_uuid, session.uuid)
@@ -2849,6 +2881,12 @@ def get_har(self):
     def get_screenshot(self):
         return r_crawler.hget(f'crawler:task:{self.uuid}', 'screenshot') == '1'
 
+    def is_cookiejar_only(self):
+        return r_crawler.hget(f'crawler:task:{self.uuid}', 'cookiejar_only') == '1'
+
+    def set_cookiejar_only(self):
+        return self._set_field('cookiejar_only', 1)
+
     def get_queue(self):
         return r_crawler.hget(f'crawler:task:{self.uuid}', 'queue')
 

var/www/blueprints/forums_explorer.py

@@ -202,6 +202,7 @@ def forum_explorer_crawler_account_interactive_cookiejar():
         'javascript': config.get('javascript'),
         'general_timeout_in_sec': 300,
         'save_cookiejar': True,
+        'cookiejar_only': True,
     }
     user_org = current_user.get_org()
     user_id = current_user.get_user_id()

var/www/templates/crawler/crawler_splash/interactive_capture.html

@@ -25,12 +25,17 @@
             {% endif %}
             <p id="remote_status_row" {% if not session.remote_status or session.status in ['finishing', 'completed', 'expired', 'error', 'closed'] %}style="display:none;"{% endif %}><b>Remote browser status:</b> <span id="remote_status">{{ session.remote_status }}</span></p>
             <p id="capture_status_row" {% if not session.capture_status %}style="display:none;"{% endif %}><b>Capture status:</b> <span id="capture_status">{{ session.capture_status }}</span></p>
+            <p id="crawled_domain_row" {% if not session.crawled_domain %}style="display:none;"{% endif %}>
+                <b>Crawled domain:</b>
+                <span id="crawled_url" class="text-muted" {% if not session.crawled_url %}style="display:none;"{% endif %}>({{ session.crawled_url }})</span>
+                <a id="crawled_domain_link" href="{{ url_for('crawler_splash.showDomain') }}?domain={{ session.crawled_domain|urlencode }}">{{ session.crawled_domain }}</a>
+            </p>
             <a id="remote_url_btn" class="btn btn-primary" target="_blank" rel="noopener" href="{{ session.remote_url }}" {% if not session.remote_url or session.status in ['finishing', 'completed', 'expired', 'error', 'closed'] %}style="display:none;"{% endif %}><i class="fas fa-external-link-alt"></i> Open interactive browser</a>
             <div id="remote_starting" class="alert alert-info" {% if session.remote_url or session.status in ['finishing', 'completed', 'expired', 'error', 'closed'] %}style="display:none;"{% endif %}>
                 The interactive browser is starting. This page updates automatically; the browser link will appear as soon as it is ready.
             </div>
             <div id="capture_processing" class="alert alert-success" {% if session.status not in ['finishing', 'completed'] %}style="display:none;"{% endif %}>
-                The interactive session is completed. AIL is saving and processing the browser state; results will appear in AIL when processing is finished.
+                The interactive session is completed. AIL is saving and processing the browser state; results URL will appear in AIL when processing is finished.
             </div>
             {% if session.status in ['starting', 'ready'] %}
                 <form class="mt-3" method="post" action="{{ url_for('crawler_splash.interactive_capture_finish', uuid=session.uuid) }}">
@@ -79,6 +84,13 @@
             $('#cookiejar_link').attr('href', '{{ url_for('crawler_splash.crawler_cookiejar_show') }}?uuid=' + data.cookiejar_uuid).text(data.cookiejar_uuid);
             $('#cookiejar_row').show();
         }
+        if(data.crawled_domain){
+            $('#crawled_domain_link').attr('href', '{{ url_for('crawler_splash.showDomain') }}?domain=' + encodeURIComponent(data.crawled_domain)).text(data.crawled_domain);
+            if(data.crawled_url){
+                $('#crawled_url').text('(' + data.crawled_url + ')').show();
+            }
+            $('#crawled_domain_row').show();
+        }
         if(data.error || data.last_status_error){
             $('#interactive_error').text(data.error || data.last_status_error).show();
         }