chg: [forum] save post and user images + extract images from HAR + remove crawl item from forum crawl queue

Author: Terrtia

bin/importer/feeders/Forum_Extractor.py

@@ -22,6 +22,7 @@
 from lib.objects.Posts import Post
 from lib.objects.UsersAccount import UserAccount
 from lib.objects.Usernames import Username
+from lib.objects import Images
 
 # TODO IMAGES + USER ACCOUNTS + FILENAMES
 class Forum_ExtractorFeeder(DefaultFeeder):
@@ -35,6 +36,7 @@ def __init__(self, json_data):
         self.seen_subforums = set()
         self.root_subforums = set()
         self.objs_to_process = set()
+        self.har_images = {}
 
     def process_meta(self):
         """Import one forum-extractor result dictionary from self.json_meta."""
@@ -60,6 +62,7 @@ def _import_success(self, result):
         forum_type = result.get('forum_type')
         forum_id = result.get('forum_id')
         extracted = result.get('extracted') or {}
+        self.har_images = result.get('har_images')
 
         if not forum_type or not forum_id:
             # TODO Exception + logs
@@ -307,16 +310,48 @@ def _upsert_post(self, post_data, parent_thread):
         post.create(post_id, post_timestamp, content, parent_thread, self.forum, state=post_data.get('post_state'), quote_ids=quote_ids)
         for reaction in post_data.get('reactions', []):
             if isinstance(reaction, dict):
-                reaction_name = reaction.get('reaction')
+                reaction_name = reaction.get('reaction_type')
                 reaction_count = reaction.get('count')
             else:
                 reaction_name = None
                 reaction_count = None
             if reaction_name and reaction_count:
+                reaction_name = reaction_name.strip().lower().replace("-", "_").strip("_")
                 post.add_reaction(reaction_name, int(reaction_count))
+
+        # images
+        # print('--------------------------------------------------')
+        # print(post_data.get('content', {}).get('images', []))
+        for url in post_data.get('content', {}).get('images', []):
+            print('URL:', url)
+            if url in self.har_images:
+                image = self._create_image_from_har_url(url, post.get_date(), post)
+                self.objs_to_process.add(image)
+
         user_account = self._create_post_user_account(post_data.get('author'), post, post_timestamp)
+        if user_account:
+            author_image_url = (post_data.get('author', {})).get('author_profile_image_url')
+            if author_image_url in self.har_images:
+                image = self._create_image_from_har_url(author_image_url, post.get_date(), user_account)
+                if image:
+                    user_account.set_icon(image.get_global_id())
+                    self.objs_to_process.add(image)
         return post
 
+    def _create_image_from_har_url(self, url, date, obj):
+        print(url)
+        image_payload = self.har_images[url]
+        content = image_payload['content']
+        if not image_payload['b64'] and isinstance(content, str):
+            content = content.encode()
+        image = Images.create(content, b64=image_payload['b64'])
+        if image:
+            # create correlation image - obj # TODO correlation forum ?
+            image.add(date, obj)
+            self.objs_to_process.add(image)
+            print(image.id)
+        return image
+
     # TODO A POST MUST HAVE AN USER ACCOUNT
     def _create_post_user_account(self, author, post, timestamp):
         """Create/update the post author user-account and username metadata."""

bin/lib/correlations_engine.py

@@ -70,11 +70,11 @@
     "subforum": ["forum", "subforum", "forum-thread"],
     "forum-thread": ["subforum", "post"],
     # TODO Extend to detection of text -> same as message
-    "post": ["forum", "forum-thread", "user-account", *COMMON_TEXT_OBJECT_CORRELATIONS],
+    "post": ["forum", "forum-thread", "image", "user-account", *COMMON_TEXT_OBJECT_CORRELATIONS],
     "file-name": ["chat", "item", "message", "pdf"],
     "gtracker": ["domain", "item"],
     "hhhash": ["domain"],
-    "image": ["barcode", "chat", "chat-subchannel", "chat-thread", "message", "ocr", "qrcode", "user-account"],  # TODO subchannel + threads ????
+    "image": ["barcode", "chat", "chat-subchannel", "chat-thread", "message", "ocr", "post", "qrcode", "user-account"],  # TODO subchannel + threads ????
     "ip": ["ssh-key"],
     "item": ["cve", "cryptocurrency", "decoded", "domain", "dom-hash", "favicon", "file-name", "gtracker", "mail", "message", "pdf", "pgp", "screenshot", "title", "username"],  # chat ???
     "mail": ["domain", "item", "message"],  # chat ??

bin/lib/crawlers.py

@@ -7,9 +7,11 @@
 
 """
 import base64
+import binascii
 import gzip
 import hashlib
 import json
+import magic
 import os
 import pickle
 import re
@@ -72,6 +74,17 @@
 D_SCREENSHOT = config_loader.get_config_boolean('Crawler', 'default_screenshot')
 config_loader = None
 
+
+# IMAGE
+MAX_IMAGE_SIZE = 5000000
+ACCEPTED_IMAGE_MIME_TYPES = {
+    # 'image/gif',
+    'image/jpeg',
+    'image/png',
+    'image/webp',
+}
+
+
 # logger_crawler = logging.getLogger('crawlers.log')
 
 # # # # # # # #
@@ -750,6 +763,55 @@ def _gzip_all_hars():
     for har_id in get_all_har_ids():
         _gzip_har(har_id)
 
+
+def extract_images_from_har(har, size_limit=MAX_IMAGE_SIZE):
+    images = {}
+    if not har:
+        return images
+    for entry in har.get('log', {}).get('entries', []):
+        request = entry.get('request', {})
+        response = entry.get('response', {})
+        url = request.get('url')
+        content = response.get('content', {})
+        mime_type = content.get('mimeType', '')
+        body = content.get('text')
+        if not url or not body:
+            continue
+        is_b64 = content.get('encoding') == 'base64'
+        if response.get('status') != 200 or mime_type not in ACCEPTED_IMAGE_MIME_TYPES:
+            continue
+        image_content = _get_image_content_bytes(body, is_b64)
+        if not image_content:
+            continue
+        if 0 < size_limit < len(image_content):
+            print('HAR IMAGE SIZE LIMIT', url)
+            continue
+        detected_mime_type = magic.from_buffer(image_content, mime=True)
+        if detected_mime_type not in ACCEPTED_IMAGE_MIME_TYPES:
+            print('HAR IMAGE INVALID MIME TYPE', detected_mime_type, url)
+            continue
+        images[url] = {
+            'content': body,
+            'b64': is_b64,
+            'mime_type': detected_mime_type,
+        }
+    return images
+
+def _get_image_content_bytes(content, b64=False):
+    if b64:
+        try:
+            if isinstance(content, str):
+                content = ''.join(content.split())
+            return base64.b64decode(content, validate=True)
+        except (binascii.Error, ValueError):
+            return None
+    if isinstance(content, bytes):
+        return content
+    if isinstance(content, str):
+        return content.encode()
+    return None
+
+
 # # # - - # # #
 
 ################################################################################

bin/lib/forums_viewer.py

@@ -456,6 +456,15 @@ def purge_forum_crawl_queue(forum_id):
         'deleted': forum.purge_crawl_queue(),
     }, 200
 
+def remove_forum_pending_crawl_item(forum_id, crawl_key):
+    forum = Forums.Forum(forum_id)
+    if not forum.exists():
+        return {"status": "error", "reason": "Unknown forum"}, 404
+    removed = forum.remove_pending_crawl_item(crawl_key)
+    if not removed:
+        return {'status': 'error', 'reason': 'not_pending'}, 404
+    return {'forum_id': forum_id, 'crawl_key': crawl_key}, 200
+
 def get_breadcrumb_for_object(obj):
     """Return parent breadcrumb entries from Forum to the given object."""
     breadcrumb = []
@@ -548,7 +557,6 @@ def api_get_forum_thread(subtype, thread_id, page=1, nb=50, translation_target=N
     }, 200
 
 
-
 def api_get_post(post_id, translation_target=None):
     post = Posts.Post(post_id)
     if not post.exists():

bin/lib/objects/Forums.py

@@ -732,6 +732,12 @@ def _cleanup_crawl_item(self, crawl_key):
     def complete_crawl_item(self, crawl_key):
         return self._cleanup_crawl_item(crawl_key)
 
+    def remove_pending_crawl_item(self, crawl_key):
+        if not r_object.zrem(f'forum:crawl:queue:{self.id}', crawl_key):
+            return False
+        self._cleanup_crawl_item(crawl_key)
+        return True
+
     def fail_crawl_item(self, crawl_key, error=None):
         item = self.get_crawl_item(crawl_key)
         if item and item.get('type') == 'forum-thread':

bin/lib/objects/Images.py

@@ -189,6 +189,8 @@ def create(content, size_limit=5000000, b64=False, force=False):
         if not image.exists():
             image.create(content)
         return image
+    else:
+        print('LIMIT SIZE CREATE IMAGE')
 
 
 class Images(AbstractDaterangeObjects):

var/www/blueprints/forums_explorer.py

@@ -92,10 +92,21 @@ def forum_explorer_crawler_status():
     return render_template('forums_explorer_crawler_index.html', forums=forums, bootstrap_label=bootstrap_label)
 
 
-@forums_explorer.route("/forums/explorer/crawler/queue", methods=['GET'])
+@forums_explorer.route("/forums/explorer/crawler/queue", methods=['GET', 'POST'])
 @login_required
 @login_admin
 def forum_explorer_crawler_queue():
+    if request.method == 'POST':
+        forum_id = request.form.get('forum_id')
+        crawl_key = request.form.get('crawl_key')
+        sample_size = request.form.get('sample_size') or 50
+        res = forums_viewer.remove_forum_pending_crawl_item(forum_id, crawl_key)
+        if res[1] != 200:
+            error = res[0].get('reason')
+            return redirect(url_for('forums_explorer.forum_explorer_crawler_queue', id=forum_id, sample_size=sample_size, error=error))
+        success = f"Removed pending crawl item: {crawl_key}"
+        return redirect(url_for('forums_explorer.forum_explorer_crawler_queue', id=forum_id, sample_size=sample_size, success=success))
+
     forum_id = request.args.get('id')
     sample_size = request.args.get('sample_size') or 50
     meta = forums_viewer.api_get_forum_crawl_queue(forum_id, sample_size=sample_size)

var/www/templates/forums_explorer/forums_explorer_crawler_queue.html

@@ -59,7 +59,7 @@ <h1 class="mb-2"><i class="fas fa-list"></i> Forum crawl queue</h1>
   <div class="forum-board-header">Pending crawl items</div>
   {% if queue.get('pending_sample') %}
     <div class="table-responsive"><table class="table table-sm table-striped mb-0">
-      <thead><tr><th>crawl_key</th><th>score</th><th>type</th><th>id</th><th>page</th><th>url</th><th>referer</th><th>crawl_mode</th><th>parent</th></tr></thead>
+      <thead><tr><th>crawl_key</th><th>score</th><th>type</th><th>id</th><th>page</th><th>url</th><th>referer</th><th>crawl_mode</th><th>parent</th><th>actions</th></tr></thead>
       <tbody>{% for pending in queue.get('pending_sample') %}{% set item = pending.get('item') or {} %}
         <tr>
           <td class="text-break">{{ pending.get('crawl_key') }}</td>
@@ -71,6 +71,16 @@ <h1 class="mb-2"><i class="fas fa-list"></i> Forum crawl queue</h1>
           <td class="text-break">{{ item.get('referer') }}</td>
           <td>{{ item.get('crawl_mode') }}</td>
           <td class="text-break">{{ item.get('parent') }}</td>
+          <td>
+            <form method="post" action="{{ url_for('forums_explorer.forum_explorer_crawler_queue') }}" class="mb-0">
+              <input type="hidden" name="forum_id" value="{{ forum['id'] }}">
+              <input type="hidden" name="crawl_key" value="{{ pending.get('crawl_key') }}">
+              <input type="hidden" name="sample_size" value="{{ meta.get('sample_size') }}">
+              <button class="btn btn-sm btn-outline-danger" onclick="return confirm('Remove pending crawl item {{ pending.get('crawl_key') }} from the queue?')">
+                <i class="fas fa-times"></i> Remove
+              </button>
+            </form>
+          </td>
         </tr>
       {% endfor %}</tbody>
     </table></div>