chg: [user-account] add forum support. show user-account threads amd posts

Terrtia · Terrtia · commit e2f19bba8b46 · 2026-07-09T14:57:53.000+02:00
diff --git a/bin/importer/feeders/Forum_Extractor.py b/bin/importer/feeders/Forum_Extractor.py
@@ -336,6 +336,7 @@ def _upsert_post(self, post_data, parent_thread):
                 if image:
                     user_account.set_icon(image.get_global_id())
                     self.objs_to_process.add(image)
+            user_account.add_correlation(parent_thread.type, parent_thread.subtype, parent_thread.id)
         return post
 
     def _create_image_from_har_url(self, url, date, obj):
diff --git a/bin/lib/ail_core.py b/bin/lib/ail_core.py
@@ -114,7 +114,8 @@ def get_object_all_subtypes(obj_type):  # TODO Dynamic subtype
         chat_protocols.add('telegram')
         return sorted(chat_protocols)
     if obj_type == 'user-account':
-        return r_object.smembers(f'all_chat:subtypes')
+        # return r_object.smembers(f'all_chat:subtypes')
+        return r_object.smembers(f'all_user-account:subtypes')
     return []
 
 def get_default_correlation_objects():
diff --git a/bin/lib/correlations_engine.py b/bin/lib/correlations_engine.py
@@ -68,7 +68,7 @@
     "favicon": ["domain", "item"],  # TODO Decoded
     "forum": ["post", "subforum", "user-account"],
     "subforum": ["forum", "subforum", "forum-thread"],
-    "forum-thread": ["subforum", "post"],
+    "forum-thread": ["subforum", "post", "user-account"],
     # TODO Extend to detection of text -> same as message
     "post": ["forum", "forum-thread", "image", "user-account", *COMMON_TEXT_OBJECT_CORRELATIONS],
     "file-name": ["chat", "item", "message", "pdf"],
@@ -86,7 +86,7 @@
     "screenshot": ["barcode", "domain", "item", "qrcode"],
     "ssh-key": ["domain", "ip"],
     "title": ["domain", "item"],
-    "user-account": ["chat", "chat-subchannel", "chat-thread", "forum", "image", "message", "ocr", "post", "username"],
+    "user-account": ["chat", "chat-subchannel", "chat-thread", "forum", "forum-thread", "image", "message", "ocr", "post", "username"],
     "username": ["domain", "item", "message", "user-account"],
 }
 
diff --git a/bin/lib/forums_viewer.py b/bin/lib/forums_viewer.py
@@ -30,7 +30,7 @@
 # config_loader = ConfigLoader()
 # config_loader = None
 
-_FORUM_OPTIONS = {'forum_type', 'info', 'name', 'url', 'banner', 'nb_subforums', 'nb_orphan_subforums'}
+_FORUM_OPTIONS = {'banner', 'forum_type', 'info', 'name', 'url', 'nb_subforums', 'nb_orphan_subforums', 'svg_icon'}
 _SUBFORUM_OPTIONS = {'info', 'url', 'nb_subforums', 'nb_threads'}
 _THREAD_OPTIONS = {'name', 'info', 'url', 'flags', 'nb_posts'}
 _POST_OPTIONS = {'content', 'images', 'language', 'link', 'reactions', 'state', 'timestamp', 'translation', 'user-account'}
@@ -365,6 +365,159 @@ def _subforum_threads_meta(subforum):
     )
 
 
+def _get_user_account_posts_sorted(user_account):
+    posts = []
+    for pid in user_account.get_posts():
+        _, post_id = pid.split(':', 1)
+        post = Posts.Post(post_id)
+        timestamp = post.get_timestamp()
+        if timestamp and post.exists():
+            posts.append((post, float(timestamp)))
+    return sorted(posts, key=lambda post_item: post_item[1], reverse=True)
+
+
+def _paginate_items(items, page=1, nb=50):
+    try:
+        page = int(page)
+    except (TypeError, ValueError):
+        page = 1
+    try:
+        nb = int(nb)
+    except (TypeError, ValueError):
+        nb = 50
+    if page < 1:
+        page = 1
+    if nb < 1:
+        nb = 50
+    total = len(items)
+    nb_pages = int(total / nb)
+    if total and total % nb:
+        nb_pages += 1
+    if not nb_pages:
+        nb_pages = 1
+    if page > nb_pages:
+        page = nb_pages
+    start = (page - 1) * nb
+    end = min(start + nb, total)
+    return items[start:end], {'nb': nb, 'page': page, 'nb_pages': nb_pages, 'total': total, 'nb_first': start + 1 if total else 0, 'nb_last': end}
+
+
+def _posts_to_date_dict(post_items, translation_target=None):
+    posts_by_date = {}
+    for post, timestamp in post_items:
+        meta = post.get_meta(_POST_OPTIONS, translation_target=translation_target, flask_context=True)
+        date_day = Date.get_utc_date_from_timestamp(timestamp, separator='/')
+        posts_by_date.setdefault(date_day, []).append(meta)
+    return posts_by_date
+
+
+def get_user_account_threads_meta(user_account):
+    threads = []
+    for thread_str in user_account.get_forum_threads():
+        thread_subtype, thread_id = thread_str.split(':', 1)
+        thread = ForumThreads.ForumThread(thread_id, thread_subtype)
+        thread_meta = _thread_meta(thread) if thread.exists() else {'type': 'forum-thread', 'subtype': thread_subtype, 'id': thread_id}
+        user_thread_posts = user_account.get_correlation_iter_obj(thread, 'post')
+        thread_meta['nb_posts'] = len(user_thread_posts)
+        first_post_timestamp = None
+        last_post_timestamp = None
+        for post_id in user_thread_posts:
+            timestamp = Posts.Post(post_id).get_timestamp()
+            if not timestamp:
+                continue
+            timestamp = float(timestamp)
+            first_post_timestamp = timestamp if first_post_timestamp is None else min(first_post_timestamp, timestamp)
+            last_post_timestamp = timestamp if last_post_timestamp is None else max(last_post_timestamp, timestamp)
+        thread_meta['first_post_timestamp'] = first_post_timestamp or 0
+        thread_meta['last_post_timestamp'] = last_post_timestamp or 0
+        if first_post_timestamp:
+            thread_meta['first_post_date'] = Date.get_utc_datetime_from_timestamp(first_post_timestamp)
+        else:
+            thread_meta['first_post_date'] = None
+        if last_post_timestamp:
+            thread_meta['last_post_date'] = Date.get_utc_datetime_from_timestamp(last_post_timestamp)
+        else:
+            thread_meta['last_post_date'] = None
+        threads.append(thread_meta)
+    return sorted(threads, key=lambda thread: thread['last_post_timestamp'], reverse=True)
+
+
+def get_user_account_nb_all_week_posts(user_account):
+    week = {day: {hour: 0 for hour in range(24)} for day in ['Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun']}
+    for post_gid in user_account.get_posts():
+        _, post_id = post_gid.split(':', 1)
+        timestamp = Posts.Post(post_id).get_timestamp()
+        if not timestamp:
+            continue
+        weekday, hour = Date.get_utc_weekday_hour_from_timestamp(timestamp)
+        week[weekday][hour] += 1
+    stats = []
+    for nb_day, day in enumerate(week):
+        for hour in week[day]:
+            stats.append({'date': day, 'day': nb_day, 'hour': hour, 'count': week[day][hour]})
+    return stats
+
+
+def get_user_account_nb_year_posts(user_account, year):
+    nb_year = {}
+    nb_max = 0
+    start = Date.convert_str_date_to_epoch(f'{year}0101')
+    end = Date.convert_str_date_to_epoch_end(f'{year}1231')
+    for post_gid in user_account.get_posts():
+        _, post_id = post_gid.split(':', 1)
+        timestamp = Posts.Post(post_id).get_timestamp()
+        if not timestamp:
+            continue
+        timestamp = int(float(timestamp))
+        if start <= timestamp <= end:
+            date = Date.get_utc_date_from_timestamp(timestamp, separator='-')
+            nb_year[date] = nb_year.get(date, 0) + 1
+            nb_max = max(nb_max, nb_year[date])
+    return nb_max, nb_year
+
+
+def api_get_user_account(user_id, forum_id, translation_target=None):
+    user_account = UsersAccount.UserAccount(user_id, forum_id)
+    if not user_account.exists():
+        return {"status": "error", "reason": "Unknown user-account"}, 404
+    meta = user_account.get_meta({'forums', 'icon', 'info', 'translation', 'username', 'usernames', 'username_meta', 'years', 'nb_posts'}, translation_target=translation_target)
+    forum = Forums.Forum(forum_id)
+    meta['forum'] = forum.get_meta(_FORUM_OPTIONS, flask_context=True) if forum.exists() else None
+    meta['threads'] = get_user_account_threads_meta(user_account)
+    return meta, 200
+
+
+def api_get_user_account_posts(user_id, forum_id, page=1, nb=50, translation_target=None):
+    user_account = UsersAccount.UserAccount(user_id, forum_id)
+    if not user_account.exists():
+        return {"status": "error", "reason": "Unknown user-account"}, 404
+    post_items, pagination = _paginate_items(_get_user_account_posts_sorted(user_account), page=page, nb=nb)
+    meta = user_account.get_meta({'icon', 'info', 'translation', 'username', 'usernames', 'username_meta'}, translation_target=translation_target)
+    forum = Forums.Forum(forum_id)
+    meta['forum'] = forum.get_meta(_FORUM_OPTIONS, flask_context=True) if forum.exists() else None
+    return {'user-account': meta, 'posts': _posts_to_date_dict(post_items, translation_target=translation_target), 'pagination': pagination}, 200
+
+
+def api_get_user_account_nb_all_week_posts(user_id, forum_id):
+    user_account = UsersAccount.UserAccount(user_id, forum_id)
+    if not user_account.exists():
+        return {"status": "error", "reason": "Unknown user-account"}, 404
+    return get_user_account_nb_all_week_posts(user_account), 200
+
+
+def api_get_user_account_nb_year_posts(user_id, forum_id, year):
+    user_account = UsersAccount.UserAccount(user_id, forum_id)
+    if not user_account.exists():
+        return {"status": "error", "reason": "Unknown user-account"}, 404
+    if not year or year == 'null':
+        years = user_account.get_years()
+        year = years[-1] if years else int(Date.get_current_year())
+    else:
+        year = int(year)
+    nb_max, nb = get_user_account_nb_year_posts(user_account, year)
+    return {'max': nb_max, 'year': year, 'nb': [[date, nb[date]] for date in nb]}, 200
+
+
 def get_forums():
     """Return metadata for all imported Forum objects."""
     forums = []
diff --git a/bin/lib/objects/Posts.py b/bin/lib/objects/Posts.py
@@ -68,7 +68,7 @@ def get_thread(self):
     def get_thread_sub_id(self):
         thread = self.get_correlation('forum-thread')
         if thread.get('forum-thread'):
-            return thread["forum-thread"].pop()
+            return thread["forum-thread"].pop().split(':', 1)[1]
         return None
 
     def get_date(self):
diff --git a/bin/lib/objects/UsersAccount.py b/bin/lib/objects/UsersAccount.py
@@ -145,6 +145,18 @@ def get_chats(self):
     def get_nb_chats(self):
         return self.get_nb_correlation('chat')
 
+    def get_forum(self):
+        return self.get_correlation('forum').get('forum', set()).pop()
+
+    def get_posts(self):
+        return self.get_correlation('post').get('post', set())
+
+    def get_nb_posts(self):
+        return self.get_nb_correlation('post')
+
+    def get_forum_threads(self):
+        return self.get_correlation('forum-thread').get('forum-thread', set())
+
     def get_chat_subchannels(self):
         chats = self.get_correlation('chat-subchannel')['chat-subchannel']
         return chats
@@ -228,6 +240,10 @@ def get_meta(self, options=set(), translation_target=None, flask_context=False):
             meta['chats'] = self.get_chats()
         if 'nb_chats' in options:
             meta['nb_chats'] = self.get_nb_chats()
+        if 'forums' in options:
+            meta['forums'] = self.get_forum()
+        if 'nb_posts' in options:
+            meta['nb_posts'] = self.get_nb_posts()
         if 'subchannels' in options:
             meta['subchannels'] = self.get_chat_subchannels()
         if 'threads' in options:
diff --git a/bin/packages/Date.py b/bin/packages/Date.py
@@ -118,6 +118,18 @@ def get_utc_datetime_from_timestamp(timestamp):
     timestamp = datetime.datetime.fromtimestamp(float(timestamp), datetime.timezone.utc)
     return timestamp.strftime('%Y-%m-%d %H:%M:%S')
 
+def get_utc_date_from_timestamp(timestamp, separator=''):
+    timestamp = datetime.datetime.fromtimestamp(float(timestamp), datetime.timezone.utc)
+    if separator:
+        return timestamp.strftime(f'%Y{separator}%m{separator}%d')
+    return timestamp.strftime('%Y%m%d')
+
+
+def get_utc_weekday_hour_from_timestamp(timestamp):
+    timestamp = datetime.datetime.fromtimestamp(float(timestamp), datetime.timezone.utc)
+    return timestamp.strftime('%a'), timestamp.hour
+
+
 def get_date_from_timestamp(timestamp):
     return datetime.datetime.fromtimestamp(timestamp).strftime('%Y%m%d')
 
diff --git a/var/www/blueprints/chats_explorer.py b/var/www/blueprints/chats_explorer.py
@@ -21,6 +21,7 @@
 ##################################
 from lib import ail_core
 from lib import chats_viewer
+from lib import forums_viewer
 from lib import Language
 from lib import Tag
 from lib import module_extractor
@@ -493,7 +494,12 @@ def objects_user_account():
     target = request.args.get('target')
     if target == "Don't Translate":
         target = None
-    user_account = chats_viewer.api_get_user_account(user_id, instance_uuid, translation_target=target)
+    obj = ail_objects.get_object('user-account', instance_uuid, user_id)
+    is_forum_account = obj.get_forum()
+    if is_forum_account:
+        user_account = forums_viewer.api_get_user_account(user_id, instance_uuid, translation_target=target)
+    else:
+        user_account = chats_viewer.api_get_user_account(user_id, instance_uuid, translation_target=target)
     # print()
     # print(user_account[0]['usernames'])
     # print()
@@ -505,10 +511,33 @@ def objects_user_account():
         translation_languages = Language.get_translation_languages()
         languages_stats = chats_viewer.api_get_languages_stats('user-account', instance_uuid, user_id)
         lang_endpoint = url_for('chats_explorer.objects_user_account_lang') + f'?subtype={instance_uuid}&id={user_id}&lang='
+        account_context = 'forum' if is_forum_account else 'chat'
         return render_template('user_account.html', meta=user_account, bootstrap_label=bootstrap_label,
                                ail_tags=Tag.get_modal_add_tags(user_account['id'], user_account['type'], user_account['subtype']),
                                languages_stats=languages_stats, lang_endpoint=lang_endpoint, all_languages=languages,
-                               translation_languages=translation_languages, translation_target=target)
+                               translation_languages=translation_languages, translation_target=target,
+                               account_context=account_context)
+
+
+@chats_explorer.route("/objects/user-account/posts", methods=['GET'])
+@login_required
+@login_read_only
+def objects_user_account_posts():
+    instance_uuid = request.args.get('subtype')
+    user_id = request.args.get('id')
+    page = request.args.get('page')
+    nb = request.args.get('nb')
+    target = request.args.get('target')
+    if target == "Don't Translate":
+        target = None
+    meta = forums_viewer.api_get_user_account_posts(user_id, instance_uuid, page=page, nb=nb, translation_target=target)
+    if meta[1] != 200:
+        return create_json_response(meta[0], meta[1])
+    languages = Language.get_all_languages()
+    translation_languages = Language.get_translation_languages()
+    return render_template('user_account_forum_posts.html', meta=meta[0], bootstrap_label=bootstrap_label,
+                           ail_tags=Tag.get_modal_add_tags(meta[0]['user-account']['id'], meta[0]['user-account']['type'], meta[0]['user-account']['subtype']),
+                           all_languages=languages, translation_languages=translation_languages, translation_target=target)
 
 @chats_explorer.route("/objects/user-account_usernames_timeline_json", methods=['GET']) # TODO API
 @login_required
@@ -591,7 +620,11 @@ def objects_user_account_lang():
 def user_account_messages_stats_week_all():
     instance_uuid = request.args.get('subtype')
     user_id = request.args.get('id')
-    week = chats_viewer.api_get_user_account_nb_all_week_messages(user_id, instance_uuid)
+    obj = ail_objects.get_object('user-account', instance_uuid, user_id)
+    if obj.get_forum():
+        week = forums_viewer.api_get_user_account_nb_all_week_posts(user_id, instance_uuid)
+    else:
+        week = chats_viewer.api_get_user_account_nb_all_week_messages(user_id, instance_uuid)
     if week[1] != 200:
         return create_json_response(week[0], week[1])
     else:
@@ -604,7 +637,11 @@ def user_account_messages_stats_year():
     instance_uuid = request.args.get('subtype')
     user_id = request.args.get('id')
     year = request.args.get('year')
-    stats = chats_viewer.api_get_user_account_nb_year_messages(user_id, instance_uuid, year)
+    obj = ail_objects.get_object('user-account', instance_uuid, user_id)
+    if obj.get_forum():
+        stats = forums_viewer.api_get_user_account_nb_year_posts(user_id, instance_uuid, year)
+    else:
+        stats = chats_viewer.api_get_user_account_nb_year_messages(user_id, instance_uuid, year)
     if stats[1] != 200:
         return create_json_response(stats[0], stats[1])
     else:
diff --git a/var/www/templates/chats_explorer/card_user_account.html b/var/www/templates/chats_explorer/card_user_account.html
@@ -23,7 +23,7 @@ <h4 class="text-secondary">{% if meta['username'] %}{{ meta["username"]["id"] }}
                         <th>ID</th>
                         <th>First Seen</th>
                         <th>Last Seen</th>
-                        <th>NB Chats</th>
+                        <th>{% if account_context == 'forum' %}NB Posts{% else %}NB Chats{% endif %}</th>
                     </tr>
                     </thead>
                     <tbody style="font-size: 15px;">
@@ -50,7 +50,7 @@ <h4 class="text-secondary">{% if meta['username'] %}{{ meta["username"]["id"] }}
                                 {{ meta['last_seen'][0:4] }}-{{ meta['last_seen'][4:6] }}-{{ meta['last_seen'][6:8] }}
                             {% endif %}
                         </td>
-                        <td>{{ meta['chats'] | length }}</td>
+                        <td>{% if account_context == 'forum' %}{{ meta.get('nb_posts', 0) }}{% else %}{{ meta.get('chats', []) | length }}{% endif %}</td>
                     </tr>
                     </tbody>
                 </table>
diff --git a/var/www/templates/chats_explorer/user_account.html b/var/www/templates/chats_explorer/user_account.html
diff --git a/var/www/templates/chats_explorer/user_account_forum_posts.html b/var/www/templates/chats_explorer/user_account_forum_posts.html
