feat(forum): show topics during backup + disk-usage storage stat (#200)

docs/CHANGELOG.md

@@ -4,6 +4,23 @@ All notable changes to this project are documented here.
 
 For upgrade instructions, see [Upgrading](#upgrading) at the bottom.
 
+## [7.17.0] - 2026-06-25
+
+### Added
+- **Forum topics appear immediately** — Topic lists for forum/Topics-mode groups are now fetched up front, at the start of each chat's backup, instead of only at the end of a full run. Large, media-heavy forums no longer show "0 topics" while their media is still downloading; topics show up within seconds and message counts fill in as the backup progresses. ([#200](https://github.com/GeiserX/Telegram-Archive/issues/200))
+- **"Backup in progress" indicator** — The Backup Statistics panel shows a live indicator while a backup run is active, so it's clear when the figures are still being collected rather than final.
+- **"View all messages" for forums without topics** — When a forum group has no topics recorded yet, the viewer now offers a direct link to browse the chat's messages instead of a dead-end "No topics found".
+
+### Changed
+- **Storage statistic reflects actual disk usage** — The "Storage" figure is now measured from on-disk files (`du`-style, counting deduplicated `_shared` blobs once) rather than summing recorded media sizes, so it matches real disk consumption. Sizes are labeled with correct binary units (GiB/MiB/TiB).
+- Forum-topic fetching now paginates beyond 100 topics and retries on FloodWait, so large forums are captured fully.
+
+### Fixed
+- Forum/Topics-mode groups showing "0 topics" in the web viewer during long, media-heavy backups. ([#200](https://github.com/GeiserX/Telegram-Archive/issues/200))
+
+### Credits
+- Thanks to [@1235789gzy1](https://github.com/1235789gzy1) for reporting the forum-topic display and storage-statistic issues in [#200](https://github.com/GeiserX/Telegram-Archive/issues/200).
+
 ## [7.16.0] - 2026-06-25
 
 ### Added

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "telegram-archive"
-version = "7.16.0"
+version = "7.17.0"
 description = "Automated Telegram backup with Docker. Performs incremental backups of messages and media on a configurable schedule."
 readme = "README.md"
 requires-python = ">=3.14"

src/__init__.py

@@ -2,4 +2,4 @@
 Telegram Backup Automation - Main Package
 """
 
-__version__ = "7.16.0"
+__version__ = "7.17.0"

src/db/adapter.py

@@ -21,7 +21,7 @@
 from sqlalchemy.dialects.postgresql import insert as pg_insert
 from sqlalchemy.dialects.sqlite import insert as sqlite_insert
 
-from ..message_utils import utcnow_naive
+from ..message_utils import compute_directory_size, utcnow_naive
 from .base import DatabaseManager
 from .models import (
     AppSettings,
@@ -1181,8 +1181,18 @@ async def get_cached_statistics(self) -> dict[str, Any]:
 
         return result
 
-    async def calculate_and_store_statistics(self) -> dict[str, Any]:
-        """Calculate statistics and store in metadata (expensive, run daily)."""
+    async def calculate_and_store_statistics(self, storage_path: str | None = None) -> dict[str, Any]:
+        """Calculate statistics and store in metadata (expensive, run daily).
+
+        When ``storage_path`` is given, total media size reflects actual on-disk
+        usage (``du`` semantics) via ``compute_directory_size`` so the figure
+        tracks real disk consumption. The filesystem walk is a blocking scan, so
+        it runs off the event loop (``asyncio.to_thread``) and outside the DB
+        session. If the path is missing/unmounted (``du`` is 0 while media rows
+        exist), or no path is given, it falls back to the DB snapshot
+        ``SUM(media.file_size WHERE downloaded=1)``.
+        """
+        import asyncio
         import json
         from datetime import datetime
 
@@ -1201,9 +1211,11 @@ async def calculate_and_store_statistics(self) -> dict[str, Any]:
             media_count = await session.execute(select(func.count(Media.id)).where(Media.downloaded == 1))
             media_count = media_count.scalar() or 0
 
-            # Total media size
-            total_size = await session.execute(select(func.sum(Media.file_size)).where(Media.downloaded == 1))
-            total_size = total_size.scalar() or 0
+            # DB snapshot of downloaded media sizes — the fallback when on-disk
+            # usage is unavailable (e.g. the backup volume is not mounted yet).
+            db_total_size = (
+                await session.execute(select(func.sum(Media.file_size)).where(Media.downloaded == 1))
+            ).scalar() or 0
 
             # Per-chat statistics
             chat_stats_query = select(Message.chat_id, func.count(Message.id).label("message_count")).group_by(
@@ -1212,15 +1224,27 @@ async def calculate_and_store_statistics(self) -> dict[str, Any]:
             chat_stats_result = await session.execute(chat_stats_query)
             per_chat_stats = {row.chat_id: row.message_count for row in chat_stats_result}
 
-            stats = {
-                "chats": int(chat_count),
-                "messages": int(msg_count),
-                "media_files": int(media_count),
-                "total_size_mb": float(round(total_size / (1024 * 1024), 2)),
-                "per_chat_message_counts": {int(k): int(v) for k, v in per_chat_stats.items()},
-            }
+        # Total media size: prefer actual on-disk usage. Run the blocking walk off
+        # the event loop and after the session is closed so it never stalls other
+        # requests or pins a DB connection.
+        if storage_path is not None:
+            total_size = await asyncio.to_thread(compute_directory_size, storage_path)
+            if total_size == 0 and media_count > 0:
+                # Path missing/unmounted: don't cache a spurious 0 over the last good value.
+                logger.warning("On-disk storage size is 0 while media exists; using DB snapshot for storage stat")
+                total_size = db_total_size
+        else:
+            total_size = db_total_size
+
+        stats = {
+            "chats": int(chat_count),
+            "messages": int(msg_count),
+            "media_files": int(media_count),
+            "total_size_mb": float(round(total_size / (1024 * 1024), 2)),
+            "per_chat_message_counts": {int(k): int(v) for k, v in per_chat_stats.items()},
+        }
 
-            logger.info(f"Statistics calculated: {chat_count} chats, {msg_count} messages, {media_count} media files")
+        logger.info(f"Statistics calculated: {chat_count} chats, {msg_count} messages, {media_count} media files")
 
         # Store in metadata
         await self.set_metadata("cached_stats", json.dumps(stats))

src/message_utils.py

@@ -6,6 +6,7 @@
 import logging
 import os
 import re
+import stat
 from datetime import UTC, datetime
 
 logger = logging.getLogger(__name__)
@@ -16,6 +17,30 @@ def utcnow_naive() -> datetime:
     return datetime.now(UTC).replace(tzinfo=None)
 
 
+def compute_directory_size(path: str) -> int:
+    """Return total on-disk size (bytes) of regular files under `path`.
+
+    Walks the tree without following symlinks, summing each regular file's
+    lstat size. Symlinks (used by the dedup _shared store) are not followed,
+    so shared blobs are counted exactly once. Missing path or per-entry errors
+    are ignored (best-effort, never raises)."""
+    if not path or not os.path.isdir(path):
+        return 0
+
+    total = 0
+    for root, _dirs, files in os.walk(path, followlinks=False):
+        for name in files:
+            full = os.path.join(root, name)
+            try:
+                st = os.lstat(full)
+            except OSError:
+                continue
+            if stat.S_ISLNK(st.st_mode):
+                continue
+            total += st.st_size
+    return total
+
+
 def sanitize_media_filename(name: str) -> str:
     """Strip path components from an attacker-controlled media filename.