feat: s3 attachment archival (#15)

closes #3
closes #12

Author: khakers

.gitignore

@@ -142,3 +142,6 @@ stack.yml
 .pdm-python
 
 compose.override.yml
+
+
+s3-data/*

CHANGELOG.md

@@ -9,17 +9,23 @@ however, insignificant breaking changes do not guarantee a major version bump, s
 # [UNRELEASED]
 
 ### Breaking
+
 - Completely rewritten blocklist system. Blocklisting now runs off its own mongoDB collection. This once again introduces backwards incompatible schema changes, so a manual migration is required. You may upgrade from both v4.0 and v4.1 using the `[p]migrate blocklist` command. This removes any need to perform the previous migration steps in v4.1.0, you may upgrade directly to this version. After running the command, blocklist functionality will return and legacy config based blocks will have been deleted. You should always back up your config before migration.
 - Remove internal logviewer plugin
 - Bump Python version to >= 3.12
 
 ### Deprecated
+
 - Legacy blocklist properties are deprecated and no longer function. They now log a warning when used and provide no functionality. They have been replaced with methods in blocklist.py 
 
 ### Added
+
 - Added `content_type` to attachments stored in the database.
 - `?log key <key>` to retrieve the log link and view a preview using a log key. ([PR #3196](https://github.com/modmail-dev/Modmail/pull/3196))
 - Add Forced plugins. Allows auto installing un-removable plugins via `FORCED_PLUGINS` environment variable contain a comma separate list of plugins. (GH#5)
+- S3 based attachment archival
+
+
 Commands:
 * `snooze`: Initiates a snooze action.
 * `snoozed`: Displays snoozed items.
@@ -54,8 +60,8 @@ Configuration Options:
     * Anonymous prompt support (`thread_creation_menu_anonymous_menu`).
 
 
-
 ### Changed
+
 - Changing a threads title or NSFW status immediately updates the status in the database.
 - Replace Pipenv with PDM
 - Bump discord.py version to 2.3.2
@@ -64,13 +70,15 @@ Configuration Options:
 - Upgrade dependencies for modern python
 
 ### Removed
+
 - Remove Discord.py dependency version check
 - Remove modmail telemetry
 - Remove lottie sticker support
 - Autoupdate system
   - Autoupdating was prone to serious issues and cannot be used within container images, the only supported distribution method of OpenModmail.
 
 ### Fixed
+
 - Persistent notes have been fixed after the previous discord.py update.
 - `is_image` now is true only if the image is actually an image.
 - Fix contact command reporting user was blocked when they weren't.
@@ -86,6 +94,7 @@ Configuration Options:
 
 
 ### Internal
+
 - Add `update_title` and `update_nsfw` methods to `ApiClient` to update thread title and nsfw status in the database.
 - `thread.set_title` now requires `channel_id` to be passed as keyword arguments.
 - New `thread.set_nsfw_status` method to set nsfw status of a thread.

README.md

@@ -49,6 +49,7 @@ logs for this fork.
   * Login via Discord to protect your logs (optional feature).
   * See past logs of a user with `?logs`.
   * Searchable by text queries using `?logs search`.
+  * **S3 Attachment Archival:** Archive message attachments to durable S3 storage to preserve access beyond Discord's URL expiration (~7 days). Supports AWS S3 and S3-compatible services.
 
 * **Robust implementation:**
   * Schedule tasks in human time, e.g. `?close in 2 hours silently`.

bot.py

@@ -1521,7 +1521,7 @@ async def on_message(self, message):
                     # Only log if not a command
                     perms = message.channel.permissions_for(message.author)
                     if perms.manage_messages or perms.administrator:
-                        await self.api.append_log(message, type_="internal")
+                        await self.api.append_log(message, thread_key=thread.key, type_="internal")
 
         await self.process_commands(message)
 

core/clients.py

@@ -1,5 +1,6 @@
 import secrets
 import sys
+import typing
 from json import JSONDecodeError
 from typing import Any, Dict, Optional, Union
 
@@ -12,6 +13,10 @@
 from pymongo.uri_parser import parse_uri
 
 from core.models import InvalidConfigError, getLogger
+from core.s3_archive import S3ArchiveConfig, S3AttachmentArchiver
+
+if typing.TYPE_CHECKING:
+    from bot import ModmailBot
 
 logger = getLogger(__name__)
 
@@ -299,7 +304,7 @@ class ApiClient:
         The bot's current running `ClientSession`.
     """
 
-    def __init__(self, bot, db):
+    def __init__(self, bot: "ModmailBot", db):
         self.bot = bot
         self.db = db
         self.session = bot.session
@@ -675,11 +680,64 @@ async def append_log(
         *,
         message_id: str | int = "",
         channel_id: str | int = "",
+        thread_key: str,
         type_: str = "thread_message",
     ) -> dict:
         channel_id = str(channel_id) or str(message.channel.id)
         message_id = str(message_id) or str(message.id)
 
+        # Build attachments list with base metadata
+        attachments = [
+            {
+                "id": a.id,
+                "filename": a.filename,
+                # In previous versions this was true for both videos and images
+                "is_image": a.content_type and a.content_type.startswith("image/"),
+                "size": a.size,
+                "url": a.url,
+                "content_type": a.content_type,
+                "description": a.description,
+                "type": "openmodmail_discord_v1",
+                "width": a.width,
+                "height": a.height,
+            }
+            for a in message.attachments
+        ]
+
+        # Archive attachments to S3 if configured
+        if message.attachments and self.bot.config.get("s3_enabled"):
+            try:
+                # We use self.bot.user later and should awlways be logged in at the point we are calling this code
+                assert self.bot.user
+                s3_config = S3ArchiveConfig(
+                    enabled=self.bot.config.get("s3_enabled") or False,
+                    bucket=self.bot.config.get("s3_bucket"),
+                    region=self.bot.config.get("s3_region") or "",
+                    access_key_id=self.bot.config.get("s3_access_key_id"),
+                    secret_access_key=self.bot.config.get("s3_secret_access_key"),
+                    endpoint=self.bot.config.get("s3_endpoint"),
+                    # We should always be logged in at this point
+                    key_prefix=self.bot.config.get("s3_key_prefix") or f"modmail/{self.bot.user.id}/attachments/",
+                )
+                archiver = S3AttachmentArchiver(s3_config)
+                
+                s3_metadata_list = await archiver.archive_attachments(
+                    message.attachments,
+                    thread_id=thread_key,
+                    message_id=message_id,
+                )
+                
+                # Merge S3 metadata into attachments
+                for i, att in enumerate(attachments):
+                    meta = s3_metadata_list[i] if i < len(s3_metadata_list) else None
+                    if meta is not None:
+                        att["s3"] = meta
+                        att["type"] = "openmodmail_s3_v1"
+            except Exception as e:
+                logger.error("S3 archival failed for message %s: %s", message_id, e)
+                raise e
+                # Continue without S3 archival; Discord URLs are still available
+
         data = {
             "timestamp": str(message.created_at),
             "message_id": message_id,
@@ -692,18 +750,7 @@ async def append_log(
             },
             "content": message.content,
             "type": type_,
-            "attachments": [
-                {
-                    "id": a.id,
-                    "filename": a.filename,
-                    # In previous versions this was true for both videos and images
-                    "is_image": a.content_type and a.content_type.startswith("image/"),
-                    "size": a.size,
-                    "url": a.url,
-                    "content_type": a.content_type,
-                }
-                for a in message.attachments
-            ],
+            "attachments": attachments,
             "messageReference": {
                 "message_id": message.reference.message_id,
                 "channel_id": message.reference.channel_id,

core/config.py

@@ -163,6 +163,12 @@ class ConfigManager:
         "thread_creation_menu_embed_large_image": False,
         "thread_creation_menu_embed_footer_icon_url": None,
         "thread_creation_menu_embed_color": str(discord.Color.green()),
+        # --- S3 ATTACHMENT ARCHIVAL ---
+        "s3_enabled": False,  # Enable archival of attachments to S3
+        "s3_bucket": None,  # S3 bucket name for attachment storage
+        "s3_region": None,  # AWS region (e.g., us-east-1, eu-west-1)
+        "s3_endpoint": None,  # Optional custom S3 endpoint (for MinIO, etc.)
+        "s3_key_prefix": "modmail/attachments/",  # Prefix for stored objects
     }
 
     private_keys = {
@@ -213,6 +219,9 @@ class ConfigManager:
         # github access token for private repositories
         "github_token": None,
         "disable_updates": False,
+        # S3 credentials
+        "s3_access_key_id": None,  # AWS access key ID for S3
+        "s3_secret_access_key": None,  # AWS secret access key for S3
         # Logging
         "log_level": "INFO",
         "stream_log_format": "plain",
@@ -279,6 +288,8 @@ class ConfigManager:
         "registry_plugins_only",
         # snooze
         "snooze_store_attachments",
+        # S3 attachment archival
+        "s3_enabled",
         # thread creation menu booleans
         "thread_creation_send_dm_embed",
         "thread_creation_menu_enabled",

core/config_help.json

@@ -1557,5 +1557,70 @@
     "notes": [
       "Color names map to the built-in palette (e.g., 'red', 'green', 'blurple')."
     ]
+  },
+  "s3_enabled": {
+    "default": "No",
+    "description": "Enable archival of message attachments to S3 storage. When enabled, attachments are uploaded to S3 after successful Discord transmission and logged metadata includes S3 references for resilient playback.",
+    "examples": [
+      "`{prefix}config set s3_enabled yes`",
+      "`{prefix}config set s3_enabled no`"
+    ],
+    "notes": [
+      "Requires `s3_bucket`, `s3_region`, and `s3_access_key_id`/`s3_secret_access_key` to be configured.",
+      "S3 archival applies to logged messages and thread replay scenarios. Live relay to Discord is unaffected.",
+      "See also: `s3_bucket`, `s3_region`, `s3_endpoint`, `s3_key_prefix`."
+    ]
+  },
+  "s3_bucket": {
+    "default": "None",
+    "description": "The name of the S3 bucket where message attachments will be archived.",
+    "examples": [
+      "`{prefix}config set s3_bucket my-modmail-attachments`",
+      "`{prefix}config set s3_bucket modmail-prod`"
+    ],
+    "notes": [
+      "This configuration can only be set through `.env` file or environment (config) variables for security.",
+      "Required when `s3_enabled` is set to yes.",
+      "See also: `s3_enabled`, `s3_region`, `s3_key_prefix`."
+    ]
+  },
+  "s3_region": {
+    "default": "None",
+    "description": "The region where the S3 bucket is located.",
+    "examples": [
+      "`{prefix}config set s3_region us-east-1`",
+      "`{prefix}config set s3_region eu-west-1`",
+      "`{prefix}config set s3_region ap-southeast-1`"
+    ],
+    "notes": [
+      "This configuration can only be set through `.env` file or environment (config) variables.",
+      "See also: `s3_enabled`, `s3_bucket`, `s3_endpoint`."
+    ]
+  },
+  "s3_endpoint": {
+    "default": "None (uses AWS S3)",
+    "description": "Optional custom S3-compatible endpoint URL for use with MinIO, DigitalOcean Spaces, or other S3-compatible services.",
+    "examples": [
+      "`{prefix}config set s3_endpoint https://minio.example.com:9000`",
+      "`{prefix}config set s3_endpoint https://nyc3.digitaloceanspaces.com`"
+    ],
+    "notes": [
+      "This configuration can only be set through `.env` file or environment (config) variables.",
+      "Leave unset to use AWS S3 directly.",
+      "See also: `s3_enabled`, `s3_region`."
+    ]
+  },
+  "s3_key_prefix": {
+    "default": "modmail/attachments/",
+    "description": "Prefix path within the S3 bucket where archived attachments will be stored.",
+    "examples": [
+      "`{prefix}config set s3_key_prefix modmail/attachments/`",
+      "`{prefix}config set s3_key_prefix archive/files/`"
+    ],
+    "notes": [
+      "Should include a trailing slash.",
+      "Useful for organizing archived attachments within a larger S3 bucket.",
+      "See also: `s3_enabled`, `s3_bucket`."
+    ]
   }
 }