MSC3911: Deduplication - save file with sha256 path (#155)

# Linked Media MSC3911 AP?: Deduplication- file path with sha256
[#3465](famedly/product-management#3465)
Fixes: #3465

Currently we are storing the file based on `media_id`. With msc3911
introduced, there are many copy actions and that creates duplicated
media. So now we try to save file with sha256 path and avoid
duplication.

There is a feature flag introduce with `use_sha256_paths`. But even with
feature flag is on, we still want to the existing media_id path logic
function well, and backward compatible. Feature flag also prevents the
existing tests failure.

Previously files were saved like this:
```
media/local_content/Ab/cD/...
media/local_thumbnails/Ab/cD/.../32-32-image-png-crop
media/remote_content/{server_name}/Ab/cD/...
```
But with `use_sha256_paths` enabled, media storage structure would be
like this:
```
media/eb/f4/...
thumbnails/eb/f4/.../32-32-image-png-crop
```

Author: itsoyou

rust/src/http_client.rs

@@ -137,7 +137,7 @@ fn get_runtime<'a>(reactor: &Bound<'a, PyAny>) -> PyResult<PyRef<'a, PyTokioRunt
 static DEFER: OnceCell<PyObject> = OnceCell::new();
 
 /// Access to the `twisted.internet.defer` module.
-fn defer(py: Python<'_>) -> PyResult<&Bound<PyAny>> {
+fn defer(py: Python<'_>) -> PyResult<&Bound<'_, PyAny>> {
     Ok(DEFER
         .get_or_try_init(|| py.import("twisted.internet.defer").map(Into::into))?
         .bind(py))

synapse/config/repository.py

@@ -194,6 +194,10 @@ def read_config(self, config: JsonDict, **kwargs: Any) -> None:
             config.get("media_store_path", "media_store")
         )
 
+        self.enable_local_media_storage_deduplication = config.get(
+            "enable_local_media_storage_deduplication", False
+        )
+
         backup_media_store_path = config.get("backup_media_store_path")
 
         synchronous_backup_media_store = config.get(

synapse/federation/transport/server/federation.py

@@ -79,7 +79,6 @@ def __init__(
     ):
         super().__init__(hs, authenticator, ratelimiter, server_name)
         self.handler = hs.get_federation_server()
-        self.enable_restricted_media = hs.config.experimental.msc3911.enabled
 
 
 class FederationSendServlet(BaseFederationServerServlet):

synapse/handlers/profile.py

@@ -80,10 +80,7 @@ def __init__(self, hs: "HomeServer"):
 
         self._third_party_rules = hs.get_module_api_callbacks().third_party_event_rules
 
-        self.enable_restricted_media = hs.config.experimental.msc3911.enabled
-        self.block_unrestricted_media = (
-            hs.config.experimental.msc3911.block_unrestricted_media_upload
-        )
+        self.msc3911_config = hs.config.experimental.msc3911
 
     async def get_profile(self, user_id: str, ignore_backoff: bool = True) -> JsonDict:
         """
@@ -334,7 +331,7 @@ async def validate_avatar_url(self, avatar_url: str, requester: Requester) -> No
             # event, it will either be copied/passed along/dropped depending on the
             # above circumstances
             if not media_info:
-                if self.block_unrestricted_media:
+                if self.msc3911_config.block_unrestricted_media_upload:
                     # The user should have done a COPY on this media previous to this
                     # attempt to set
                     raise SynapseError(
@@ -345,7 +342,10 @@ async def validate_avatar_url(self, avatar_url: str, requester: Requester) -> No
                 # For backwards compatible behavior, treat the media as unrestricted
                 return
 
-        if self.block_unrestricted_media and not media_info.restricted:
+        if (
+            self.msc3911_config.block_unrestricted_media_upload
+            and not media_info.restricted
+        ):
             raise SynapseError(
                 HTTPStatus.BAD_REQUEST,
                 f"The media attachment request is invalid as the media '{mxc_uri.media_id}' is not restricted",
@@ -424,7 +424,7 @@ async def set_avatar_url(
             )
 
         # msc3911: Update the media restrictions to include the profile user ID
-        if self.enable_restricted_media and avatar_url_to_set:
+        if self.msc3911_config.enabled and avatar_url_to_set:
             await self.validate_avatar_url(avatar_url_to_set, requester)
             await self.hs.get_datastores().main.set_media_restricted_to_user_profile(
                 self.hs.config.server.server_name,

synapse/handlers/room_member.py

@@ -109,10 +109,7 @@ def __init__(self, hs: "HomeServer"):
         self.account_data_handler = hs.get_account_data_handler()
         self.event_auth_handler = hs.get_event_auth_handler()
         self._worker_lock_handler = hs.get_worker_locks_handler()
-        self.enable_restricted_media = hs.config.experimental.msc3911.enabled
-        self.allow_legacy_media = (
-            not hs.config.experimental.msc3911.block_unrestricted_media_upload
-        )
+        self.msc3911_config = hs.config.experimental.msc3911
 
         self._membership_types_to_include_profile_data_in = {
             Membership.JOIN,
@@ -865,7 +862,7 @@ async def update_membership_locked(
             except Exception as e:
                 logger.info("Failed to get profile information for %r: %s", target, e)
 
-            if self.enable_restricted_media and not media_info_for_attachment:
+            if self.msc3911_config.enabled and not media_info_for_attachment:
                 # This code path should only be taken for memberships updating an
                 # avatar url
                 avatar_url = content.get(EventContentFields.MEMBERSHIP_AVATAR_URL)
@@ -898,7 +895,7 @@ async def update_membership_locked(
                         # hope that it is not restricted when it finally shows up.
                         # Guard against other potentials escaping by raising if it
                         # should occur when unrestricted media begins to be disallowed.
-                        if self.allow_legacy_media:
+                        if not self.msc3911_config.block_unrestricted_media_upload:
                             logger.debug(
                                 "Ignoring media copy request; the media is unknown and "
                                 "will not be treated as restricted"

synapse/media/_base.py

@@ -540,6 +540,8 @@ class FileInfo:
     url_cache: bool = False
     # Whether the file is a thumbnail or not.
     thumbnail: Optional[ThumbnailInfo] = None
+    # The sha256 of the media. When enable_local_media_storage_deduplication is True, this will be generating the filepath.
+    sha256: Optional[str] = None
 
     # The below properties exist to maintain compatibility with third-party modules.
     @property

synapse/media/filepath.py

@@ -184,6 +184,16 @@ def local_media_filepath_rel(self, media_id: str) -> str:
 
     local_media_filepath = _wrap_in_base_path(local_media_filepath_rel)
 
+    @_wrap_with_jail_check(relative=True)
+    def filepath_sha_rel(self, sha256: str) -> str:
+        return os.path.join(
+            _validate_path_component(sha256[0:2]),
+            _validate_path_component(sha256[2:4]),
+            _validate_path_component(sha256[4:]),
+        )
+
+    filepath_sha = _wrap_in_base_path(filepath_sha_rel)
+
     @_wrap_with_jail_check(relative=True)
     def local_media_thumbnail_rel(
         self, media_id: str, width: int, height: int, content_type: str, method: str
@@ -200,6 +210,22 @@ def local_media_thumbnail_rel(
 
     local_media_thumbnail = _wrap_in_base_path(local_media_thumbnail_rel)
 
+    @_wrap_with_jail_check(relative=True)
+    def thumbnail_sha_rel(
+        self, sha256: str, width: int, height: int, content_type: str, method: str
+    ) -> str:
+        top_level_type, sub_type = content_type.split("/")
+        file_name = "%i-%i-%s-%s-%s" % (width, height, top_level_type, sub_type, method)
+        return os.path.join(
+            "thumbnails",
+            _validate_path_component(sha256[0:2]),
+            _validate_path_component(sha256[2:4]),
+            _validate_path_component(sha256[4:]),
+            _validate_path_component(file_name),
+        )
+
+    thumbnail_sha = _wrap_in_base_path(thumbnail_sha_rel)
+
     @_wrap_with_jail_check(relative=False)
     def local_media_thumbnail_dir(self, media_id: str) -> str:
         """
@@ -218,6 +244,24 @@ def local_media_thumbnail_dir(self, media_id: str) -> str:
             _validate_path_component(media_id[4:]),
         )
 
+    @_wrap_with_jail_check(relative=False)
+    def thumbnail_sha_dir(self, sha256: str) -> str:
+        """
+        Retrieve the local store path of thumbnails of a given media_id
+
+        Args:
+            sha256: The sha256 to query.
+        Returns:
+            Path of local_thumbnails from sha256
+        """
+        return os.path.join(
+            self.base_path,
+            "thumbnails",
+            _validate_path_component(sha256[0:2]),
+            _validate_path_component(sha256[2:4]),
+            _validate_path_component(sha256[4:]),
+        )
+
     @_wrap_with_jail_check(relative=True)
     def remote_media_filepath_rel(self, server_name: str, file_id: str) -> str:
         return os.path.join(