Support for whitelisting extensions/mimetype

Author: harminius

deployment/community/.env.template

@@ -109,6 +109,10 @@ LOCAL_PROJECTS=/data
 
 #BLACKLIST='.mergin/, .DS_Store, .directory'  # cast=Csv()
 
+# extra file types to permit beyond the default block-list (e.g. scripts)
+#UPLOAD_EXTENSIONS_WHITELIST=''  # cast=Csv()
+#UPLOAD_MIME_TYPES_WHITELIST=''  # cast=Csv()
+
 #FILE_EXPIRATION=48 * 3600  # for clean up of old files where diffs were applied, in seconds
 
 #LOCKFILE_EXPIRATION=300  # in seconds

server/mergin/sync/config.py

@@ -82,5 +82,13 @@ class Configuration(object):
     )
     # files that should be ignored during extension and MIME type checks
     UPLOAD_FILES_WHITELIST = config("UPLOAD_FILES_WHITELIST", default="", cast=Csv())
+    # extra extensions to permit beyond the default block-list
+    UPLOAD_EXTENSIONS_WHITELIST = config(
+        "UPLOAD_EXTENSIONS_WHITELIST", default="", cast=Csv()
+    )
+    # extra MIME types to permit beyond the default block-list
+    UPLOAD_MIME_TYPES_WHITELIST = config(
+        "UPLOAD_MIME_TYPES_WHITELIST", default="", cast=Csv()
+    )
     # max batch size for fetch projects in batch endpoint
     MAX_BATCH_SIZE = config("MAX_BATCH_SIZE", default=100, cast=int)

server/mergin/sync/files.py

@@ -224,7 +224,7 @@ def validate(self, data, **kwargs):
 
             if not is_supported_extension(file_path):
                 raise ValidationError(
-                    f"Unsupported file type detected: '{file_path}'. "
+                    f"stop Unsupported file type detected: '{file_path}'. "
                     f"Please remove the file or try compressing it into a ZIP file before uploading.",
                 )
         # new checks must restrict only new files not to block existing projects

server/mergin/sync/utils.py

@@ -315,6 +315,8 @@ def is_supported_extension(filepath) -> bool:
     if check_skip_validation(filepath):
         return True
     ext = os.path.splitext(filepath)[1].lower()
+    if ext in {e.lower() for e in Configuration.UPLOAD_EXTENSIONS_WHITELIST}:
+        return True
     return ext and ext not in FORBIDDEN_EXTENSIONS
 
 
@@ -493,6 +495,8 @@ def is_supported_type(filepath) -> bool:
     if check_skip_validation(filepath):
         return True
     mime_type = get_mimetype(filepath)
+    if mime_type in Configuration.UPLOAD_MIME_TYPES_WHITELIST:
+        return True
     return mime_type.startswith("image/") or mime_type not in FORBIDDEN_MIME_TYPES
 
 

server/mergin/tests/test_utils.py

@@ -402,3 +402,32 @@ def test_mime_type_validation_skip():
 
         # Should be forbidden
         assert not is_supported_type("other.js")
+
+
+def test_allowed_extensions_override():
+    """Extensions in UPLOAD_EXTENSIONS_WHITELIST are accepted even though they are in FORBIDDEN_EXTENSIONS."""
+    with patch(
+        "mergin.sync.utils.Configuration.UPLOAD_EXTENSIONS_WHITELIST", [".py", ".sh"]
+    ):
+        # forbidden by default, now explicitly allowed
+        assert is_supported_extension("model.py")
+        assert is_supported_extension("scripts/deploy.sh")
+        # match is case-insensitive
+        assert is_supported_extension("MODEL.PY")
+        # extensions not in the override stay blocked
+        assert not is_supported_extension("malware.exe")
+        assert not is_supported_extension("app.js")
+
+
+def test_allowed_mime_types_override():
+    """MIME types in UPLOAD_MIME_TYPES_WHITELIST are accepted even though they are in FORBIDDEN_MIME_TYPES."""
+    with patch("mergin.sync.utils.get_mimetype", return_value="text/x-shellscript"):
+        # blocked by default
+        with patch("mergin.sync.utils.Configuration.UPLOAD_MIME_TYPES_WHITELIST", []):
+            assert not is_supported_type("deploy.sh")
+        # explicitly allowed
+        with patch(
+            "mergin.sync.utils.Configuration.UPLOAD_MIME_TYPES_WHITELIST",
+            ["text/x-shellscript"],
+        ):
+            assert is_supported_type("deploy.sh")