make configurable

Author: martenson

lib/galaxy/config/sample/file_sources_conf.yml.sample

@@ -15,7 +15,7 @@
   # for accessing passwords stored in a vault:
   # password: ${user.user_vault.read_secret('preferences/owncloud/password')}
 
-  # By default, the plugin will use temp files to avoid loading entire files into memory. 
+  # By default, the plugin will use temp files to avoid loading entire files into memory.
   # You can change the directory here or omit to use the default temp directory.
   temp_path: /your/temp/path
   # Set writable to true if you have write access to this source
@@ -210,6 +210,10 @@
   # token: ${user.preferences['invenio_sandbox|token']} # Alternatively use this for retrieving the token from user preferences instead of the Vault
   public_name: ${user.preferences['invenio_sandbox|public_name']}
   writable: true
+  # Enable multipart upload for files of size >= threshold (values in MB, optional, disabled by default)
+  # multipart_threshold: 100
+  # Part size for multipart uploads (values in MB, optional, defaults to 5 MiB minimum)
+  # multipart_chunk_size: 50
 
 - type: zenodo
   id: zenodo

lib/galaxy/files/sources/_rdm.py

@@ -25,15 +25,15 @@
 class RDMFileSourceTemplateConfiguration(BaseFileSourceTemplateConfiguration):
     token: Optional[Union[str, TemplateExpansion]] = None
     public_name: Optional[Union[str, TemplateExpansion]] = None
-    multipart_threshold: Optional[Union[int, TemplateExpansion]] = None  # bytes
-    multipart_chunk_size: Optional[Union[int, TemplateExpansion]] = None  # bytes
+    multipart_threshold: Optional[Union[int, TemplateExpansion]] = None  # MB
+    multipart_chunk_size: Optional[Union[int, TemplateExpansion]] = None  # MB
 
 
 class RDMFileSourceConfiguration(BaseFileSourceConfiguration):
     token: Optional[str] = None
     public_name: Optional[str] = None
-    multipart_threshold: Optional[int] = None  # bytes
-    multipart_chunk_size: Optional[int] = None  # bytes
+    multipart_threshold: Optional[int] = None  # MB
+    multipart_chunk_size: Optional[int] = None  # MB
 
 
 class ContainerAndFileIdentifier(NamedTuple):

lib/galaxy/files/sources/invenio.py

@@ -113,14 +113,11 @@ class RecordLinks(TypedDict):
     reserve_doi: str
 
 
-# AWS S3 multipart limits (used by Invenio RDM)
-MIN_UPLOAD_PART_SIZE = 50 * 1024 * 1024  # 50 MiB
+# AWS S3 multipart default limits (used by Invenio RDM)
+MIN_UPLOAD_PART_SIZE = 5 * 1024 * 1024  # 5 MiB
 MAX_UPLOAD_PART_SIZE = 5 * 1024**3  # 5 GiB
 MAX_UPLOAD_PARTS = 10_000
 
-# Default threshold for using multipart upload (100 MiB)
-DEFAULT_MULTIPART_THRESHOLD = 100 * 1024 * 1024
-
 
 def calculate_multipart_params(file_size: int, preferred_part_size: int | None = None) -> tuple[int, int]:
     """Calculate optimal parts count and part size for multipart upload.
@@ -391,16 +388,12 @@ def upload_file_to_draft_container(
         context: FilesSourceRuntimeContext[RDMFileSourceConfiguration],
     ):
         file_size = os.path.getsize(file_path)
-        threshold = context.config.multipart_threshold
-
-        # Use default threshold if not configured
-        if threshold is None or threshold <= 0:
-            threshold = DEFAULT_MULTIPART_THRESHOLD
-
-        use_multipart = file_size >= threshold
 
+        threshold_mb = context.config.multipart_threshold
+        # Convert threshold from MB to bytes (config value is always in MB)
+        threshold_bytes = threshold_mb * 1024 * 1024 if threshold_mb else None
+        use_multipart = file_size >= threshold_bytes if threshold_bytes else False
         if use_multipart:
-            log.info(f"Using multipart upload for file '{filename}' ({file_size} bytes >= threshold {threshold})")
             self._upload_file_multipart(record_id, filename, file_path, file_size, context)
         else:
             self._upload_file_single(record_id, filename, file_path, context, file_size)
@@ -434,11 +427,10 @@ def _upload_file_single(
             response = requests.put(upload_file_content_url, data=file, headers=headers)
             # Handle 413 (Payload Too Large) - suggest using multipart upload
             if response.status_code == 413:
-                threshold_mb = DEFAULT_MULTIPART_THRESHOLD / (1024 * 1024)
                 raise Exception(
                     f"Failed to upload file '{filename}' ({file_size} bytes): HTTP 413 Payload Too Large. "
                     f"The server rejected the upload because the file is too large for a single request. "
-                    f"Please configure 'multipart_threshold' to {threshold_mb}MB or lower to enable multipart upload for files of this size."
+                    f"Please configure 'multipart_threshold' in the file source configuration to enable multipart upload for files of this size."
                 )
             self._ensure_response_has_expected_status_code(response, 200)
 
@@ -463,7 +455,9 @@ def _upload_file_multipart(
         4. Upload parts (parallel for > 2 parts)
         5. POST to commit URL
         """
-        preferred_part_size = context.config.multipart_chunk_size
+        preferred_part_size_mb = context.config.multipart_chunk_size
+        # Convert chunk size from MB to bytes (config value is always in MB)
+        preferred_part_size = preferred_part_size_mb * 1024 * 1024 if preferred_part_size_mb else None
         num_parts, part_size = calculate_multipart_params(file_size, preferred_part_size)
 
         log.info(f"Multipart upload: {num_parts} parts of {part_size} bytes each for '{filename}'")
@@ -472,7 +466,6 @@ def _upload_file_multipart(
         upload_file_url = record["links"]["files"]
         headers = self._get_request_headers(context, auth_required=True)
 
-        # Initialize multipart upload with transfer metadata
         file_metadata = {
             "key": filename,
             "size": file_size,
@@ -497,13 +490,8 @@ def _upload_file_multipart(
             )
 
         # Sort part links by part number to ensure correct ordering
-        # Invenio uses 'part' key, not 'part_number'
         part_links = sorted(part_links, key=lambda p: p.get("part", 0))
-
-        # Upload parts
         self._upload_parts(file_path, file_size, part_size, part_links, headers)
-
-        # Commit multipart upload
         response = requests.post(commit_url, json={}, headers=headers)
         self._ensure_response_has_expected_status_code(response, 200)
         log.info(f"Multipart upload completed for '{filename}'")

test/unit/files/test_invenio_multipart.py

@@ -20,25 +20,25 @@ def test_calculate_multipart_params_zero_byte(self):
         assert part_size == 0
 
     def test_calculate_multipart_params_small_file(self):
-        """Files under 50 MiB should use minimum part size."""
-        # 10 MiB file
-        file_size = 10 * 1024 * 1024
+        """Files under 5 MiB should use minimum part size."""
+        # 2 MiB file
+        file_size = 2 * 1024 * 1024
         parts, part_size = calculate_multipart_params(file_size)
         assert parts == 1
         assert part_size == MIN_UPLOAD_PART_SIZE
 
     def test_calculate_multipart_params_medium_file(self):
-        """Files between 50 MiB and 100 MiB."""
-        # 75 MiB file
-        file_size = 75 * 1024 * 1024
+        """Files between 5 MiB and 10 MiB."""
+        # 7.5 MiB file
+        file_size = 7 * 1024 * 1024 + 512 * 1024
         parts, part_size = calculate_multipart_params(file_size)
         assert parts == 2
         assert part_size == MIN_UPLOAD_PART_SIZE
 
     def test_calculate_multipart_params_large_file(self):
         """Large files requiring multiple parts."""
-        # 250 MiB file
-        file_size = 250 * 1024 * 1024
+        # 25 MiB file
+        file_size = 25 * 1024 * 1024
         parts, part_size = calculate_multipart_params(file_size)
         assert parts == 5
         assert part_size == MIN_UPLOAD_PART_SIZE