Set ZARR_LARGE_CHUNK_THRESHOLD to 100MB

Author: jjnesbitt

dandi/consts.py

@@ -200,7 +200,7 @@ def urls(self) -> Iterator[str]:
 ZARR_DELETE_BATCH_SIZE = 100
 
 #: Zarr chunks above this size (bytes) are uploaded via multipart upload
-ZARR_LARGE_CHUNK_THRESHOLD = 5 * 1024 * 1024 * 1024  # 5 GB
+ZARR_LARGE_CHUNK_THRESHOLD = 100 * 1024 * 1024  # 100 MB
 
 BIDS_DATASET_DESCRIPTION = "dataset_description.json"
 

dandi/files/zarr.py

@@ -756,7 +756,7 @@ def mkzarr() -> str:
                     for it in all_items:
                         zcc.add_leaf(Path(it.entry_path), it.size, it.digest)
 
-                    # Upload chunks above 5GB individually via multipart upload
+                    # Upload chunks above ZARR_LARGE_CHUNK_THRESHOLD individually via multipart upload
                     for it in large_items:
                         # Yield uploading status
                         yield from _multipart_upload(