feat: add multipart encoder when uploading (large) files

With the previous implementation of `upload_files`, the SDK was loading
the whole file to be uploaded in the memory and only then sending it
over the socket. This was painfully slow, memory constrained and lacked
real feedback.

Therefore the `requests_toolbelt`'s `MultipartEncoderMonitor` is
introduced in this PR, which keeps the memory consumption under control.

This is quite useful not only for CLI consumers of the SDK, but also for
QFieldCloud `qgis` workers, as the memory does not grow too much when
uploading large files after `package` or `apply_deltas` job.

Author: suricactus

qfieldcloud_sdk/sdk.py

@@ -11,6 +11,7 @@
 import requests
 import urllib3
 from requests.adapters import HTTPAdapter, Retry
+from requests_toolbelt.multipart.encoder import MultipartEncoderMonitor
 
 from .interfaces import QfcException, QfcRequest, QfcRequestException
 from .utils import calc_etag, log, add_trailing_slash_to_url
@@ -665,20 +666,39 @@ def upload_file(
         # if the filepath is invalid, it will throw a new error `pathvalidate.ValidationError`
         is_valid_filepath(str(local_filename))
 
+
+        local_file_size = local_filename.stat().st_size
         with open(local_filename, "rb") as local_file:
-            upload_file = local_file
             if show_progress:
                 from tqdm import tqdm
-                from tqdm.utils import CallbackIOWrapper
 
                 progress_bar = tqdm(
-                    total=local_filename.stat().st_size,
+                    total=local_file_size,
                     unit_scale=True,
-                    desc=local_filename.stem,
+                    unit="B",
+                    desc=f'Uploading "{remote_filename}"...',
                 )
-                upload_file = CallbackIOWrapper(progress_bar.update, local_file, "read")
+                # upload_file = CallbackIOWrapper(progress_bar.update, local_file, "read")
+                def callback(monitor: MultipartEncoderMonitor) -> None:
+                    progress_bar.n = monitor.bytes_read
+                    progress_bar.refresh()
             else:
                 logger.info(f'Uploading file "{remote_filename}"…')
+                callback = lambda monitor: None
+
+            multipart_data = MultipartEncoderMonitor.from_fields(
+                fields={
+                    "file": (
+                        str(remote_filename),
+                        local_file,
+                        None,
+                        # {
+                        #     "Content-Length": str(local_file_size),
+                        # }
+                    ),
+                },
+                callback=callback,
+            )
 
             if upload_type == FileTransferType.PROJECT:
                 url = f"files/{project_id}/{remote_filename}"
@@ -695,8 +715,10 @@ def upload_file(
             return self._request(
                 "POST",
                 url,
-                files={
-                    "file": upload_file,
+                data=multipart_data,
+                headers={
+                    "Content-Type": multipart_data.content_type,
+                    "Accept": "application/json",
                 },
             )
 

requirements.txt

@@ -3,6 +3,7 @@ charset-normalizer>=3.2.0
 click>=8.1.5
 idna>=3.4
 requests>=2.31.0
+requests-toolbelt>=1.0.0
 tqdm>=4.65.0
 urllib3>=2.0.7
 pathvalidate>=3.2.1