Add proxy support to dataset file retrieval

Introduces an optional proxy parameter to dataset file retrieval functions and methods in dvuploader.py and utils.py, allowing HTTP requests to be routed through a specified proxy. This enhances flexibility for users operating behind network proxies.

Author: JR-1991

dvuploader/dvuploader.py

@@ -105,6 +105,7 @@ def upload(
             persistent_id=persistent_id,
             api_token=api_token,
             replace_existing=replace_existing,
+            proxy=proxy,
         )
 
         # Sort files by size
@@ -198,7 +199,8 @@ def _check_duplicates(
         persistent_id: str,
         api_token: str,
         replace_existing: bool,
-    ):
+        proxy: Optional[str] = None,
+    ) -> None:
         """
         Checks for duplicate files in the dataset by comparing paths and filenames.
 
@@ -207,7 +209,7 @@ def _check_duplicates(
             persistent_id (str): The persistent ID of the dataset.
             api_token (str): The API token for accessing the Dataverse repository.
             replace_existing (bool): Whether to replace files that already exist.
-
+            proxy (Optional[str]): The proxy to use for the request.
         Returns:
             None
         """
@@ -216,6 +218,7 @@ def _check_duplicates(
             dataverse_url=dataverse_url,
             persistent_id=persistent_id,
             api_token=api_token,
+            proxy=proxy,
         )
 
         table = Table(
@@ -241,14 +244,14 @@ def _check_duplicates(
                 to_skip.append(file.file_id)
 
                 if replace_existing:
-                    ds_file = self._get_dsfile_by_id(file.file_id, ds_files)
-                    if not self._check_size(file, ds_file):
+                    ds_file = self._get_dsfile_by_id(file.file_id, ds_files)  # type: ignore
+                    if not self._check_size(file, ds_file):  # type: ignore
                         file._unchanged_data = False
                     else:
                         # calculate checksum
                         file.update_checksum_chunked()
                         file.apply_checksum()
-                        file._unchanged_data = self._check_hashes(file, ds_file)
+                        file._unchanged_data = self._check_hashes(file, ds_file)  # type: ignore
                     if file._unchanged_data:
                         table.add_row(
                             file.file_name,

dvuploader/utils.py

@@ -1,8 +1,9 @@
 import os
 import pathlib
 import re
-from typing import List
+from typing import List, Optional
 from urllib.parse import urljoin
+
 import httpx
 from rich.progress import Progress
 
@@ -40,6 +41,7 @@ def retrieve_dataset_files(
     dataverse_url: str,
     persistent_id: str,
     api_token: str,
+    proxy: Optional[str] = None,
 ):
     """
     Retrieve the files of a specific dataset from a Dataverse repository.
@@ -48,6 +50,7 @@ def retrieve_dataset_files(
         dataverse_url (str): The base URL of the Dataverse repository.
         persistent_id (str): The persistent identifier (PID) of the dataset.
         api_token (str): API token for authentication.
+        proxy (Optional[str]): The proxy to use for the request.
 
     Returns:
         list: A list of files in the dataset.
@@ -61,6 +64,7 @@ def retrieve_dataset_files(
     response = httpx.get(
         urljoin(dataverse_url, DATASET_ENDPOINT),
         headers={"X-Dataverse-key": api_token},
+        proxy=proxy,
     )
 
     response.raise_for_status()