Merge branch 'main' into include-tab-ingest

Author: JR-1991

.github/workflows/test.yml

@@ -1,14 +1,21 @@
 name: Integration Tests
 
-on: [push]
+on: [push, pull_request]
 
 jobs:
   build:
     runs-on: ubuntu-latest
+
+    services:
+      squid:
+        image: ubuntu/squid:latest
+        ports:
+          - 3128:3128
+
     strategy:
       max-parallel: 4
       matrix:
-        python-version: ['3.8', '3.9', '3.10', '3.11']
+        python-version: ["3.8", "3.9", "3.10", "3.11"]
 
     env:
       PORT: 8080

dvuploader/cli.py

@@ -1,9 +1,11 @@
 import yaml
 import typer
 
+from pathlib import Path
 from pydantic import BaseModel
 from typing import List, Optional
 from dvuploader import DVUploader, File
+from dvuploader.utils import add_directory
 
 
 class CliInput(BaseModel):
@@ -27,6 +29,29 @@ class CliInput(BaseModel):
 
 app = typer.Typer()
 
+def _enumerate_filepaths(filepaths: List[str], recurse: bool) -> List[File]:
+    """
+    Take a list of filepaths and transform it into a list of File objects, optionally recursing into each of them.
+
+    Args:
+        filepaths (List[str]): a list of files or paths for upload
+        recurse (bool): whether to recurse into each given filepath
+
+    Returns:
+        List[File]: A list of File objects representing the files extracted from all filepaths.
+    
+    Raises:
+        FileNotFoundError: If a filepath does not exist.
+        IsADirectoryError: If recurse is False and a filepath points to a directory instead of a file.
+    """
+    if not recurse:
+        return [File(filepath=filepath) for filepath in filepaths]
+
+    files = []
+    for fp in filepaths:
+        files.extend(add_directory(fp) if Path(fp).is_dir() else [File(filepath=fp)])
+    return files
+
 
 def _parse_yaml_config(path: str) -> CliInput:
     """
@@ -50,6 +75,7 @@ def _validate_inputs(
     pid: str,
     dataverse_url: str,
     api_token: str,
+    recurse: bool,
     config_path: Optional[str],
 ) -> None:
     """
@@ -62,16 +88,23 @@ def _validate_inputs(
         pid (str): Persistent identifier of the dataset
         dataverse_url (str): URL of the Dataverse instance
         api_token (str): API token for authentication
+        recurse (bool): Whether to recurse into filepaths
         config_path (Optional[str]): Path to configuration file
 
     Raises:
         typer.BadParameter: If both config file and filepaths are specified
+        typer.BadParameter: If both config file and recurse are specified
         typer.BadParameter: If neither config file nor required parameters are provided
     """
-    if config_path is not None and len(filepaths) > 0:
-        raise typer.BadParameter(
-            "Cannot specify both a JSON/YAML file and a list of filepaths."
-        )
+    if config_path is not None:
+        if len(filepaths) > 0:
+            raise typer.BadParameter(
+                "Cannot specify both a JSON/YAML file and a list of filepaths."
+            )
+        if recurse:
+            raise typer.BadParameter(
+                "Cannot specify both a JSON/YAML file and recurse into filepaths."
+            )
 
     _has_meta_params = all(arg is not None for arg in [pid, dataverse_url, api_token])
     _has_config_file = config_path is not None
@@ -90,10 +123,14 @@ def _validate_inputs(
 
 @app.command()
 def main(
-    filepaths: List[str] = typer.Argument(
+    filepaths: Optional[List[str]] = typer.Argument(
         default=None,
         help="A list of filepaths to upload.",
     ),
+    recurse: Optional[bool] = typer.Option(
+        default=False,
+        help="Enable recursion into filepaths.",
+    ),
     pid: str = typer.Option(
         default=None,
         help="The persistent identifier of the Dataverse dataset.",
@@ -123,6 +160,7 @@ def main(
 
     If using command line arguments, you must specify:
     - One or more filepaths to upload
+    - (Optional) whether to recurse into the filepaths
     - The dataset's persistent identifier
     - A valid API token
     - The Dataverse repository URL
@@ -141,11 +179,16 @@ def main(
         Upload files via config file:
         $ dvuploader --config-path upload_config.yaml
     """
+
+    if filepaths is None:
+        filepaths = []
+
     _validate_inputs(
         filepaths=filepaths,
         pid=pid,
         dataverse_url=dataverse_url,
         api_token=api_token,
+        recurse=recurse,
         config_path=config_path,
     )
 
@@ -157,7 +200,7 @@ def main(
             api_token=api_token,
             dataverse_url=dataverse_url,
             persistent_id=pid,
-            files=[File(filepath=filepath) for filepath in filepaths],
+            files=_enumerate_filepaths(filepaths=filepaths, recurse=recurse),
         )
 
     uploader = DVUploader(files=cli_input.files)

dvuploader/directupload.py

@@ -36,6 +36,7 @@ async def direct_upload(
     progress,
     pbars,
     n_parallel_uploads: int,
+    proxy: Optional[str] = None,
 ) -> None:
     """
     Perform parallel direct upload of files to the specified Dataverse repository.
@@ -48,7 +49,7 @@ async def direct_upload(
         progress: Progress object to track upload progress.
         pbars: List of progress bars for each file.
         n_parallel_uploads (int): Number of concurrent uploads to perform.
-
+        proxy (str): The proxy to use for the upload.
     Returns:
         None
     """
@@ -58,6 +59,7 @@ async def direct_upload(
     session_params = {
         "timeout": None,
         "limits": httpx.Limits(max_connections=n_parallel_uploads),
+        "proxy": proxy,
     }
 
     async with httpx.AsyncClient(**session_params) as session:

dvuploader/dvuploader.py

@@ -58,6 +58,7 @@ def upload(
         n_parallel_uploads: int = 1,
         force_native: bool = False,
         replace_existing: bool = True,
+        proxy: Optional[str] = None,
     ) -> None:
         """
         Uploads the files to the specified Dataverse repository.
@@ -70,6 +71,7 @@ def upload(
                 this restricts parallel chunks per upload. Use n_jobs to control parallel files.
             force_native (bool): Forces the use of the native upload method instead of direct upload.
             replace_existing (bool): Whether to replace files that already exist in the dataset.
+            proxy (str): The proxy to use for the upload.
 
         Returns:
             None

dvuploader/nativeupload.py

@@ -7,7 +7,7 @@
 import tempfile
 import rich
 import tenacity
-from typing import List, Tuple, Dict
+from typing import List, Optional, Tuple, Dict
 
 from rich.progress import Progress, TaskID
 
@@ -63,6 +63,7 @@ async def native_upload(
     n_parallel_uploads: int,
     pbars,
     progress,
+    proxy: Optional[str] = None,
 ):
     """
     Executes native uploads for the given files in parallel.
@@ -75,7 +76,7 @@ async def native_upload(
         n_parallel_uploads (int): The number of parallel uploads to execute.
         pbars: List of progress bar IDs to track upload progress.
         progress: Progress object to manage progress bars.
-
+        proxy (str): The proxy to use for the upload.
     Returns:
         None
     """
@@ -87,6 +88,7 @@ async def native_upload(
         "headers": {"X-Dataverse-key": api_token},
         "timeout": None,
         "limits": httpx.Limits(max_connections=n_parallel_uploads),
+        "proxy": proxy,
     }
 
     async with httpx.AsyncClient(**session_params) as session:

pyproject.toml

@@ -9,7 +9,7 @@ readme = "README.md"
 python = "^3.9"
 pydantic = "^2.5.3"
 httpx = "^0.28"
-typer = "^0.9.0"
+typer = ">=0.9, <0.16"
 pyyaml = "^6.0.1"
 nest-asyncio = "^1.5.8"
 aiofiles = "^23.2.1"

tests/integration/test_native_upload.py

@@ -108,6 +108,55 @@ def test_forced_native_upload(
             assert len(files) == 3
             assert sorted([file["label"] for file in files]) == sorted(expected_files)
 
+    def test_native_upload_with_proxy(
+        self,
+        credentials,
+    ):
+        BASE_URL, API_TOKEN = credentials
+        proxy = "http://127.0.0.1:3128"
+
+        with tempfile.TemporaryDirectory() as directory:
+            # Arrange
+            create_mock_file(directory, "small_file.txt", size=1)
+            create_mock_file(directory, "mid_file.txt", size=50)
+            create_mock_file(directory, "large_file.txt", size=200)
+
+            # Add all files in the directory
+            files = add_directory(directory=directory)
+
+            # Create Dataset
+            pid = create_dataset(
+                parent="Root",
+                server_url=BASE_URL,
+                api_token=API_TOKEN,
+            )
+
+            # Act
+            uploader = DVUploader(files=files)
+            uploader.upload(
+                persistent_id=pid,
+                api_token=API_TOKEN,
+                dataverse_url=BASE_URL,
+                n_parallel_uploads=1,
+                proxy=proxy,
+            )
+
+            # Assert
+            files = retrieve_dataset_files(
+                dataverse_url=BASE_URL,
+                persistent_id=pid,
+                api_token=API_TOKEN,
+            )
+
+            expected_files = [
+                "small_file.txt",
+                "mid_file.txt",
+                "large_file.txt",
+            ]
+
+            assert len(files) == 3
+            assert sorted([file["label"] for file in files]) == sorted(expected_files)
+
     def test_native_upload_by_handler(
         self,
         credentials,

tests/unit/test_cli.py

@@ -57,6 +57,32 @@ def test_kwarg_arg_input(self, credentials):
         )
         assert result.exit_code == 0
 
+    def test_recurse(self, credentials):
+        # Arrange
+        BASE_URL, API_TOKEN = credentials
+        pid = create_dataset(
+            parent="Root",
+            server_url=BASE_URL,
+            api_token=API_TOKEN,
+        )
+
+        # Act
+        result = runner.invoke(
+            app,
+            [
+                "./tests/fixtures/create_dataset.json",
+                "./tests/fixtures/add_dir_files",
+                "--recurse",
+                "--pid",
+                pid,
+                "--api-token",
+                API_TOKEN,
+                "--dataverse-url",
+                BASE_URL,
+            ],
+        )
+        assert result.exit_code == 0
+
     def test_yaml_input(self, credentials):
         # Arrange
         BASE_URL, API_TOKEN = credentials