Fix command line too long on windows. (#509)

* Fix command line too long on windows.

* Fix command line too long on windows.

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* Add linux / macos case.

* Split functions, legacy and now through script.

* Fix transfer on linux / macos.

* Remove breakpoint.

* Fix tests.

* Tidy up.

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

Author: JoeZiminski

datashuttle/tui/shared/validate_content.py

@@ -155,7 +155,7 @@ def on_button_pressed(self, event: Button.Pressed) -> None:
                 )
                 if not success:
                     self.parent_class.mainwindow.show_modal_error_dialog(
-                        cast(str, output)  # noqa
+                        cast("str", output)
                     )
                 else:
                     self.write_results_to_richlog(output)

datashuttle/utils/rclone.py

@@ -1,4 +1,7 @@
+import os
+import platform
 import subprocess
+import tempfile
 from pathlib import Path
 from subprocess import CompletedProcess
 from typing import Dict, List, Literal
@@ -30,6 +33,45 @@ def call_rclone(command: str, pipe_std: bool = False) -> CompletedProcess:
     return output
 
 
+def call_rclone_through_script(command: str) -> CompletedProcess:
+    """
+    Call rclone through a script, to avoid limits on command-line calls
+    (in particular on Windows). Used for transfers due to generation of
+    large call strings.
+    """
+    system = platform.system()
+
+    command = "rclone " + command
+
+    if system == "Windows":
+        suffix = ".bat"
+    else:
+        suffix = ".sh"
+        command = "#!/bin/bash\n" + command
+
+    with tempfile.NamedTemporaryFile(
+        mode="w", suffix=suffix, delete=False
+    ) as tmp_script:
+        tmp_script.write(command)
+        tmp_script_path = tmp_script.name
+
+    try:
+        if system != "Windows":
+            os.chmod(tmp_script_path, 0o700)
+
+        output = subprocess.run(
+            [tmp_script_path],
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            shell=False,
+        )
+
+    finally:
+        os.remove(tmp_script_path)
+
+    return output
+
+
 # -----------------------------------------------------------------------------
 # Setup
 # -----------------------------------------------------------------------------
@@ -189,19 +231,17 @@ def transfer_data(
     extra_arguments = handle_rclone_arguments(rclone_options, include_list)
 
     if upload_or_download == "upload":
-        output = call_rclone(
+        output = call_rclone_through_script(
             f"{rclone_args('copy')} "
             f'"{local_filepath}" "{cfg.get_rclone_config_name()}:'
             f'{central_filepath}" {extra_arguments}',
-            pipe_std=True,
         )
 
     elif upload_or_download == "download":
-        output = call_rclone(
+        output = call_rclone_through_script(
             f"{rclone_args('copy')} "
             f'"{cfg.get_rclone_config_name()}:'
             f'{central_filepath}" "{local_filepath}"  {extra_arguments}',
-            pipe_std=True,
         )
 
     return output

tests/tests_integration/test_datatypes.py

@@ -63,20 +63,18 @@ def test_transfer_datatypes(
 
         # Unfortunately on Windows we are encountering 'The command line is too long'
         # and so cannot test against all datatypes here.
-        some_narrow_datatypes = canonical_configs.quick_get_narrow_datatypes()[
-            :10
-        ]
+        narrow_datatypes = canonical_configs.quick_get_narrow_datatypes()
 
         datatypes_used = self.get_narrow_only_datatypes_used(used=False)
-        for key in some_narrow_datatypes:
+        for key in narrow_datatypes:
             datatypes_used[key] = True
 
         test_utils.make_and_check_local_project_folders(
             project,
             "rawdata",
             subs,
             sessions,
-            some_narrow_datatypes,
+            narrow_datatypes,
             datatypes_used,
         )
 
@@ -87,7 +85,7 @@ def test_transfer_datatypes(
             project, upload_or_download, "custom", "rawdata"
         )
 
-        transfer_function("rawdata", "all", "all", some_narrow_datatypes)
+        transfer_function("rawdata", "all", "all", narrow_datatypes)
 
         test_utils.check_folder_tree_is_correct(
             os.path.join(base_path_to_check, "rawdata"),