diff --git a/datashuttle/tui/shared/validate_content.py b/datashuttle/tui/shared/validate_content.py index 949e8494c..ede87a31d 100644 --- a/datashuttle/tui/shared/validate_content.py +++ b/datashuttle/tui/shared/validate_content.py @@ -155,7 +155,7 @@ def on_button_pressed(self, event: Button.Pressed) -> None: ) if not success: self.parent_class.mainwindow.show_modal_error_dialog( - cast(str, output) # noqa + cast("str", output) ) else: self.write_results_to_richlog(output) diff --git a/datashuttle/utils/rclone.py b/datashuttle/utils/rclone.py index 9644c103c..49d7da826 100644 --- a/datashuttle/utils/rclone.py +++ b/datashuttle/utils/rclone.py @@ -1,4 +1,7 @@ +import os +import platform import subprocess +import tempfile from pathlib import Path from subprocess import CompletedProcess from typing import Dict, List, Literal @@ -30,6 +33,45 @@ def call_rclone(command: str, pipe_std: bool = False) -> CompletedProcess: return output +def call_rclone_through_script(command: str) -> CompletedProcess: + """ + Call rclone through a script, to avoid limits on command-line calls + (in particular on Windows). Used for transfers due to generation of + large call strings. + """ + system = platform.system() + + command = "rclone " + command + + if system == "Windows": + suffix = ".bat" + else: + suffix = ".sh" + command = "#!/bin/bash\n" + command + + with tempfile.NamedTemporaryFile( + mode="w", suffix=suffix, delete=False + ) as tmp_script: + tmp_script.write(command) + tmp_script_path = tmp_script.name + + try: + if system != "Windows": + os.chmod(tmp_script_path, 0o700) + + output = subprocess.run( + [tmp_script_path], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + shell=False, + ) + + finally: + os.remove(tmp_script_path) + + return output + + # ----------------------------------------------------------------------------- # Setup # ----------------------------------------------------------------------------- @@ -189,19 +231,17 @@ def transfer_data( extra_arguments = handle_rclone_arguments(rclone_options, include_list) if upload_or_download == "upload": - output = call_rclone( + output = call_rclone_through_script( f"{rclone_args('copy')} " f'"{local_filepath}" "{cfg.get_rclone_config_name()}:' f'{central_filepath}" {extra_arguments}', - pipe_std=True, ) elif upload_or_download == "download": - output = call_rclone( + output = call_rclone_through_script( f"{rclone_args('copy')} " f'"{cfg.get_rclone_config_name()}:' f'{central_filepath}" "{local_filepath}" {extra_arguments}', - pipe_std=True, ) return output diff --git a/tests/tests_integration/test_datatypes.py b/tests/tests_integration/test_datatypes.py index 1aec45480..1e1776e41 100644 --- a/tests/tests_integration/test_datatypes.py +++ b/tests/tests_integration/test_datatypes.py @@ -63,12 +63,10 @@ def test_transfer_datatypes( # Unfortunately on Windows we are encountering 'The command line is too long' # and so cannot test against all datatypes here. - some_narrow_datatypes = canonical_configs.quick_get_narrow_datatypes()[ - :10 - ] + narrow_datatypes = canonical_configs.quick_get_narrow_datatypes() datatypes_used = self.get_narrow_only_datatypes_used(used=False) - for key in some_narrow_datatypes: + for key in narrow_datatypes: datatypes_used[key] = True test_utils.make_and_check_local_project_folders( @@ -76,7 +74,7 @@ def test_transfer_datatypes( "rawdata", subs, sessions, - some_narrow_datatypes, + narrow_datatypes, datatypes_used, ) @@ -87,7 +85,7 @@ def test_transfer_datatypes( project, upload_or_download, "custom", "rawdata" ) - transfer_function("rawdata", "all", "all", some_narrow_datatypes) + transfer_function("rawdata", "all", "all", narrow_datatypes) test_utils.check_folder_tree_is_correct( os.path.join(base_path_to_check, "rawdata"),