Improve transfer overwrite options (#696)

JoeZiminski · Sushma-1706 · pre-commit-ci[bot] · web-flow · commit 1603dff022b6 · 2026-03-04T11:22:54.000Z
* refactor: clarify overwrite transfer options * docs: clarify overwrite transfer behavior * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Implement, manual test, update docs. * Working through tests. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Fix tests. * Update docs/source/pages/user_guides/transfer-data.md Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * Update docs/source/pages/get_started/getting-started.md Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * Update datashuttle/utils/data_transfer.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * Update datashuttle/datashuttle_class.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * Fix all docstrings. --------- Co-authored-by: Sushma-1706 <damacharlasushma@gmail.com> Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
diff --git a/datashuttle/datashuttle_class.py b/datashuttle/datashuttle_class.py
@@ -352,10 +352,12 @@ def upload_custom(
 
         overwrite_existing_files
             If ``"never"`` files on target will never be overwritten by source.
-            If ``"always"`` files on target will be overwritten by source if
-            there is any difference in date or size.
             If ``"if_source_newer"`` files on target will only be overwritten
             by files on source with newer creation / modification datetime.
+            If ``"if_different"``, target will always be overwritten if the
+            size or creation / modification datetimes differ.
+            If ``"always"`` files on target will always be overwritten by source,
+            even when size and creation / modification datetimes match.
 
         dry_run
             Perform a dry-run of transfer. This will output as if file
@@ -442,10 +444,12 @@ def download_custom(
 
         overwrite_existing_files
             If ``"never"`` files on target will never be overwritten by source.
-            If ``"always"`` files on target will be overwritten by source if
-            there is any difference in date or size.
             If ``"if_source_newer"`` files on target will only be overwritten
             by files on source with newer creation / modification datetime.
+            If ``"if_different"``, target will always be overwritten if the
+            size or creation / modification datetimes differ.
+            If ``"always"`` files on target will always be overwritten by source,
+            even when size and creation / modification datetimes match.
 
         dry_run
             Perform a dry-run of transfer. This will output as if file
@@ -513,10 +517,12 @@ def upload_rawdata(
         ----------
         overwrite_existing_files
             If ``"never"`` files on target will never be overwritten by source.
-            If ``"always"`` files on target will be overwritten by source if
-            there is any difference in date or size.
             If ``"if_source_newer"`` files on target will only be overwritten
             by files on source with newer creation / modification datetime.
+            If ``"if_different"``, target will always be overwritten if the
+            size or creation / modification datetimes differ.
+            If ``"always"`` files on target will always be overwritten by source,
+            even when size and creation / modification datetimes match.
 
         dry_run
             Perform a dry-run of transfer. This will output as if file
@@ -543,10 +549,12 @@ def upload_derivatives(
         ----------
         overwrite_existing_files
             If ``"never"`` files on target will never be overwritten by source.
-            If ``"always"`` files on target will be overwritten by source if
-            there is any difference in date or size.
             If ``"if_source_newer"`` files on target will only be overwritten
             by files on source with newer creation / modification datetime.
+            If ``"if_different"``, target will always be overwritten if the
+            size or creation / modification datetimes differ.
+            If ``"always"`` files on target will always be overwritten by source,
+            even when size and creation / modification datetimes match.
 
         dry_run
             Perform a dry-run of transfer. This will output as if file
@@ -573,10 +581,12 @@ def download_rawdata(
         ----------
         overwrite_existing_files
             If ``"never"`` files on target will never be overwritten by source.
-            If ``"always"`` files on target will be overwritten by source if
-            there is any difference in date or size.
             If ``"if_source_newer"`` files on target will only be overwritten
             by files on source with newer creation / modification datetime.
+            If ``"if_different"``, target will always be overwritten if the
+            size or creation / modification datetimes differ.
+            If ``"always"`` files on target will always be overwritten by source,
+            even when size and creation / modification datetimes match.
 
         dry_run
             Perform a dry-run of transfer. This will output as if file
@@ -603,10 +613,12 @@ def download_derivatives(
         ----------
         overwrite_existing_files
             If ``"never"`` files on target will never be overwritten by source.
-            If ``"always"`` files on target will be overwritten by source if
-            there is any difference in date or size.
             If ``"if_source_newer"`` files on target will only be overwritten
             by files on source with newer creation / modification datetime.
+            If ``"if_different"``, target will always be overwritten if the
+            size or creation / modification datetimes differ.
+            If ``"always"`` files on target will always be overwritten by source,
+            even when size and creation / modification datetimes match.
 
         dry_run
             Perform a dry-run of transfer. This will output as if file
@@ -635,10 +647,12 @@ def upload_entire_project(
         ----------
         overwrite_existing_files
             If ``"never"`` files on target will never be overwritten by source.
-            If ``"always"`` files on target will be overwritten by source if
-            there is any difference in date or size.
             If ``"if_source_newer"`` files on target will only be overwritten
             by files on source with newer creation / modification datetime.
+            If ``"if_different"``, target will always be overwritten if the
+            size or creation / modification datetimes differ.
+            If ``"always"`` files on target will always be overwritten by source,
+            even when size and creation / modification datetimes match.
 
         dry_run
             Perform a dry-run of transfer. This will output as if file
@@ -675,10 +689,12 @@ def download_entire_project(
         ----------
         overwrite_existing_files
             If ``"never"`` files on target will never be overwritten by source.
-            If ``"always"`` files on target will be overwritten by source if
-            there is any difference in date or size.
             If ``"if_source_newer"`` files on target will only be overwritten
             by files on source with newer creation / modification datetime.
+            If ``"if_different"``, target will always be overwritten if the
+            size or creation / modification datetimes differ.
+            If ``"always"`` files on target will always be overwritten by source,
+            even when size and creation / modification datetimes match.
 
         dry_run
             Perform a dry-run of transfer. This will output as if file
@@ -722,10 +738,12 @@ def upload_specific_folder_or_file(
 
         overwrite_existing_files
             If ``"never"`` files on target will never be overwritten by source.
-            If ``"always"`` files on target will be overwritten by source if
-            there is any difference in date or size.
             If ``"if_source_newer"`` files on target will only be overwritten
             by files on source with newer creation / modification datetime.
+            If ``"if_different"``, target will always be overwritten if the
+            size or creation / modification datetimes differ.
+            If ``"always"`` files on target will always be overwritten by source,
+            even when size and creation / modification datetimes match.
 
         dry_run
             Perform a dry-run of transfer. This will output as if file
@@ -771,10 +789,12 @@ def download_specific_folder_or_file(
 
         overwrite_existing_files
             If ``"never"`` files on target will never be overwritten by source.
-            If ``"always"`` files on target will be overwritten by source if
-            there is any difference in date or size.
             If ``"if_source_newer"`` files on target will only be overwritten
             by files on source with newer creation / modification datetime.
+            IF ``"if_different"``, target will always be overwritten if the
+            size or creation / modification datetimes differ.
+            If ``"always"`` files on target will be overwritten by source if
+            there is any difference in datetimes or size.
 
         dry_run
             Perform a dry-run of transfer. This will output as if file
diff --git a/datashuttle/tui/tabs/transfer.py b/datashuttle/tui/tabs/transfer.py
@@ -179,7 +179,12 @@ def compose(self) -> ComposeResult:
                     Select(
                         (
                             (name, name)
-                            for name in ["Never", "Always", "If Source Newer"]
+                            for name in [
+                                "Never",
+                                "If Source Newer",
+                                "If Different",
+                                "Always",
+                            ]
                         ),
                         value=self.interface.tui_settings[
                             "overwrite_existing_files"
@@ -238,7 +243,12 @@ def on_mount(self) -> None:
     def on_select_changed(self, event: Select.Changed) -> None:
         """Handle a Select widget changed on the tab."""
         if event.select.id == "transfer_tab_overwrite_select":
-            assert event.select.value in ["Never", "Always", "If Source Newer"]
+            assert event.select.value in [
+                "Never",
+                "If Source Newer",
+                "If Different",
+                "Always",
+            ]
             format_select = event.select.value.lower().replace(" ", "_")
             self.interface.save_tui_settings(
                 format_select,
diff --git a/datashuttle/utils/custom_types.py b/datashuttle/utils/custom_types.py
@@ -6,7 +6,9 @@
 
 TopLevelFolder = Literal["rawdata", "derivatives"]
 
-OverwriteExistingFiles = Literal["never", "always", "if_source_newer"]
+OverwriteExistingFiles = Literal[
+    "never", "if_source_newer", "if_different", "always"
+]
 
 Prefix = Literal["sub", "ses"]
 
diff --git a/datashuttle/utils/data_transfer.py b/datashuttle/utils/data_transfer.py
@@ -62,10 +62,13 @@ def __init__(
 
         overwrite_existing_files
             If ``"never"`` files on target will never be overwritten by source.
-            If ``"always"`` files on target will be overwritten by source if
-            there is any difference in date or size.
             If ``"if_source_newer"`` files on target will only be overwritten
             by files on source with newer creation / modification datetime.
+            If ``"if_different"``, target will always be overwritten if the
+            size or creation / modification datetimes differ.
+            If ``"always"``, files on target will always be overwritten by
+            source, even when size and creation / modification datetimes
+            are identical.
 
         dry_run
             Perform a dry-run of transfer. This will output as if file
diff --git a/datashuttle/utils/rclone.py b/datashuttle/utils/rclone.py
@@ -799,7 +799,7 @@ def make_rclone_transfer_options(
     overwrite_existing_files: OverwriteExistingFiles, dry_run: bool
 ) -> Dict:
     """Create a dictionary of rclone transfer options."""
-    allowed_overwrite = ["never", "always", "if_source_newer"]
+    allowed_overwrite = ["never", "if_source_newer", "if_different", "always"]
 
     if overwrite_existing_files not in allowed_overwrite:
         utils.log_and_raise_error(
@@ -967,9 +967,13 @@ def handle_rclone_arguments(
     if overwrite == "never":
         extra_arguments_list += [rclone_args("never_overwrite")]
 
-    elif overwrite == "always":
+    elif overwrite == "if_different":
+        # default rclone behavior (no flags)
         pass
 
+    elif overwrite == "always":
+        extra_arguments_list += [rclone_args("always_overwrite")]
+
     elif overwrite == "if_source_newer":
         extra_arguments_list += [rclone_args("if_source_newer_overwrite")]
 
@@ -993,6 +997,7 @@ def rclone_args(name: str) -> str:
         "copy",
         "never_overwrite",
         "if_source_newer_overwrite",
+        "always_overwrite",
         "progress",
         "check",
     ]
@@ -1010,6 +1015,9 @@ def rclone_args(name: str) -> str:
     if name == "if_source_newer_overwrite":
         arg = "--update"
 
+    if name == "always_overwrite":
+        arg = "--ignore-times"
+
     if name == "progress":
         arg = "--progress"
 
diff --git a/docs/source/pages/get_started/getting-started.md b/docs/source/pages/get_started/getting-started.md
@@ -498,7 +498,7 @@ central storage machine connected as a mounted drive or through SSH.
 
 ```{warning}
 The **overwrite existing files** setting is very important.
-It takes on the options **never**, **always** or **if source newer**.
+It takes on the options **never**, **if source newer**, **if_different**, or **always**.
 
 See the [transfer options](transfer-options) section for full details.
 ```
diff --git a/docs/source/pages/user_guides/transfer-data.md b/docs/source/pages/user_guides/transfer-data.md
@@ -46,7 +46,7 @@ allow transfer across:
 
 ```{warning}
 The **overwrite existing files** setting is very important.
-It takes on the options **never**, **always** or **if source newer**.
+It takes on the options **never**, **if source newer**, **if different** or **always**.
 
 See the [transfer options](transfer-options) section for full details on
 this and other transfer settings.
@@ -304,28 +304,20 @@ Transfer a range
 (transfer-options)=
 
 overwrite existing files
-: By default this option is set to **never**—a transfer will never overwrite a
-file that already exists, even if the source and destination modification datetimes
-or sizes are different.
-
-: If *always**, when there are differences in datetime or size
-between the source and destination file the destination file will be overwritten.
-This includes when the source file is older or smaller than the destination.
-
-: Finally, **if source newer** ensures data is only overwritten
-when the
-[source file has a more recent modification time](https://rclone.org/docs/#u-update)
-than the destination.
-If modification datetimes are equal, the destination will be overwritten if the
-sizes or checksums are different.
-
-: Under the hood, transfers are made with calls to
-[Rclone](https://rclone.org/). Using **never**
-calls
-[Rclone's copy](https://rclone.org/commands/rclone_copy/)
-function with the flag `--ignore_existing`. Using
-**always** copies without this flag and (using Rclone's default overwrite behaviour.)
-Using **if source newer** calls copy with the `--update` flag.
+: Controls how existing files are handled during transfer.
+
+: **never**
+  Never overwrite existing files.
+
+: **if_source_newer**
+  Only overwrite files if the source file is newer than the destination.
+
+: **if_different**
+  Only overwrite files if the source and destination differ in
+  modification time or size.
+
+: **always**
+  Always overwrite files, even if sizes and timestamps are identical.
 
 (dry-run-argument)=
 dry run
diff --git a/tests/tests_transfers/local_filesystem/test_transfer.py b/tests/tests_transfers/local_filesystem/test_transfer.py
diff --git a/tests/tests_tui/test_tui_transfer.py b/tests/tests_tui/test_tui_transfer.py
diff --git a/tests/tests_tui/test_tui_widgets_and_defaults.py b/tests/tests_tui/test_tui_widgets_and_defaults.py
