cleaning input path to account for ending '/' as well as 'incoming'

Author: davezuckerman

mokelumne/dags/copy_files.py

@@ -14,6 +14,7 @@
 from mokelumne.util.storage import run_dir
 from mokelumne.util.file_transfer import (
     build_file_manifest,
+    clean_destination_path,
     copy_files_from_manifest,
     load_json,
     save_json,
@@ -76,7 +77,7 @@ def prepare_destination() -> str:
 
         ctx = get_current_context()
         destination = ctx["params"]["destination"]
-
+        destination = clean_destination_path(destination)
         destination_path = Path(destination)
 
         if not destination_path.exists():

mokelumne/util/file_transfer.py

@@ -2,6 +2,7 @@
 
 import hashlib
 import json
+import re
 import shutil
 
 from pathlib import Path
@@ -64,6 +65,11 @@ def load_json(path: Path) -> Manifest:
     with open(path, "r", encoding="utf-8") as json_file:
         return json.load(json_file)
 
+def clean_destination_path(destination_path: str) -> str:
+    clean_path = re.sub(r'/incoming$|/incoming/$|/$', '', destination_path)
+    clean_path = clean_path + '/incoming'
+
+    return clean_path
 
 def verify_file_manifest(destination_path: Path, manifest_path: Path) -> list[ManifestEntry]:
     """Verify all copied files exist at the destination."""

test/unit/test_file_transfer.py

@@ -95,6 +95,20 @@ def test_build_manifest_excludes_hidden_and_excluded_files(self, tmp_path: Path)
         assert paths == {"visible.tif"}
         assert len(result["files"]) == 1
 
+    @pytest.mark.parametrize(
+        ("input_path", "expected_path"),
+        [
+            ("/srv/pa/aerial/ucb", "/srv/pa/aerial/ucb/incoming"),
+            ("/srv/pa/aerial/ucb/", "/srv/pa/aerial/ucb/incoming"),
+            ("/srv/pa/aerial/ucb/incoming", "/srv/pa/aerial/ucb/incoming"),
+            ("/srv/pa/aerial/ucb/incoming/", "/srv/pa/aerial/ucb/incoming"),
+        ],
+    )
+    def test_clean_destination_path(self, input_path: str, expected_path: str):
+        """Ensure clean_destination_path normalizes the incoming directory suffix."""
+
+        assert file_transfer.clean_destination_path(input_path) == expected_path
+
     def test_verify_manifest_success(self, tmp_path: Path):
         """Ensure verify_manifest succeeds for valid files."""