Use lsjson for searches (neuroinformatics-unit#551)

* Add get_rclone_config_name_local and rename get_rclone_config_name to get_rclone_config_name_central.

* Add new transfer function.

* Fix rename in tests.

* Tidy up the search functions.

* Fix circular import.

* Adding more tests and fixing an edge case.

* Add wildcards to local filesystem transfer tests.

* Added tests to ssh.

* Remove unused function.

* Move teardown to fix tests on macos.

* Revert get_rclone_config_name_central name change.

* Add documentation to tests.

* Remove unecessary sorted.

* Fix cyclical import.

* Remove unecessary wildcard.

* Remove unused functions.

Author: JoeZiminski

datashuttle/configs/config_class.py

@@ -250,7 +250,7 @@ def get_base_folder(
     def get_rclone_config_name(
         self, connection_method: Optional[str] = None
     ) -> str:
-        """Generate the rclone configuration name for the project.
+        """Generate the rclone configuration name for the central project.
 
         These configs are created by datashuttle but managed and stored by rclone.
         """

datashuttle/utils/folders.py

@@ -17,11 +17,11 @@
     from datashuttle.configs.config_class import Configs
     from datashuttle.utils.custom_types import TopLevelFolder
 
-import glob
+import fnmatch
 from pathlib import Path
 
 from datashuttle.configs import canonical_folders, canonical_tags
-from datashuttle.utils import rclone, ssh, utils, validation
+from datashuttle.utils import rclone, utils, validation
 from datashuttle.utils.custom_exceptions import NeuroBlueprintError
 
 # -----------------------------------------------------------------------------
@@ -599,56 +599,62 @@ def search_for_folders(
     Discovered folders (`all_folder_names`) and files (`all_filenames`).
 
     """
-    if local_or_central == "central" and cfg["connection_method"] in [
-        "ssh",
-        "gdrive",
-        "aws",
-    ]:
-        if cfg["connection_method"] == "ssh":
-            all_folder_names, all_filenames = (
-                ssh.search_ssh_central_for_folders(
-                    search_path,
-                    search_prefix,
-                    cfg,
-                    verbose,
-                    return_full_path,
-                )
-            )
-
-        else:
-            all_folder_names, all_filenames = search_gdrive_or_aws_for_folders(
-                search_path, search_prefix, cfg, return_full_path
-            )
-
+    if (
+        local_or_central == "local"
+        or cfg["connection_method"] == "local_filesystem"
+    ) and not search_path.exists():
+        if verbose:
+            utils.log_and_message(f"No file found at {search_path.as_posix()}")
+        return [], []
+
+    if local_or_central == "local":
+        rclone_config_name = None
     else:
-        if not search_path.exists():
-            if verbose:
-                utils.log_and_message(
-                    f"No file found at {search_path.as_posix()}"
-                )
-            return [], []
-
-        all_folder_names, all_filenames = search_filesystem_path_for_folders(
-            search_path / search_prefix, return_full_path
+        rclone_config_name = cfg.get_rclone_config_name(
+            cfg["connection_method"]
         )
+
+    all_folder_names, all_filenames = search_local_or_remote(
+        search_path,
+        search_prefix,
+        rclone_config_name,
+        return_full_path,
+    )
+
     return all_folder_names, all_filenames
 
 
-def search_gdrive_or_aws_for_folders(
+def search_local_or_remote(
     search_path: Path,
     search_prefix: str,
-    cfg: Configs,
+    rclone_config_name: str | None,
     return_full_path: bool = False,
 ) -> Tuple[List[Any], List[Any]]:
     """Search for files and folders in central path using `rclone lsjson` command.
 
     This command lists all the files and folders in the central path in a json format.
     The json contains file/folder info about each file/folder like name, type, etc.
+
+    Parameters
+    ----------
+    search_path
+        The path to search (relative to the local or remote drive). For example,
+        for "local_filesystem" this is the path on the local machine. For "ssh", this
+        is the path on the machine that has been connected to.
+    search_prefix
+        The search string e.g. "sub-*".
+    rclone_config_name
+        Name of the rclone config for the remote (not set for local). `rclone config`
+        can be used in the terminal to see how rclone has stored these. In datashuttle,
+        these are managed by `Configs`.
+    return_full_path
+        If `True`, return the full filepath, otherwise return only the folder/file name.
+
     """
+    config_prefix = "" if not rclone_config_name else f"{rclone_config_name}:"
+
     output = rclone.call_rclone(
-        "lsjson "
-        f"{cfg.get_rclone_config_name()}:{search_path.as_posix()} "
-        f'--include "{search_prefix}"',
+        f'lsjson {config_prefix}"{search_path.as_posix()}"',
         pipe_std=True,
     )
 
@@ -657,73 +663,26 @@ def search_gdrive_or_aws_for_folders(
 
     if output.returncode != 0:
         utils.log_and_message(
-            f"Error searching files at {search_path.as_posix()} \n {output.stderr.decode('utf-8') if output.stderr else ''}"
+            f"Error searching files at {search_path.as_posix()}\n"
+            f"{output.stderr.decode('utf-8') if output.stderr else ''}"
         )
         return all_folder_names, all_filenames
 
     files_and_folders = json.loads(output.stdout)
 
-    try:
-        for file_or_folder in files_and_folders:
-            name = file_or_folder["Name"]
-            is_dir = file_or_folder.get("IsDir", False)
-
-            to_append = (
-                (search_path / name).as_posix() if return_full_path else name
-            )
-
-            if is_dir:
-                all_folder_names.append(to_append)
-            else:
-                all_filenames.append(to_append)
-
-    except Exception:
-        utils.log_and_message(
-            f"Error searching files at {search_path.as_posix()}"
-        )
-
-    return all_folder_names, all_filenames
-
-
-# Actual function implementation
-def search_filesystem_path_for_folders(
-    search_path_with_prefix: Path, return_full_path: bool = False
-) -> Tuple[List[Path | str], List[Path | str]]:
-    r"""Search a folder through the local filesystem.
-
-    Use glob to search the full search path (including prefix) with glob.
-    Files are filtered out of results, returning folders only.
-
-    Parameters
-    ----------
-    search_path_with_prefix
-        Path to search along with search prefix e.g. "C:\drive\project\sub-*"
-
-    return_full_path
-        If `True` returns the path to the discovered folder or file,
-        otherwise just the name.
-
-    Returns
-    -------
-    Discovered folders (`all_folder_names`) and files (`all_filenames`).
+    for file_or_folder in files_and_folders:
+        name = file_or_folder["Name"]
 
-    """
-    all_folder_names = []
-    all_filenames = []
+        if not fnmatch.fnmatch(name, search_prefix):
+            continue
 
-    all_files_and_folders = list(glob.glob(search_path_with_prefix.as_posix()))
-    sorter_files_and_folders = sorted(all_files_and_folders)
+        is_dir = file_or_folder.get("IsDir", False)
 
-    for file_or_folder_str in sorter_files_and_folders:
-        file_or_folder = Path(file_or_folder_str)
+        to_append = search_path / name if return_full_path else name
 
-        if file_or_folder.is_dir():
-            all_folder_names.append(
-                file_or_folder if return_full_path else file_or_folder.name
-            )
+        if is_dir:
+            all_folder_names.append(to_append)
         else:
-            all_filenames.append(
-                file_or_folder if return_full_path else file_or_folder.name
-            )
+            all_filenames.append(to_append)
 
     return all_folder_names, all_filenames

datashuttle/utils/formatting.py

@@ -64,7 +64,13 @@ def check_and_format_names(
     names_to_format, reserved_keywords = [], []
     for name in names:
         if name in canonical_reserved_keywords() or tags("*") in name:
-            reserved_keywords.append(name)
+            if tags("to") in name:
+                # handle an edge case where use searches with both tags
+                reserved_keywords += update_names_with_range_to_flag(
+                    [name], prefix
+                )
+            else:
+                reserved_keywords.append(name)
         else:
             names_to_format.append(name)
 

datashuttle/utils/getters.py

@@ -293,9 +293,9 @@ def get_existing_project_paths() -> List[Path]:
     """
     datashuttle_path = canonical_folders.get_datashuttle_path()
 
-    all_folders, _ = folders.search_filesystem_path_for_folders(
-        datashuttle_path / "*"
-    )
+    all_folders = [
+        path_ for path_ in datashuttle_path.glob("*") if path_.is_dir()
+    ]
 
     existing_project_paths = []
     for folder_name in all_folders:

datashuttle/utils/rclone.py

@@ -325,11 +325,7 @@ def check_successful_connection_and_raise_error_on_fail(cfg: Configs) -> None:
     If the command fails, it raises a ConnectionError. The created file is
     deleted thereafter.
     """
-    if cfg["central_path"] is None:
-        tempfile_path = "temp.txt"
-    else:
-        tempfile_path = (cfg["central_path"] / "temp.txt").as_posix()
-
+    tempfile_path = (cfg["central_path"] / "temp.txt").as_posix()
     output = call_rclone(
         f"touch {cfg.get_rclone_config_name()}:{tempfile_path}", pipe_std=True
     )

datashuttle/utils/ssh.py

@@ -5,10 +5,8 @@
 if TYPE_CHECKING:
     from datashuttle.configs.config_class import Configs
 
-import fnmatch
-import stat
 from pathlib import Path
-from typing import Any, List, Optional, Tuple
+from typing import Optional
 
 import paramiko
 
@@ -306,119 +304,3 @@ def verify_ssh_central_host(
             utils.log("Host not accepted. No connection made.")
 
     return success
-
-
-# -----------------------------------------------------------------------------
-# Search over SSH
-# -----------------------------------------------------------------------------
-
-
-def search_ssh_central_for_folders(
-    search_path: Path,
-    search_prefix: str,
-    cfg: Configs,
-    verbose: bool = True,
-    return_full_path: bool = False,
-) -> Tuple[List[Any], List[Any]]:
-    """Search for the search prefix in the search path over SSH.
-
-    Parameters
-    ----------
-    search_path
-        Path to search for folders in.
-
-    search_prefix
-        Search prefix for folder names e.g. "sub-*".
-
-    cfg
-        See connect_client_with_logging().
-
-    verbose
-        If `True`, if a search folder cannot be found, a message
-        will be printed with the un-found path.
-
-    return_full_path
-        include the search_path in the returned paths
-
-    Returns
-    -------
-    Discovered folders (`all_folder_names`) and files (`all_filenames`).
-
-    """
-    client: paramiko.SSHClient
-    with paramiko.SSHClient() as client:
-        connect_client_with_logging(
-            client, cfg, message_on_sucessful_connection=verbose
-        )
-
-        sftp = client.open_sftp()
-
-        all_folder_names, all_filenames = get_list_of_folder_names_over_sftp(
-            sftp,
-            search_path,
-            search_prefix,
-            verbose,
-            return_full_path,
-        )
-
-    return all_folder_names, all_filenames
-
-
-def get_list_of_folder_names_over_sftp(
-    sftp: paramiko.sftp_client.SFTPClient,
-    search_path: Path,
-    search_prefix: str,
-    verbose: bool = True,
-    return_full_path: bool = False,
-) -> Tuple[List[Any], List[Any]]:
-    """Use paramiko's sftp to search a path over ssh for folders.
-
-    Return the folder names.
-
-    Parameters
-    ----------
-    sftp
-        Connected paramiko stfp object
-        (see search_ssh_central_for_folders()).
-
-    search_path
-        Path to search for folders in.
-
-    search_prefix
-        Prefix (can include wildcards)
-        to search folder names.
-
-    verbose
-        If `True`, if a search folder cannot be found, a message
-        will be printed with the un-found path.
-
-    return_full_path
-        include the search_path in the returned paths.
-
-    Returns
-    -------
-    Discovered folders (`all_folder_names`) and files (`all_filenames`).
-
-    """
-    all_folder_names = []
-    all_filenames = []
-    try:
-        for file_or_folder in sftp.listdir_attr(search_path.as_posix()):
-            if file_or_folder.st_mode is not None and fnmatch.fnmatch(
-                file_or_folder.filename, search_prefix
-            ):
-                to_append = (
-                    search_path / file_or_folder.filename
-                    if return_full_path
-                    else file_or_folder.filename
-                )
-                if stat.S_ISDIR(file_or_folder.st_mode):
-                    all_folder_names.append(to_append)
-                else:
-                    all_filenames.append(to_append)
-
-    except FileNotFoundError:
-        if verbose:
-            utils.log_and_message(f"No file found at {search_path.as_posix()}")
-
-    return all_folder_names, all_filenames