From 47d7a712f5634c7fefcfbd83a94527c1a8d61931 Mon Sep 17 00:00:00 2001 From: MoffittAndrew Date: Tue, 18 Mar 2025 22:40:36 +0000 Subject: [PATCH 01/70] Removing type hints from all docstrings --- datashuttle/configs/canonical_configs.py | 3 +- datashuttle/configs/config_class.py | 13 +- datashuttle/configs/load_configs.py | 16 +- datashuttle/datashuttle_class.py | 166 +++++++++++--------- datashuttle/datashuttle_functions.py | 6 +- datashuttle/tui/configs.py | 6 +- datashuttle/tui/custom_widgets.py | 12 +- datashuttle/tui/interface.py | 40 ++--- datashuttle/tui/screens/datatypes.py | 18 ++- datashuttle/tui/screens/modal_dialogs.py | 8 +- datashuttle/tui/screens/new_project.py | 3 +- datashuttle/tui/screens/project_selector.py | 2 +- datashuttle/tui/tabs/create_folders.py | 7 +- datashuttle/tui/tabs/transfer.py | 12 +- datashuttle/utils/data_transfer.py | 21 +-- datashuttle/utils/ds_logger.py | 8 +- datashuttle/utils/folders.py | 82 ++++++---- datashuttle/utils/formatting.py | 26 +-- datashuttle/utils/getters.py | 29 ++-- datashuttle/utils/rclone.py | 47 +++--- datashuttle/utils/ssh.py | 40 +++-- datashuttle/utils/validation.py | 38 ++--- 22 files changed, 342 insertions(+), 261 deletions(-) diff --git a/datashuttle/configs/canonical_configs.py b/datashuttle/configs/canonical_configs.py index b65ad6c66..7213b3d88 100644 --- a/datashuttle/configs/canonical_configs.py +++ b/datashuttle/configs/canonical_configs.py @@ -76,7 +76,8 @@ def check_dict_values_raise_on_fail(config_dict: Configs) -> None: Parameters ---------- - config_dict : datashuttle config UserDict + config_dict + datashuttle config UserDict """ canonical_dict = get_canonical_configs() diff --git a/datashuttle/configs/config_class.py b/datashuttle/configs/config_class.py index 5fd70fcad..7f5a8525c 100644 --- a/datashuttle/configs/config_class.py +++ b/datashuttle/configs/config_class.py @@ -40,10 +40,10 @@ class Configs(UserDict): Parameters ---------- - file_path : + file_path full filepath to save the config .yaml file to. - input_dict : + input_dict a dict of config key-value pairs to input dict. This must contain all canonical_config keys """ @@ -144,9 +144,11 @@ def build_project_path( Parameters ---------- - base: "local", "central" or "datashuttle" + base + "local", "central" or "datashuttle" - sub_folders: a list (or string for 1) of + sub_folders + a list (or string for 1) of folder names to be joined into a path. If file included, must be last entry (with ext). """ @@ -177,7 +179,8 @@ def get_base_folder( Parameters ---------- - base : base path, "local", "central" or "datashuttle" + base + base path, "local", "central" or "datashuttle" """ if base == "local": diff --git a/datashuttle/configs/load_configs.py b/datashuttle/configs/load_configs.py index aac1bc93d..c15e3977d 100644 --- a/datashuttle/configs/load_configs.py +++ b/datashuttle/configs/load_configs.py @@ -25,11 +25,14 @@ def attempt_load_configs( Parameters ---------- - project_name : name of project + project_name + name of project - config_path : path to datashuttle config .yaml file + config_path + path to datashuttle config .yaml file - verbose : warnings and error messages will be printed. + verbose + warnings and error messages will be printed. """ exists = config_path.is_file() @@ -72,8 +75,11 @@ def convert_str_and_pathlib_paths( Parameters ---------- - config_dict : DataShuttle.cfg dict of configs - direction : "path_to_str" or "str_to_path" + config_dict + DataShuttle.cfg dict of configs + + direction + "path_to_str" or "str_to_path" """ for path_key in canonical_configs.keys_str_on_file_but_path_in_class(): value = config_dict[path_key] diff --git a/datashuttle/datashuttle_class.py b/datashuttle/datashuttle_class.py index 173a3f46b..e58b9087f 100644 --- a/datashuttle/datashuttle_class.py +++ b/datashuttle/datashuttle_class.py @@ -90,17 +90,19 @@ class DataShuttle: Parameters ---------- - project_name : The project name to use the datashuttle - Folders containing all project files - and folders are specified in make_config_file(). - Datashuttle-related files are stored in - a .datashuttle folder in the user home - folder. Use get_datashuttle_path() to - see the path to this folder. - - print_startup_message : If `True`, a start-up message displaying the - current state of the program (e.g. persistent - settings such as the 'top-level folder') is shown. + project_name + The project name to use the datashuttle + Folders containing all project files + and folders are specified in make_config_file(). + Datashuttle-related files are stored in + a .datashuttle folder in the user home + folder. Use get_datashuttle_path() to + see the path to this folder. + + print_startup_message + If `True`, a start-up message displaying the + current state of the program (e.g. persistent + settings such as the 'top-level folder') is shown. """ def __init__(self, project_name: str, print_startup_message: bool = True): @@ -164,39 +166,39 @@ def create_folders( Parameters ---------- - top_level_folder : TopLevelFolder + top_level_folder Whether to make the folders in `rawdata` or `derivatives`. - sub_names : Union[str, List[str]] + sub_names subject name / list of subject names to make within the top-level project folder (if not already, these will be prefixed with "sub-") - ses_names : Optional[Union[str, List[str]]] + ses_names (Optional). session name / list of session names. (if not already, these will be prefixed with "ses-"). If no session is provided, no session-level folders are made. - datatype : Union[str, List[str]] + datatype The datatype to make in the sub / ses folders. (e.g. "ephys", "behav", "anat"). If "" is passed no datatype will be created. Broad or Narrow canonical NeuroBlueprint datatypes are accepted. - bypass_validation : bool + bypass_validation If `True`, folders will be created even if they are not valid to NeuroBlueprint style. - log : bool + log If `True`, details of folder creation will be logged. Returns ------- - created_paths : + created_paths A dictionary of the full filepaths made during folder creation, where the keys are the type of folder made and the values are a list of created folder paths (Path objects). If datatype were @@ -343,37 +345,37 @@ def upload_custom( Parameters ---------- - top_level_folder : + top_level_folder The top-level folder (e.g. `"rawdata"`, `"derivatives"`) to transfer files and folders within. - sub_names : + sub_names a subject name / list of subject names. These must be prefixed with "sub-", or the prefix will be automatically added. "@*@" can be used as a wildcard. "all" will search for all sub-folders in the datatype folder to upload. - ses_names : + ses_names a session name / list of session names, similar to sub_names but requiring a "ses-" prefix. - datatype : + datatype The (broad or narrow) NeuroBlueprint datatypes to transfer. If "all", any broad or narrow datatype folder will be transferred. - overwrite_existing_files : + overwrite_existing_files If `False`, files on central will never be overwritten by files transferred from local. If `True`, central files will be overwritten if there is any difference (date, size) between central and local files. - dry_run : + dry_run perform a dry-run of transfer. This will output as if file transfer was taking place, but no files will be moved. Useful to check which files will be moved on data transfer. - init_log : + init_log (Optional). Whether to handle logging. This should always be True, unless logger is handled elsewhere (e.g. in a calling function). @@ -417,37 +419,37 @@ def download_custom( Parameters ---------- - top_level_folder : + top_level_folder The top-level folder (e.g. `rawdata`) to transfer files and folders within. - sub_names : + sub_names a subject name / list of subject names. These must be prefixed with "sub-", or the prefix will be automatically added. "@*@" can be used as a wildcard. "all" will search for all sub-folders in the datatype folder to upload. - ses_names : + ses_names a session name / list of session names, similar to sub_names but requiring a "ses-" prefix. - datatype : + datatype see create_folders() - overwrite_existing_files : + overwrite_existing_files If "never" files on target will never be overwritten by source. If "always" files on target will be overwritten by source if there is any difference in date or size. If "if_source_newer" files on target will only be overwritten by files on source with newer creation / modification datetime. - dry_run : + dry_run perform a dry-run of transfer. This will output as if file transfer was taking place, but no files will be moved. Useful to check which files will be moved on data transfer. - init_log : + init_log (Optional). Whether to handle logging. This should always be True, unless logger is handled elsewhere (e.g. in a calling function). @@ -490,14 +492,14 @@ def upload_rawdata( Parameters ---------- - overwrite_existing_files : + overwrite_existing_files If "never" files on target will never be overwritten by source. If "always" files on target will be overwritten by source if there is any difference in date or size. If "if_source_newer" files on target will only be overwritten by files on source with newer creation / modification datetime. - dry_run : + dry_run perform a dry-run of transfer. This will output as if file transfer was taking place, but no files will be moved. Useful to check which files will be moved on data transfer. @@ -522,14 +524,14 @@ def upload_derivatives( Parameters ---------- - overwrite_existing_files : + overwrite_existing_files If "never" files on target will never be overwritten by source. If "always" files on target will be overwritten by source if there is any difference in date or size. If "if_source_newer" files on target will only be overwritten by files on source with newer creation / modification datetime. - dry_run : + dry_run perform a dry-run of transfer. This will output as if file transfer was taking place, but no files will be moved. Useful to check which files will be moved on data transfer. @@ -554,14 +556,14 @@ def download_rawdata( Parameters ---------- - overwrite_existing_files : + overwrite_existing_files If "never" files on target will never be overwritten by source. If "always" files on target will be overwritten by source if there is any difference in date or size. If "if_source_newer" files on target will only be overwritten by files on source with newer creation / modification datetime. - dry_run : + dry_run perform a dry-run of transfer. This will output as if file transfer was taking place, but no files will be moved. Useful to check which files will be moved on data transfer. @@ -586,14 +588,14 @@ def download_derivatives( Parameters ---------- - overwrite_existing_files : + overwrite_existing_files If "never" files on target will never be overwritten by source. If "always" files on target will be overwritten by source if there is any difference in date or size. If "if_source_newer" files on target will only be overwritten by files on source with newer creation / modification datetime. - dry_run : + dry_run perform a dry-run of transfer. This will output as if file transfer was taking place, but no files will be moved. Useful to check which files will be moved on data transfer. @@ -620,14 +622,14 @@ def upload_entire_project( Parameters ---------- - overwrite_existing_files : + overwrite_existing_files If "never" files on target will never be overwritten by source. If "always" files on target will be overwritten by source if there is any difference in date or size. If "if_source_newer" files on target will only be overwritten by files on source with newer creation / modification datetime. - dry_run : + dry_run perform a dry-run of transfer. This will output as if file transfer was taking place, but no files will be moved. Useful to check which files will be moved on data transfer. @@ -653,14 +655,14 @@ def download_entire_project( Parameters ---------- - overwrite_existing_files : + overwrite_existing_files If "never" files on target will never be overwritten by source. If "always" files on target will be overwritten by source if there is any difference in date or size. If "if_source_newer" files on target will only be overwritten by files on source with newer creation / modification datetime. - dry_run : + dry_run perform a dry-run of transfer. This will output as if file transfer was taking place, but no files will be moved. Useful to check which files will be moved on data transfer. @@ -690,17 +692,17 @@ def upload_specific_folder_or_file( Parameters ---------- - filepath : + filepath a string containing the full filepath. - overwrite_existing_files : + overwrite_existing_files If "never" files on target will never be overwritten by source. If "always" files on target will be overwritten by source if there is any difference in date or size. If "if_source_newer" files on target will only be overwritten by files on source with newer creation / modification datetime. - dry_run : + dry_run perform a dry-run of transfer. This will output as if file transfer was taking place, but no files will be moved. Useful to check which files will be moved on data transfer. @@ -732,17 +734,17 @@ def download_specific_folder_or_file( Parameters ---------- - filepath : + filepath a string containing the full filepath. - overwrite_existing_files : + overwrite_existing_files If "never" files on target will never be overwritten by source. If "always" files on target will be overwritten by source if there is any difference in date or size. If "if_source_newer" files on target will only be overwritten by files on source with newer creation / modification datetime. - dry_run : + dry_run perform a dry-run of transfer. This will output as if file transfer was taking place, but no files will be moved. Useful to check which files will be moved on data transfer. @@ -879,7 +881,7 @@ def write_public_key(self, filepath: str) -> None: Parameters ---------- - filepath : + filepath full filepath (inc filename) to write the public key to. """ @@ -920,10 +922,10 @@ def make_config_file( Parameters ---------- - local_path : + local_path path to project folder on local machine - central_path : + central_path Filepath to central project. If this is local (i.e. connection_method = "local_filesystem"), this is the full path on the local filesystem @@ -933,15 +935,15 @@ def make_config_file( include ~ home folder syntax, must contain the full path (e.g. /nfs/nhome/live/jziminski) - connection_method : + connection_method The method used to connect to the central project filesystem, e.g. "local_filesystem" (e.g. mounted drive) or "ssh" - central_host_id : + central_host_id server address for central host for ssh connection e.g. "ssh.swc.ucl.ac.uk" - central_host_username : + central_host_username username for which to log in to central host. e.g. "jziminski" """ @@ -1081,10 +1083,10 @@ def get_next_sub( Parameters ---------- - return_with_prefix : bool + return_with_prefix If `True`, return with the "sub-" prefix. - include_central : bool + include_central If `False, only get names from `local_path`, otherwise from `local_path` and `central_path`. If in local-project mode, this flag is ignored. @@ -1122,16 +1124,16 @@ def get_next_ses( Parameters ---------- - top_level_folder: + top_level_folder "rawdata" or "derivatives" - sub: Optional[str] + sub Name of the subject to find the next session of. - return_with_prefix : bool + return_with_prefix If `True`, return with the "ses-" prefix. - include_central : bool + include_central If `False, only get names from `local_path`, otherwise from `local_path` and `central_path`. If in local-project mode, this flag is ignored. @@ -1173,7 +1175,7 @@ def get_name_templates(self) -> Dict: Returns ------- - name_templates : Dict + name_templates e.g. {"name_templates": {"on": False, "sub": None, "ses": None}} """ settings = self._load_persistent_settings() @@ -1189,7 +1191,7 @@ def set_name_templates(self, new_name_templates: Dict) -> None: Parameters ---------- - new_name_templates : Dict + new_name_templates e.g. {"name_templates": {"on": False, "sub": None, "ses": None}} where "sub" or "ses" can be a regexp that subject and session names respectively are validated against. @@ -1227,20 +1229,20 @@ def validate_project( Parameters ---------- - top_level_folder : TopLevelFolder | None + top_level_folder Folder to check, either "rawdata" or "derivatives". If ``None``, will check both folders. - display_mode : DisplayMode + display_mode The validation issues are displayed as ``"error"`` (raise error) ``"warn"`` (show warning) or ``"print"`` - include_central : bool + include_central If ``False``, only the local project is validated. Otherwise, both local and central projects are validated. If in local-project mode, this flag is ignored. - strict_mode: bool + strict_mode If `True`, only allow NeuroBlueprint-formatted folders to exist in the project. By default, non-NeuroBlueprint folders (e.g. a folder called 'my_stuff' in the 'rawdata') are allowed, and only folders @@ -1293,9 +1295,10 @@ def check_name_formatting(names: Union[str, list], prefix: Prefix) -> None: Parameters ---------- - names : + names A string or list of subject or session names. - prefix: + + prefix The relevant subject or session prefix, e.g. "sub-" or "ses-" """ @@ -1328,8 +1331,9 @@ def _transfer_entire_project( Parameters ---------- - upload_or_download : direction to transfer the data, either "upload" (from - local to central) or "download" (from central to local). + upload_or_download + direction to transfer the data, either "upload" (from + local to central) or "download" (from central to local). """ for top_level_folder in canonical_folders.get_top_level_folders(): @@ -1358,12 +1362,14 @@ def _start_log( Parameters ---------- - command_name : name of the command, for the log output files. + command_name + name of the command, for the log output files. - local_vars : local_vars are passed to fancylog variables argument. - see ds_logger.wrap_variables_for_fancylog for more info + local_vars + local_vars are passed to fancylog variables argument. + see ds_logger.wrap_variables_for_fancylog for more info - store_in_temp_folder : + store_in_temp_folder if `False`, existing logging path will be used (local project .datashuttle). """ @@ -1476,8 +1482,12 @@ def _update_persistent_setting( Parameters ---------- - setting_name : dictionary key of the persistent setting to change - setting_value : value to change the persistent setting to + + setting_name + dictionary key of the persistent setting to change + + setting_value + value to change the persistent setting to """ settings = self._load_persistent_settings() diff --git a/datashuttle/datashuttle_functions.py b/datashuttle/datashuttle_functions.py index 94868f7bc..8c220b536 100644 --- a/datashuttle/datashuttle_functions.py +++ b/datashuttle/datashuttle_functions.py @@ -40,15 +40,15 @@ def quick_validate_project( Path to the project to validate. Must include the project name, and hold a "rawdata" or "derivatives" folder. - top_level_folder : TopLevelFolder + top_level_folder The top-level folder ("rawdata" or "derivatives") to perform validation. If `None`, both are checked. - display_mode : DisplayMode + display_mode The validation issues are displayed as ``"error"`` (raise error) ``"warn"`` (show warning) or ``"print"``. - name_templates : Dict + name_templates A dictionary of templates for subject and session name to validate against. See ``DataShuttle.set_name_templates()`` for details. diff --git a/datashuttle/tui/configs.py b/datashuttle/tui/configs.py index 974ee08af..c16b1a425 100644 --- a/datashuttle/tui/configs.py +++ b/datashuttle/tui/configs.py @@ -299,7 +299,7 @@ def switch_ssh_widgets_display(self, display_ssh: bool) -> None: Parameters ---------- - display_ssh : bool + display_ssh If `True`, display the SSH-related widgets. """ for widget in self.config_ssh_widgets: @@ -373,11 +373,11 @@ def handle_input_fill_from_select_directory( Parameters ---------- - path_ : Union[Literal[False], Path] + path_ The path returned from `SelectDirectoryTreeScreen`. If `False`, the screen exited with no directory selected. - local_or_central : str + local_or_central The Input to fill with the path. """ if path_ is False: diff --git a/datashuttle/tui/custom_widgets.py b/datashuttle/tui/custom_widgets.py index dc7f58887..5f5559e2e 100644 --- a/datashuttle/tui/custom_widgets.py +++ b/datashuttle/tui/custom_widgets.py @@ -316,13 +316,13 @@ def handle_fill_input_from_directorytree( Parameters ---------- - sub_input_key : str + sub_input_key The textual widget id for the subject input (prefixed with #) - ses_input_key : str + ses_input_key The textual widget id for the session input (prefixed with #) - event : DirectoryTreeSpecialKeyPress + event A DirectoryTreeSpecialKeyPress event triggered from the CustomDirectoryTree. """ @@ -346,7 +346,7 @@ def insert_sub_or_ses_name_to_input( see `handle_directorytree_key_pressed` for `sub_input_key` and `ses_input_key`. - name : str + name The sub or ses name to append to the input. """ if name.startswith("sub-"): @@ -399,12 +399,12 @@ class TopLevelFolderSelect(Select): Parameters ---------- - existing_only : bool + existing_only If `True`, only top level folders that actually exist in the project are displayed. Otherwise, all possible canonical top-level-folders are displayed. - id : str + id Textualize widget id """ diff --git a/datashuttle/tui/interface.py b/datashuttle/tui/interface.py index 26ee2f8ba..fd3e7b2c4 100644 --- a/datashuttle/tui/interface.py +++ b/datashuttle/tui/interface.py @@ -43,7 +43,7 @@ def select_existing_project(self, project_name: str) -> InterfaceOutput: Parameters ---------- - project_name : str + project_name The name of the datashuttle project to load. Must already exist. """ @@ -64,10 +64,10 @@ def setup_new_project( Parameters ---------- - project_name : str + project_name Name of the project to set up. - cfg_kwargs : Dict + cfg_kwargs The configurations to set the new project to. """ try: @@ -92,7 +92,7 @@ def set_configs_on_existing_project( Parameters ---------- - cfg_kwargs : Dict + cfg_kwargs The configs and new values to update. """ try: @@ -114,13 +114,13 @@ def create_folders( Parameters ---------- - sub_names : List[str] + sub_names A list of un-formatted / unvalidated subject names to create. - ses_names : List[str] + ses_names A list of un-formatted / unvalidated session names to create. - datatype : List[str] + datatype A list of canonical datatype names to create. """ top_level_folder = self.tui_settings["top_level_folder_select"][ @@ -154,10 +154,10 @@ def validate_names( Parameters ---------- - sub_names : List[str] + sub_names List of subject names to format. - ses_names : List[str] + ses_names List of session names to format. """ top_level_folder = self.tui_settings["top_level_folder_select"][ @@ -191,7 +191,7 @@ def transfer_entire_project(self, upload: bool) -> InterfaceOutput: Parameters ---------- - upload : bool + upload Upload from local to central if `True`, otherwise download from central to remote. """ @@ -222,10 +222,10 @@ def transfer_top_level_only( Parameters ---------- - selected_top_level_folder : str + selected_top_level_folder The top level folder selected in the TUI for this transfer window. - upload : bool + upload Upload from local to central if `True`, otherwise download from central to remote. @@ -272,19 +272,19 @@ def transfer_custom_selection( Parameters ---------- - selected_top_level_folder : str + selected_top_level_folder The top level folder selected in the TUI for this transfer window. - sub_names : List[str] + sub_names Subject names or subject-level canonical transfer keys to transfer. - ses_names : List[str] + ses_names Session names or session-level canonical transfer keys to transfer. - datatype : List[str] + datatype Datatypes or datatype-level canonical transfer keys to transfer. - upload : bool + upload Upload from local to central if `True`, otherwise download from central to remote. """ @@ -360,14 +360,14 @@ def save_tui_settings( Parameters ---------- - value : Any + value Value to set the `persistent_settings` tui field to - key_1 : str + key_1 First key of the tui `persistent_settings` to update e.g. "top_level_folder_select" - key_2 : str + key_2 Optionals second level of the dictionary to update. e.g. "create_tab" """ diff --git a/datashuttle/tui/screens/datatypes.py b/datashuttle/tui/screens/datatypes.py index 790cca553..aa731761d 100644 --- a/datashuttle/tui/screens/datatypes.py +++ b/datashuttle/tui/screens/datatypes.py @@ -189,18 +189,20 @@ class DatatypeCheckboxes(Static): Parameters ---------- - settings_key : 'create' if datatype checkboxes for the create tab, - 'transfer' for the transfer tab. Transfer tab includes - additional datatype options (e.g. "all"). + settings_key + 'create' if datatype checkboxes for the create tab, + 'transfer' for the transfer tab. Transfer tab includes + additional datatype options (e.g. "all"). Attributes ---------- - datatype_config : a Dictionary containing datatype as key (e.g. "ephys", "behav") - and values are `bool` indicating whether the checkbox is on / off. - If 'transfer', then transfer datatype arguments (e.g. "all") - are also included. This structure mirrors - the `persistent_settings` dictionaries. + datatype_config + a Dictionary containing datatype as key (e.g. "ephys", "behav") + and values are `bool` indicating whether the checkbox is on / off. + If 'transfer', then transfer datatype arguments (e.g. "all") + are also included. This structure mirrors + the `persistent_settings` dictionaries. Notes ----- diff --git a/datashuttle/tui/screens/modal_dialogs.py b/datashuttle/tui/screens/modal_dialogs.py index 97706db07..4d801e823 100644 --- a/datashuttle/tui/screens/modal_dialogs.py +++ b/datashuttle/tui/screens/modal_dialogs.py @@ -26,10 +26,10 @@ class MessageBox(ModalScreen): """ A screen for rendering error messages. - message : str + message The message to display in the message box - border_color : str + border_color The color to pass to the `border` style on the widget. Note that the keywords 'red' 'grey' 'green' are overridden for custom style. """ @@ -147,10 +147,10 @@ class SelectDirectoryTreeScreen(ModalScreen): Parameters ---------- - mainwindow : App + mainwindow Textual main app screen - path_ : Optional[Path] + path_ Path to use as the DirectoryTree root, if `None` set to the system user home. """ diff --git a/datashuttle/tui/screens/new_project.py b/datashuttle/tui/screens/new_project.py index 43232e17e..ea7c0826c 100644 --- a/datashuttle/tui/screens/new_project.py +++ b/datashuttle/tui/screens/new_project.py @@ -31,7 +31,8 @@ class NewProjectScreen(Screen): Parameters ---------- - mainwindow : TuiApp + mainwindow + The main TUI app """ TITLE = "Make New Project" diff --git a/datashuttle/tui/screens/project_selector.py b/datashuttle/tui/screens/project_selector.py index 43f9858ab..289990e5c 100644 --- a/datashuttle/tui/screens/project_selector.py +++ b/datashuttle/tui/screens/project_selector.py @@ -29,7 +29,7 @@ class ProjectSelectorScreen(Screen): Parameters ---------- - mainwindow : TuiApp + mainwindow The main TUI app, functions on which are used to coordinate screen display. diff --git a/datashuttle/tui/tabs/create_folders.py b/datashuttle/tui/tabs/create_folders.py index 6907f684c..e596566da 100644 --- a/datashuttle/tui/tabs/create_folders.py +++ b/datashuttle/tui/tabs/create_folders.py @@ -286,10 +286,10 @@ def fill_input_with_next_sub_or_ses_template( Parameters - prefix : Prefix + prefix Whether to fill the subject or session Input - input_id : str + input_id The textual input name to update. """ top_level_folder = self.interface.tui_settings[ @@ -367,7 +367,8 @@ def run_local_validation(self, prefix: Prefix): Parameters ---------- - prefix : Prefix + prefix + Whether to run validation on the subject or session Input """ sub_names = self.query_one( "#create_folders_subject_input" diff --git a/datashuttle/tui/tabs/transfer.py b/datashuttle/tui/tabs/transfer.py index cd9ba9c4b..4603c6674 100644 --- a/datashuttle/tui/tabs/transfer.py +++ b/datashuttle/tui/tabs/transfer.py @@ -52,20 +52,22 @@ class TransferTab(TreeAndInputTab): Parameters ---------- - title : str + title + The title of the tab - mainwindow : App + mainwindow + The main TUI app - interface : Interface + interface TUI-datashuttle interface object - id : str + id The textual widget id. Attributes ---------- - show_legend : bool + show_legend Convenience attribute linked to a global setting exists that turns off / on styling of directorytree nodes based on transfer status. ` diff --git a/datashuttle/utils/data_transfer.py b/datashuttle/utils/data_transfer.py index 21121b8e6..a39e6377f 100644 --- a/datashuttle/utils/data_transfer.py +++ b/datashuttle/utils/data_transfer.py @@ -24,38 +24,39 @@ class TransferData: Parameters ---------- - cfg : Configs, + cfg datashuttle configs UserDict. - upload_or_download : Literal["upload", "download"] + upload_or_download Direction to perform the transfer. - top_level_folder: TopLevelFolder + top_level_folder + The top-level folder structure where data is organized. - sub_names : Union[str, List[str]] + sub_names List of subject names or single subject to transfer. This can include transfer keywords (e.g. "all_non_sub"). - ses_names : Union[str, List[str]] + ses_names List of sessions or single session to transfer, for each subject. May include session-level transfer keywords. - datatype : Union[str, List[str]] + datatype List of datatypes to transfer, for the sessions / subjects specified. Can include datatype-level tranfser keywords. - overwrite_existing_files : OverwriteExistingFiles + overwrite_existing_files If "never" files on target will never be overwritten by source. If "always" files on target will be overwritten by source if there is any difference in date or size. If "if_source_newer" files on target will only be overwritten by files on source with newer creation / modification datetime. - dry_run : bool, + dry_run If `True`, transfer will not actually occur but will be logged as if it did (to see what would happen for a transfer). - log : bool, + log if `True`, log and print the transfer output. """ @@ -123,7 +124,7 @@ def build_a_list_of_all_files_and_folders_to_transfer(self) -> List[str]: Returns ------- - include_list : List[str] + include_list A list of paths to pass to rclone's `--include` flag. """ # Find sub names to transfer diff --git a/datashuttle/utils/ds_logger.py b/datashuttle/utils/ds_logger.py index ca1c6bf94..3d0ac7b69 100644 --- a/datashuttle/utils/ds_logger.py +++ b/datashuttle/utils/ds_logger.py @@ -76,10 +76,12 @@ def log_names(list_of_headers: List[Any], list_of_names: List[Any]) -> None: Parameters ---------- - list_of_headers : a list of titles that the names - will be printed under, e.g. "sub_names", "ses_names" + list_of_headers + a list of titles that the names + will be printed under, e.g. "sub_names", "ses_names" - list_of_names : list of names to print to log + list_of_names + list of names to print to log """ for header, names in zip(list_of_headers, list_of_names): utils.log(f"{header}: {names}") diff --git a/datashuttle/utils/folders.py b/datashuttle/utils/folders.py index 56852640c..03cfb38af 100644 --- a/datashuttle/utils/folders.py +++ b/datashuttle/utils/folders.py @@ -49,9 +49,11 @@ def create_folder_trees( Parameters ---------- - sub_names, ses_names, datatype : see create_folders() + sub_names, ses_names, datatype + see create_folders() - log : whether to log or not. If True, logging must + log + whether to log or not. If True, logging must already be initialised. """ datatype_passed = datatype not in [[""], ""] @@ -128,22 +130,28 @@ def make_datatype_folders( Parameters ---------- - cfg : ConfigsClass + cfg + datashuttle configs - datatype : datatype (e.g. "behav", "all") to use. Use + datatype + datatype (e.g. "behav", "all") to use. Use empty string ("") for none. - sub_or_ses_level_path : Full path to the subject + sub_or_ses_level_path + Full path to the subject or session folder where the new folder will be written. - level : The folder level that the + level + The folder level that the folder will be made at, "sub" or "ses" - save_paths : A dictionary, which will be filled + save_paths + A dictionary, which will be filled with created paths split by datatype name. - log : whether to log on or not (if True, logging must + log + whether to log on or not (if True, logging must already be initialised). """ datatype_items = cfg.get_datatype_as_dict_items(datatype) @@ -175,9 +183,11 @@ def create_folders(paths: Union[Path, List[Path]], log: bool = True) -> None: Parameters ---------- - paths : Path or list of Paths to create + paths + Path or list of Paths to create - log : if True, log all made folders. This + log + if True, log all made folders. This requires the logger to already be initialised. """ if isinstance(paths, Path): @@ -383,19 +393,24 @@ def search_for_wildcards( Parameters ---------- - project : initialised datashuttle project + project + initialised datashuttle project - base_folder : folder to search for wildcards in + base_folder + folder to search for wildcards in - local_or_central : "local" or "central" project path to + local_or_central + "local" or "central" project path to search in - all_names : list of subject or session names that + all_names + list of subject or session names that may or may not include the wildcard flag. If sub (below) is passed, it is assumed these are session names. Otherwise, it is assumed these are subject names. - sub : optional subject to search for sessions in. If not provided, + sub + optional subject to search for sessions in. If not provided, will search for subjects rather than sessions. """ @@ -448,26 +463,32 @@ def search_sub_or_ses_level( Parameters ---------- - cfg : datashuttle project cfg. Currently, this is used + cfg + datashuttle project cfg. Currently, this is used as a holder for ssh configs to avoid too many arguments, but this is not nice and breaks the general rule that these functions should operate project-agnostic. - local_or_central : search in local or central project + local_or_central + search in local or central project - sub : either a subject name (string) or None. If None, the search + sub + either a subject name (string) or None. If None, the search is performed at the top_level_folder level - ses : either a session name (string) or None, This must not + ses + either a session name (string) or None, This must not be a session name if sub is None. If provided (with sub) then the session folder is searched - str : glob-format search string to search at the + str + glob-format search string to search at the folder level. - verbose : If `True`, if a search folder cannot be found, a message - will be printed with the un-found path. + verbose + If `True`, if a search folder cannot be found, a message + will be printed with the un-found path. """ if ses and not sub: utils.log_and_raise_error( @@ -509,11 +530,18 @@ def search_for_folders( Parameters ---------- - local_or_central : "local" or "central" - search_path : full filepath to search in - search_prefix : file / folder name to search (e.g. "sub-*") - verbose : If `True`, when a search folder cannot be found, a message - will be printed with the missing path. + local_or_central + "local" or "central" + + search_path + full filepath to search in + + search_prefix + file / folder name to search (e.g. "sub-*") + + verbose + If `True`, when a search folder cannot be found, a message + will be printed with the missing path. """ if local_or_central == "central" and cfg["connection_method"] == "ssh": all_folder_names, all_filenames = ssh.search_ssh_central_for_folders( diff --git a/datashuttle/utils/formatting.py b/datashuttle/utils/formatting.py index 1cebf49d0..d5cee1d29 100644 --- a/datashuttle/utils/formatting.py +++ b/datashuttle/utils/formatting.py @@ -39,18 +39,18 @@ def check_and_format_names( Parameters ---------- - names : Union[list, str] + names str or list containing sub or ses names (e.g. to create folders) - prefix : Prefix + prefix "sub" or "ses" - this defines the prefix checks. - name_templates : Dict + name_templates A dictionary of templates to validate subject and session name against. e.g. {"name_templates": {"on": False, "sub": None, "ses": None}} where the "sub" and "ses" may contain a regexp to validate against. - bypass_validation : Dict + bypass_validation If `True`, NeuroBlueprint validation will be performed on the passed names. """ @@ -89,9 +89,11 @@ def format_names(names: List, prefix: Prefix) -> List[str]: Parameters ----------- - names: str or list containing sub or ses names (e.g. to make folders) + names + str or list containing sub or ses names (e.g. to make folders) - prefix: "sub" or "ses" - this defines the prefix checks. + prefix + "sub" or "ses" - this defines the prefix checks. """ assert prefix in ["sub", "ses"], "`prefix` must be 'sub' or 'ses'." @@ -198,13 +200,17 @@ def make_list_of_zero_padded_names_across_range( Parameters ---------- - left_number : left (start) number from the range, e.g. "001" + left_number + left (start) number from the range, e.g. "001" - right_number : right (end) number from the range, e.g. "005" + right_number + right (end) number from the range, e.g. "005" - name_start_str : part of the name before the flag, usually "sub-" + name_start_str + part of the name before the flag, usually "sub-" - name_end_str : rest of the name after the flag, i.e. all other + name_end_str + rest of the name after the flag, i.e. all other key-value pairs. """ max_leading_zeros = max( diff --git a/datashuttle/utils/getters.py b/datashuttle/utils/getters.py index 6ef64870b..1f55465bd 100644 --- a/datashuttle/utils/getters.py +++ b/datashuttle/utils/getters.py @@ -50,29 +50,29 @@ def get_next_sub_or_ses( Parameters ---------- - cfg : Configs + cfg datashuttle configs class - top_level_folder: TopLevelFolder + top_level_folder The top-level folder (e.g. `"rawdata"`, `"derivatives"`) - sub : Optional[str] + sub subject name to search within if searching for sessions, otherwise None to search for subjects - search_str : str + search_str the string to search for within the top-level or subject-level folder ("sub-*") or ("ses-*") are suggested, respectively. - include_central : bool + include_central If `False, only get names from `local_path`, otherwise from `local_path` and `central_path`. - return_with_prefix : bool + return_with_prefix If `True`, the next sub or ses value will include the prefix e.g. "sub-001", otherwise the value alone will be returned (e.g. "001") - default_num_value_digits : int + default_num_value_digits If no sub or ses exist in the project, the starting number is 1. Because the number of digits for the project is not accessible, the desired value can be entered here. e.g. if 3 (the default), @@ -80,7 +80,8 @@ def get_next_sub_or_ses( Returns ------- - suggested_new_num : the new suggested sub / ses. + suggested_new_num + the new suggested sub / ses. """ prefix: Prefix @@ -131,7 +132,7 @@ def get_max_sub_or_ses_num_and_value_length( Parameters ---------- - all_folders : List[str] + all_folders A list of BIDS-style formatted folder names. see `get_next_sub_or_ses()` for other arguments. @@ -139,10 +140,10 @@ def get_max_sub_or_ses_num_and_value_length( Returns ------- - max_existing_num : int + max_existing_num The largest number sub / ses value in the past list. - num_value_digits : int + num_value_digits The length of the value in all sub / ses values within the passed list. If these are not consistent, an error is raised. @@ -310,13 +311,13 @@ def get_all_sub_and_ses_paths( Parameters ---------- - cfg : Configs + cfg datashuttle Configs - top_level_folder: TopLevelFolder + top_level_folder The top-level folder (e.g. `"rawdata"`, `"derivatives"`) - include_central : bool + include_central If `False, only get names from `local_path`, otherwise from `local_path` and `central_path`. """ diff --git a/datashuttle/utils/rclone.py b/datashuttle/utils/rclone.py index 9644c103c..d754e47ee 100644 --- a/datashuttle/utils/rclone.py +++ b/datashuttle/utils/rclone.py @@ -15,9 +15,11 @@ def call_rclone(command: str, pipe_std: bool = False) -> CompletedProcess: Parameters ---------- - command: Rclone command to be run + command + Rclone command to be run - pipe_std: if True, do not output anything to stdout. + pipe_std + if True, do not output anything to stdout. """ command = "rclone " + command if pipe_std: @@ -55,11 +57,12 @@ def setup_rclone_config_for_local_filesystem( Parameters ---------- - rclone_config_name : rclone config name - canonical config name, generated by - datashuttle.cfg.get_rclone_config_name() + rclone_config_name + canonical config name, generated by + datashuttle.cfg.get_rclone_config_name() - log : whether to log, if True logger must already be initialised. + log + whether to log, if True logger must already be initialised. """ call_rclone(f"config create {rclone_config_name} local", pipe_std=True) @@ -81,17 +84,19 @@ def setup_rclone_config_for_ssh( Parameters ---------- - cfg : Configs - datashuttle configs UserDict. + cfg + datashuttle configs UserDict. - rclone_config_name : rclone config name - canonical config name, generated by - datashuttle.cfg.get_rclone_config_name() + rclone_config_name + canonical config name, generated by + datashuttle.cfg.get_rclone_config_name() - ssh_key_path : path to the ssh key used for connecting to - ssh central filesystem, + ssh_key_path + path to the ssh key used for connecting to + ssh central filesystem - log : whether to log, if True logger must already be initialised. + log + whether to log, if True logger must already be initialised. """ call_rclone( f"config create " @@ -158,20 +163,20 @@ def transfer_data( Parameters ---------- - cfg: Configs + cfg datashuttle configs - upload_or_download : Literal["upload", "download"] + upload_or_download If "upload", transfer from `local_path` to `central_path`. "download" proceeds in the opposite direction. - top_level_folder: Literal["rawdata", "derivatives"] + top_level_folder The top-level-folder to transfer files within. - include_list : List[str] + include_list A list of filepaths to include in the transfer - rclone_options : Dict + rclone_options A list of options to pass to Rclone's copy function. see `cfg.make_rclone_transfer_options()`. """ @@ -222,13 +227,13 @@ def get_local_and_central_file_differences( Parameters ---------- - top_level_folders_to_check : + top_level_folders_to_check List of top-level folders to check. Returns ------- - parsed_output : Dict[str, List] + parsed_output A dictionary where the keys are the cases (e.g. "same" across local and central) and the values are lists of paths that fall into these cases. Note the paths are relative to the "rawdata" diff --git a/datashuttle/utils/ssh.py b/datashuttle/utils/ssh.py index 8f6de6785..eb8025bd2 100644 --- a/datashuttle/utils/ssh.py +++ b/datashuttle/utils/ssh.py @@ -117,15 +117,19 @@ def setup_ssh_key( Parameters ----------- - ssh_key_path : path to the ssh private key + ssh_key_path + path to the ssh private key - hostkeys_path : path to the ssh host key, once the user + hostkeys_path + path to the ssh host key, once the user has confirmed the key ID this is saved so verification is not required each time. - cfg : datashuttle config UserDict + cfg + datashuttle config UserDict - log : log if True, logger must already be initialised. + log + log if True, logger must already be initialised. """ if not sys.stdin.isatty(): proceed = input( @@ -253,14 +257,18 @@ def search_ssh_central_for_folders( Parameters ----------- - search_path : path to search for folders in + search_path + path to search for folders in - search_prefix : search prefix for folder names e.g. "sub-*" + search_prefix + search prefix for folder names e.g. "sub-*" - cfg : see connect_client_with_logging() + cfg + see connect_client_with_logging() - verbose : If `True`, if a search folder cannot be found, a message - will be printed with the un-found path. + verbose + If `True`, if a search folder cannot be found, a message + will be printed with the un-found path. """ client: paramiko.SSHClient with paramiko.SSHClient() as client: @@ -295,16 +303,20 @@ def get_list_of_folder_names_over_sftp( Parameters ---------- - stfp : connected paramiko stfp object + stfp + connected paramiko stfp object (see search_ssh_central_for_folders()) - search_path : path to search for folders in + search_path + path to search for folders in - search_prefix : prefix (can include wildcards) + search_prefix + prefix (can include wildcards) to search folder names. - verbose : If `True`, if a search folder cannot be found, a message - will be printed with the un-found path. + verbose + If `True`, if a search folder cannot be found, a message + will be printed with the un-found path. """ all_folder_names = [] all_filenames = [] diff --git a/datashuttle/utils/validation.py b/datashuttle/utils/validation.py index e85d757da..05f58f6f3 100644 --- a/datashuttle/utils/validation.py +++ b/datashuttle/utils/validation.py @@ -144,16 +144,16 @@ def validate_list_of_names( Parameters ---------- - path_or_name_list : List[Path] + path_or_name_list A list of pathlib.Path to NeuroBlueprint-formatted folders to validate - prefix: Prefix + prefix Whether these are subject (sub) or session (ses) level names - name_templates: Optional[Dict] + name_templates A `name_template` dictionary to validate against. See `set_name_templates()`. - check_value_lengths : bool + check_value_lengths If `True`, check that the prefix- value lengths are consistent across the passed list. """ @@ -507,27 +507,27 @@ def validate_project( Parameters ----------- - cfg : Configs + cfg datashuttle Configs class. - top_level_folder_list: List[TopLevelFolder] + top_level_folder_list The top level folders to validate. - include_central : bool + include_central If `False`, only project folders in the `local_path` will be validated. Otherwise, project folders in both the `local_path` and `central_path` will be validated. - display_mode : DisplayMode + display_mode Determine whether error or warning is raised. - log : bool + log If `True`, errors or warnings are logged to "datashuttle" logger. - name_templates: Optional[Dict] + name_templates A `name_template` dictionary to validate against. See `set_name_templates()`. - strict_mode: bool + strict_mode If `True`, only allow NeuroBlueprint-formatted folders to exist in the project. By default, non-NeuroBlueprint folders (e.g. a folder called 'my_stuff' in the 'rawdata') are allowed, and only folders @@ -616,34 +616,34 @@ def validate_names_against_project( Parameters ---------- - cfg : Configs + cfg datashuttle Configs class. - top_level_folder : TopLevelFolder + top_level_folder The top level folder to validate - sub_names : List[str] + sub_names A list of subject-level names to validate against the subject names that exist in the project. - ses_names : List[str] + ses_names A list of session-level names to validate against the session names that exist in the project. Note that duplicate checks will only be performed for sessions within the passed `sub_names`. - include_central : bool + include_central If `True`, only project folders in the `local_path` will be validated against. Otherwise, project folders in both the `local_path` and `central_path` will be validated against. - display_mode : DisplayMode + display_mode Determine whether error or warning is raised. - log : bool + log If `True`, errors or warnings are logged to "datashuttle" logger. - name_templates: Optional[Dict] + name_templates A `name_template` dictionary to validate against. See `set_name_templates()`. """ error_messages = [] From 0fa4b7f2f54d6da89ca828a230fa296a9eb47794 Mon Sep 17 00:00:00 2001 From: MoffittAndrew Date: Tue, 18 Mar 2025 23:16:10 +0000 Subject: [PATCH 02/70] formatting parameter headings --- datashuttle/datashuttle_class.py | 1 + datashuttle/tui/tabs/create_folders.py | 1 + datashuttle/utils/folders.py | 1 + datashuttle/utils/formatting.py | 1 + datashuttle/utils/getters.py | 1 + datashuttle/utils/rclone.py | 11 ++++++----- 6 files changed, 11 insertions(+), 5 deletions(-) diff --git a/datashuttle/datashuttle_class.py b/datashuttle/datashuttle_class.py index e58b9087f..ad757ac38 100644 --- a/datashuttle/datashuttle_class.py +++ b/datashuttle/datashuttle_class.py @@ -1191,6 +1191,7 @@ def set_name_templates(self, new_name_templates: Dict) -> None: Parameters ---------- + new_name_templates e.g. {"name_templates": {"on": False, "sub": None, "ses": None}} where "sub" or "ses" can be a regexp that subject and session diff --git a/datashuttle/tui/tabs/create_folders.py b/datashuttle/tui/tabs/create_folders.py index e596566da..1e3ef7de8 100644 --- a/datashuttle/tui/tabs/create_folders.py +++ b/datashuttle/tui/tabs/create_folders.py @@ -285,6 +285,7 @@ def fill_input_with_next_sub_or_ses_template( will be suggested. Parameters + ---------- prefix Whether to fill the subject or session Input diff --git a/datashuttle/utils/folders.py b/datashuttle/utils/folders.py index 03cfb38af..33d470088 100644 --- a/datashuttle/utils/folders.py +++ b/datashuttle/utils/folders.py @@ -130,6 +130,7 @@ def make_datatype_folders( Parameters ---------- + cfg datashuttle configs diff --git a/datashuttle/utils/formatting.py b/datashuttle/utils/formatting.py index d5cee1d29..12e9a6027 100644 --- a/datashuttle/utils/formatting.py +++ b/datashuttle/utils/formatting.py @@ -89,6 +89,7 @@ def format_names(names: List, prefix: Prefix) -> List[str]: Parameters ----------- + names str or list containing sub or ses names (e.g. to make folders) diff --git a/datashuttle/utils/getters.py b/datashuttle/utils/getters.py index 1f55465bd..8e1e8cd95 100644 --- a/datashuttle/utils/getters.py +++ b/datashuttle/utils/getters.py @@ -50,6 +50,7 @@ def get_next_sub_or_ses( Parameters ---------- + cfg datashuttle configs class diff --git a/datashuttle/utils/rclone.py b/datashuttle/utils/rclone.py index d754e47ee..0462af2b7 100644 --- a/datashuttle/utils/rclone.py +++ b/datashuttle/utils/rclone.py @@ -15,6 +15,7 @@ def call_rclone(command: str, pipe_std: bool = False) -> CompletedProcess: Parameters ---------- + command Rclone command to be run @@ -77,12 +78,12 @@ def setup_rclone_config_for_ssh( log: bool = True, ): """ - RClone sets remote targets in a config file that are - used at transfer. For SSH, this must contain the central path, - username and ssh key. The relative path is supplied at transfer time. + RClone sets remote targets in a config file that are + used at transfer. For SSH, this must contain the central path, + username and ssh key. The relative path is supplied at transfer time. - Parameters - ---------- + Parameters + ---------- cfg datashuttle configs UserDict. From 6c1cf06aa3772ea75c2c776e9ff1d415f0e4074b Mon Sep 17 00:00:00 2001 From: MoffittAndrew Date: Fri, 21 Mar 2025 10:31:41 +0000 Subject: [PATCH 03/70] Removing black + configuring ruff using movement repo config --- .pre-commit-config.yaml | 18 ++++++++++++---- pyproject.toml | 48 ++++++++++++++++++++++++++++++++++------- 2 files changed, 54 insertions(+), 12 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 515c16ca8..8fbbd4031 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -8,23 +8,33 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks rev: v5.0.0 hooks: + - id: check-added-large-files - id: check-docstring-first - id: check-executables-have-shebangs + - id: check-case-conflict - id: check-merge-conflict + - id: check-symlinks + - id: check-yaml - id: check-toml + - id: debug-statements - id: end-of-file-fixer - id: mixed-line-ending args: [--fix=lf] + - id: name-tests-test + args: ["--pytest-test-first"] - id: requirements-txt-fixer - id: trailing-whitespace + - repo: https://github.com/pre-commit/pygrep-hooks + rev: v1.10.0 + hooks: + - id: rst-backticks + - id: rst-directive-colons + - id: rst-inline-touching-normal - repo: https://github.com/astral-sh/ruff-pre-commit rev: v0.9.9 hooks: - id: ruff - - repo: https://github.com/psf/black - rev: 25.1.0 - hooks: - - id: black + - id: ruff-format - repo: https://github.com/pre-commit/mirrors-mypy rev: v1.15.0 hooks: diff --git a/pyproject.toml b/pyproject.toml index 997f6f406..c8ce1baca 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -51,7 +51,6 @@ dev = [ "pytest-mock", "coverage", "tox", - "black", "mypy", "pre-commit", "ruff", @@ -91,19 +90,52 @@ exclude = ["tests*", "docs*"] [tool.pytest.ini_options] addopts = "--cov=datashuttle" -[tool.black] -target-version = ['py39', 'py310', 'py311', 'py312'] -skip-string-normalization = false -line-length = 79 - [tool.ruff] line-length = 79 exclude = ["__init__.py","build",".eggs"] fix = true [tool.ruff.lint] -ignore = ["E203","E501","E731","C901","W291","W293","E402","E722"] -select = ["I", "E", "F", "TCH", "TID252"] +# See https://docs.astral.sh/ruff/rules/ +ignore = [ + "D203", # one blank line before class + "D213", # multi-line-summary second line +] +select = [ + "E", # pycodestyle errors + "F", # Pyflakes + "UP", # pyupgrade + "I", # isort + "B", # flake8 bugbear + "SIM", # flake8 simplify + "C90", # McCabe complexity + "D", # pydocstyle +] +per-file-ignores = { "tests/*" = [ + "D100", # missing docstring in public module + "D205", # missing blank line between summary and description + "D103", # missing docstring in public function +], "examples/*" = [ + "D400", # first line should end with a period. + "D415", # first line should end with a period, question mark... + "D205", # missing blank line between summary and description +] } + +# Old ruff ruleset + pydocstyle added +# Inconsistent with movement repo, but saving this here for +# now in case there are good reasons to keep these rules +#ignore = ["E203","E501","E731","C901","W291","W293","E402","E722"] +#select = [ +# "I", # isort +# "E", # pycodestyle errors +# "F", # Pyflakes +# "TC", # flake8-type-checking +# "TID252", # flake8-tidy-imports relative-imports +# "D", # pydocstyle +#] + +[tool.ruff.format] +docstring-code-format = true # Also format code in docstrings [tool.ruff.lint.per-file-ignores] "__init__.py" = ["F401"] From ff9d470ad8006e8eb83b204d9ffa8a3080629ebb Mon Sep 17 00:00:00 2001 From: MoffittAndrew Date: Fri, 21 Mar 2025 10:40:51 +0000 Subject: [PATCH 04/70] moving per-file-ignores for __init__.py to the list in tool.ruff.lint --- pyproject.toml | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index c8ce1baca..77052bbf8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -98,8 +98,8 @@ fix = true [tool.ruff.lint] # See https://docs.astral.sh/ruff/rules/ ignore = [ - "D203", # one blank line before class - "D213", # multi-line-summary second line + "D203", # one blank line before class + "D213", # multi-line-summary second line ] select = [ "E", # pycodestyle errors @@ -112,13 +112,17 @@ select = [ "D", # pydocstyle ] per-file-ignores = { "tests/*" = [ - "D100", # missing docstring in public module - "D205", # missing blank line between summary and description - "D103", # missing docstring in public function + "D100", # missing docstring in public module + "D205", # missing blank line between summary and description + "D103", # missing docstring in public function ], "examples/*" = [ - "D400", # first line should end with a period. - "D415", # first line should end with a period, question mark... - "D205", # missing blank line between summary and description + "D400", # first line should end with a period. + "D415", # first line should end with a period, question mark... + "D205", # missing blank line between summary and description +], "__init__.py" = [ + # This was part of the old config + # Is this needed? __init__.py is already part of tool.ruff.exclude + "F401", # auto remove unused imports ] } # Old ruff ruleset + pydocstyle added @@ -137,9 +141,6 @@ per-file-ignores = { "tests/*" = [ [tool.ruff.format] docstring-code-format = true # Also format code in docstrings -[tool.ruff.lint.per-file-ignores] -"__init__.py" = ["F401"] - [tool.ruff.lint.mccabe] max-complexity = 18 From 0818a66597a1a3ceb7bbc540d1b6e4e039a70075 Mon Sep 17 00:00:00 2001 From: MoffittAndrew Date: Fri, 21 Mar 2025 10:44:23 +0000 Subject: [PATCH 05/70] run pre-commit --- .pre-commit-config.yaml | 2 +- datashuttle/configs/canonical_configs.py | 47 ++--- datashuttle/configs/canonical_folders.py | 19 +- datashuttle/configs/canonical_tags.py | 3 +- datashuttle/configs/config_class.py | 41 ++-- datashuttle/configs/load_configs.py | 9 +- datashuttle/datashuttle_class.py | 197 +++++++----------- datashuttle/datashuttle_functions.py | 8 +- datashuttle/tui/app.py | 21 +- datashuttle/tui/configs.py | 115 +++++----- datashuttle/tui/custom_widgets.py | 63 ++---- datashuttle/tui/interface.py | 61 ++---- .../tui/screens/create_folder_settings.py | 25 +-- datashuttle/tui/screens/datatypes.py | 26 +-- datashuttle/tui/screens/get_help.py | 1 - datashuttle/tui/screens/modal_dialogs.py | 13 +- datashuttle/tui/screens/new_project.py | 5 +- datashuttle/tui/screens/project_manager.py | 16 +- datashuttle/tui/screens/project_selector.py | 5 +- datashuttle/tui/screens/settings.py | 3 +- datashuttle/tui/screens/setup_ssh.py | 17 +- datashuttle/tui/tabs/create_folders.py | 43 ++-- datashuttle/tui/tabs/logging.py | 2 +- datashuttle/tui/tabs/transfer.py | 24 +-- datashuttle/tui/tabs/transfer_status_tree.py | 24 +-- datashuttle/tui/tooltips.py | 3 +- datashuttle/tui/utils/tui_decorators.py | 3 +- datashuttle/tui/utils/tui_validators.py | 10 +- datashuttle/tui_launcher.py | 4 +- datashuttle/utils/data_transfer.py | 37 ++-- datashuttle/utils/decorators.py | 9 +- datashuttle/utils/ds_logger.py | 19 +- datashuttle/utils/folder_class.py | 3 +- datashuttle/utils/folders.py | 54 ++--- datashuttle/utils/formatting.py | 45 ++-- datashuttle/utils/getters.py | 32 +-- datashuttle/utils/rclone.py | 53 ++--- datashuttle/utils/ssh.py | 35 ++-- datashuttle/utils/utils.py | 39 ++-- datashuttle/utils/validation.py | 81 +++---- tests/conftest.py | 3 +- tests/ssh_test_utils.py | 14 +- tests/test_utils.py | 68 +++--- tests/tests_integration/_test_configs.py | 35 +--- tests/tests_integration/base.py | 10 +- .../tests_integration/test_create_folders.py | 37 ++-- tests/tests_integration/test_datatypes.py | 12 +- .../test_filesystem_transfer.py | 46 ++-- tests/tests_integration/test_formatting.py | 9 +- .../tests_integration/test_local_only_mode.py | 16 +- tests/tests_integration/test_logging.py | 53 ++--- tests/tests_integration/test_settings.py | 22 +- .../test_ssh_file_transfer.py | 18 +- tests/tests_integration/test_ssh_setup.py | 19 +- .../tests_integration/test_transfer_checks.py | 3 +- tests/tests_integration/test_validation.py | 55 ++--- tests/tests_tui/test_local_only_project.py | 16 +- tests/tests_tui/test_tui_configs.py | 30 +-- tests/tests_tui/test_tui_create_folders.py | 30 +-- tests/tests_tui/test_tui_datatypes.py | 6 +- tests/tests_tui/test_tui_directorytree.py | 15 +- tests/tests_tui/test_tui_get_help.py | 5 +- tests/tests_tui/test_tui_logging.py | 9 +- tests/tests_tui/test_tui_settings.py | 12 +- tests/tests_tui/test_tui_transfer.py | 13 +- .../test_tui_widgets_and_defaults.py | 47 +---- tests/tests_tui/tui_base.py | 66 ++---- tests/tests_unit/test_links.py | 3 +- tests/tests_unit/test_unit.py | 35 ++-- tests/tests_unit/test_validation_unit.py | 28 +-- 70 files changed, 654 insertions(+), 1298 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8fbbd4031..299fba1d1 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -34,7 +34,7 @@ repos: rev: v0.9.9 hooks: - id: ruff - - id: ruff-format + #- id: ruff-format - repo: https://github.com/pre-commit/mirrors-mypy rev: v1.15.0 hooks: diff --git a/datashuttle/configs/canonical_configs.py b/datashuttle/configs/canonical_configs.py index 7213b3d88..8d03eed6e 100644 --- a/datashuttle/configs/canonical_configs.py +++ b/datashuttle/configs/canonical_configs.py @@ -1,5 +1,4 @@ -""" -This module contains all information for the required +"""This module contains all information for the required format of the configs class. This is clearly defined as configs can be provided from file or input dynamically and so careful checks must be done. @@ -32,8 +31,7 @@ def get_canonical_configs() -> dict: - """ - The only permitted types for DataShuttle + """The only permitted types for DataShuttle config values. """ canonical_configs = { @@ -48,8 +46,7 @@ def get_canonical_configs() -> dict: def keys_str_on_file_but_path_in_class() -> list[str]: - """ - All configs which are paths are converted to pathlib.Path + """All configs which are paths are converted to pathlib.Path objects on load. This list indicates which config entries are to be converted to Path. """ @@ -65,8 +62,7 @@ def keys_str_on_file_but_path_in_class() -> list[str]: def check_dict_values_raise_on_fail(config_dict: Configs) -> None: - """ - Central function for performing checks on a + """Central function for performing checks on a DataShuttle Configs UserDict class. This should be run after any change to the configs (e.g. make_config_file, update_config_file, supply_config_file). @@ -75,9 +71,9 @@ def check_dict_values_raise_on_fail(config_dict: Configs) -> None: Parameters ---------- - config_dict datashuttle config UserDict + """ canonical_dict = get_canonical_configs() @@ -144,8 +140,7 @@ def check_dict_values_raise_on_fail(config_dict: Configs) -> None: def raise_on_bad_local_only_project_configs(config_dict: Configs) -> None: - """ - There is no circumstance where one of `central_path` and `connection_method` + """There is no circumstance where one of `central_path` and `connection_method` should be set and not the other. Either both are set ('full' project) or neither are ('local only' project). Check this assumption here. """ @@ -171,14 +166,12 @@ def raise_on_bad_path_syntax( path_name: str, path_type: str, ) -> None: - """ - Error if some common, unsupported patterns are observed + """Error if some common, unsupported patterns are observed (e.g. ~, .) for path. """ if path_name[0] == "~": utils.log_and_raise_error( - f"{path_type} must contain the full folder path " - "with no ~ syntax.", + f"{path_type} must contain the full folder path with no ~ syntax.", ConfigError, ) @@ -193,13 +186,10 @@ def raise_on_bad_path_syntax( def check_config_types(config_dict: Configs) -> None: - """ - Check the type of passed configs matches the canonical types. - """ + """Check the type of passed configs matches the canonical types.""" required_types = get_canonical_configs() for key in config_dict.keys(): - expected_type = required_types[key] try: typeguard.check_type(config_dict[key], expected_type) @@ -218,8 +208,7 @@ def check_config_types(config_dict: Configs) -> None: def get_tui_config_defaults() -> Dict: - """ - Get the default settings for the datatype checkboxes + """Get the default settings for the datatype checkboxes in the TUI. Two sets are maintained (one for creating, @@ -248,7 +237,6 @@ def get_tui_config_defaults() -> Dict: # Fill all datatype options for broad_key in get_broad_datatypes(): - settings["tui"]["create_checkboxes_on"][broad_key] = { # type: ignore "on": True, "displayed": True, @@ -276,8 +264,7 @@ def get_name_templates_defaults() -> Dict: def get_persistent_settings_defaults() -> Dict: - """ - Persistent settings are settings that are maintained + """Persistent settings are settings that are maintained across sessions. Currently, persistent settings for both the API and TUI are stored in the same place. @@ -293,8 +280,7 @@ def get_persistent_settings_defaults() -> Dict: def get_datatypes() -> List[str]: - """ - Canonical list of datatype flags based on NeuroBlueprint. + """Canonical list of datatype flags based on NeuroBlueprint. This must be kept up to date with the datatypes in the NeuroBlueprint specification. """ @@ -306,8 +292,7 @@ def get_broad_datatypes(): def get_narrow_datatypes(): - """ - Return the narrow datatype associated with each broad datatype. + """Return the narrow datatype associated with each broad datatype. The mapping between broad and narrow datatypes is required for validation. """ return { @@ -337,8 +322,7 @@ def get_narrow_datatypes(): def quick_get_narrow_datatypes(): - """ - A convenience wrapper around `get_narrow_datatypes()` + """A convenience wrapper around `get_narrow_datatypes()` to quickly get a list of all narrow datatypes. """ all_narrow_datatypes = get_narrow_datatypes() @@ -352,8 +336,7 @@ def quick_get_narrow_datatypes(): def in_place_update_settings_for_narrow_datatype(settings: dict): - """ - In versions < v0.6.0, only 'broad' datatypes were implemented + """In versions < v0.6.0, only 'broad' datatypes were implemented and available in the TUI. Since, 'narrow' datatypes are introduced and datatype tui can be set to be both on / off but also displayed / not displayed. diff --git a/datashuttle/configs/canonical_folders.py b/datashuttle/configs/canonical_folders.py index a6b77128c..104019704 100644 --- a/datashuttle/configs/canonical_folders.py +++ b/datashuttle/configs/canonical_folders.py @@ -11,8 +11,7 @@ def get_datatype_folders() -> dict: - """ - This function holds the canonical folders + """This function holds the canonical folders managed by datashuttle. Notes @@ -32,6 +31,7 @@ def get_datatype_folders() -> dict: an option for rare cases in which advanced users want to change it. level : "sub" or "ses", level to make the folder at. + """ return { datatype: Folder(name=datatype, level="ses") @@ -40,8 +40,7 @@ def get_datatype_folders() -> dict: def get_non_sub_names() -> List[str]: - """ - Get all arguments that are not allowed at the + """Get all arguments that are not allowed at the subject level for data transfer, i.e. as sub_names """ return [ @@ -53,8 +52,7 @@ def get_non_sub_names() -> List[str]: def get_non_ses_names() -> List[str]: - """ - Get all arguments that are not allowed at the + """Get all arguments that are not allowed at the session level for data transfer, i.e. as ses_names """ return [ @@ -66,8 +64,7 @@ def get_non_ses_names() -> List[str]: def canonical_reserved_keywords() -> List[str]: - """ - Key keyword arguments that are passed to `sub_names` or + """Key keyword arguments that are passed to `sub_names` or `ses_names` but that we """ return get_non_sub_names() + get_non_ses_names() @@ -78,16 +75,14 @@ def get_top_level_folders() -> List[TopLevelFolder]: def get_datashuttle_path() -> Path: - """ - Get the datashuttle path where all project + """Get the datashuttle path where all project configs are stored. """ return Path.home() / ".datashuttle" def get_project_datashuttle_path(project_name: str) -> Tuple[Path, Path]: - """ - Get the datashuttle path for the project, + """Get the datashuttle path for the project, where configuration files are stored. Also, return a temporary path in this for logging in some cases where local_path location is not clear. diff --git a/datashuttle/configs/canonical_tags.py b/datashuttle/configs/canonical_tags.py index 233350bc6..7cc274dd0 100644 --- a/datashuttle/configs/canonical_tags.py +++ b/datashuttle/configs/canonical_tags.py @@ -1,6 +1,5 @@ def tags(tag_name: str) -> str: - """ - Centralised function to get the tags used + """Centralised function to get the tags used in subject / session name processing. If changing the formatting of these tags, it is only required to change the dict values here. diff --git a/datashuttle/configs/config_class.py b/datashuttle/configs/config_class.py index 7f5a8525c..f1282d935 100644 --- a/datashuttle/configs/config_class.py +++ b/datashuttle/configs/config_class.py @@ -25,8 +25,7 @@ class Configs(UserDict): - """ - Class to hold the datashuttle configs. + """Class to hold the datashuttle configs. The configs must match exactly the standard set in canonical_configs.py. If updating these configs, @@ -39,13 +38,13 @@ class Configs(UserDict): Parameters ---------- - file_path full filepath to save the config .yaml file to. input_dict a dict of config key-value pairs to input dict. This must contain all canonical_config keys + """ def __init__( @@ -81,8 +80,7 @@ def ensure_local_and_central_path_end_in_project_name(self): self[path_type] = self[path_type] / self.project_name def check_dict_values_raise_on_fail(self) -> None: - """ - Check the values of the current dictionary are set + """Check the values of the current dictionary are set correctly and will not cause downstream errors. This will raise an error if the dictionary @@ -104,9 +102,7 @@ def values(self) -> ValuesView: # ------------------------------------------------------------------------- def dump_to_file(self) -> None: - """ - Save the dictionary to .yaml file stored in self.file_path. - """ + """Save the dictionary to .yaml file stored in self.file_path.""" cfg_to_save = copy.deepcopy(self.data) load_configs.convert_str_and_pathlib_paths(cfg_to_save, "path_to_str") @@ -114,12 +110,11 @@ def dump_to_file(self) -> None: yaml.dump(cfg_to_save, config_file, sort_keys=False) def load_from_file(self) -> None: - """ - Load a config dict saved at .yaml file. Note this will + """Load a config dict saved at .yaml file. Note this will not automatically check the configs are valid, this requires calling self.check_dict_values_raise_on_fail() """ - with open(self.file_path, "r") as config_file: + with open(self.file_path) as config_file: config_dict = yaml.full_load(config_file) load_configs.convert_str_and_pathlib_paths(config_dict, "str_to_path") @@ -136,14 +131,12 @@ def build_project_path( sub_folders: Union[str, list], top_level_folder: TopLevelFolder, ) -> Path: - """ - Function for joining relative path to base dir. + """Function for joining relative path to base dir. If path already starts with base dir, the base dir will not be joined. Parameters ---------- - base "local", "central" or "datashuttle" @@ -151,6 +144,7 @@ def build_project_path( a list (or string for 1) of folder names to be joined into a path. If file included, must be last entry (with ext). + """ if isinstance(sub_folders, list): sub_folders_str = "/".join(sub_folders) @@ -173,12 +167,10 @@ def get_base_folder( base: str, top_level_folder: TopLevelFolder, ) -> Path: - """ - Convenience function to return the full base path. + """Convenience function to return the full base path. Parameters ---------- - base base path, "local", "central" or "datashuttle" @@ -193,8 +185,7 @@ def get_base_folder( def get_rclone_config_name( self, connection_method: Optional[str] = None ) -> str: - """ - Convenience function to get the rclone config + """Convenience function to get the rclone config name (these configs are created by datashuttle but managed and stored by rclone). """ @@ -206,8 +197,7 @@ def get_rclone_config_name( def make_rclone_transfer_options( self, overwrite_existing_files: OverwriteExistingFiles, dry_run: bool ) -> Dict: - """ - This function originally collected the relevant arguments + """This function originally collected the relevant arguments from configs. Now, all are passed via function arguments However, now we fix the previously configurable arguments `show_transfer_progress` and `dry_run` here. @@ -246,8 +236,7 @@ def init_paths(self) -> None: def make_and_get_logging_path( self, ) -> Path: - """ - Build (and create if does not exist) the path where + """Build (and create if does not exist) the path where logs are stored. """ logging_path = self.project_metadata_path / "logs" @@ -257,8 +246,7 @@ def make_and_get_logging_path( def get_datatype_as_dict_items( self, datatype: Union[str, list] ) -> Union[ItemsView, zip]: - """ - Get the .items() structure of the datatype, either all of + """Get the .items() structure of the datatype, either all of the canonical datatypes or as a single item. """ if isinstance(datatype, str): @@ -279,8 +267,7 @@ def get_datatype_as_dict_items( return items def is_local_project(self): - """ - A project is 'local-only' if it has no `central_path` and `connection_method`. + """A project is 'local-only' if it has no `central_path` and `connection_method`. It can be used to make folders and validate, but not for transfer. """ canonical_configs.raise_on_bad_local_only_project_configs(self) diff --git a/datashuttle/configs/load_configs.py b/datashuttle/configs/load_configs.py index c15e3977d..edcd4b6a9 100644 --- a/datashuttle/configs/load_configs.py +++ b/datashuttle/configs/load_configs.py @@ -17,8 +17,7 @@ def attempt_load_configs( config_path: Path, verbose: bool = True, ) -> Optional[Configs]: - """ - Try to load an existing config file, that was previously + """Try to load an existing config file, that was previously saved by Datashuttle. This should always work, unless not already initialised (prompt) or these have been changed manually. @@ -33,6 +32,7 @@ def attempt_load_configs( verbose warnings and error messages will be printed. + """ exists = config_path.is_file() @@ -68,18 +68,17 @@ def attempt_load_configs( def convert_str_and_pathlib_paths( config_dict: Union["Configs", dict], direction: str ) -> None: - """ - Config paths are stored as str in the .yaml but used as Path + """Config paths are stored as str in the .yaml but used as Path in the module, so make the conversion here. Parameters ---------- - config_dict DataShuttle.cfg dict of configs direction "path_to_str" or "str_to_path" + """ for path_key in canonical_configs.keys_str_on_file_but_path_in_class(): value = config_dict[path_key] diff --git a/datashuttle/datashuttle_class.py b/datashuttle/datashuttle_class.py index ad757ac38..6cc044625 100644 --- a/datashuttle/datashuttle_class.py +++ b/datashuttle/datashuttle_class.py @@ -62,8 +62,7 @@ class DataShuttle: - """ - DataShuttle is a tool for convenient scientific + """DataShuttle is a tool for convenient scientific project management and data transfer in BIDS format. The expected organisation is a central repository @@ -89,7 +88,6 @@ class DataShuttle: Parameters ---------- - project_name The project name to use the datashuttle Folders containing all project files @@ -103,10 +101,10 @@ class DataShuttle: If `True`, a start-up message displaying the current state of the program (e.g. persistent settings such as the 'top-level folder') is shown. + """ def __init__(self, project_name: str, print_startup_message: bool = True): - self._error_on_base_project_name(project_name) self.project_name = project_name ( @@ -133,8 +131,7 @@ def __init__(self, project_name: str, print_startup_message: bool = True): rclone.prompt_rclone_download_if_does_not_exist() def _set_attributes_after_config_load(self) -> None: - """ - Once config file is loaded, update all private attributes + """Once config file is loaded, update all private attributes according to config contents. """ self.cfg.init_paths() @@ -155,8 +152,7 @@ def create_folders( bypass_validation: bool = False, log: bool = True, ) -> Dict[str, List[Path]]: - """ - Create a subject / session folder tree in the project + """Create a subject / session folder tree in the project folder. The passed subject / session names are formatted and validated. If this succeeds, fully validation against all subject / session folders in @@ -165,7 +161,6 @@ def create_folders( Parameters ---------- - top_level_folder Whether to make the folders in `rawdata` or `derivatives`. @@ -208,7 +203,6 @@ def create_folders( Notes ----- - sub_names or ses_names may contain formatting tags @TO@ @@ -227,6 +221,7 @@ def create_folders( project.create_folders("rawdata", "sub-001", datatype="behav") project.create_folders("rawdata", "sub-002@TO@005", ["ses-001", "ses-002"], ["ephys", "behav"]) + """ if log: self._start_log("create-folders", local_vars=locals()) @@ -289,8 +284,7 @@ def _format_and_validate_names( bypass_validation: bool, log: bool = True, ) -> Tuple[List[str], List[str]]: - """ - A central method for the formatting and validation of subject / session + """A central method for the formatting and validation of subject / session names for folder creation. This is called by both DataShuttle and during TUI validation. """ @@ -335,8 +329,7 @@ def upload_custom( dry_run: bool = False, init_log: bool = True, ) -> None: - """ - Upload data from a local project to the central project + """Upload data from a local project to the central project folder. In the case that a file / folder exists on the central and local, the central will not be overwritten even if the central file is an older version. Data @@ -344,7 +337,6 @@ def upload_custom( Parameters ---------- - top_level_folder The top-level folder (e.g. `"rawdata"`, `"derivatives"`) to transfer files and folders within. @@ -379,6 +371,7 @@ def upload_custom( (Optional). Whether to handle logging. This should always be True, unless logger is handled elsewhere (e.g. in a calling function). + """ if init_log: self._start_log("upload-custom", local_vars=locals()) @@ -412,13 +405,11 @@ def download_custom( dry_run: bool = False, init_log: bool = True, ) -> None: - """ - Download data from the central project folder to the + """Download data from the central project folder to the local project folder. Parameters ---------- - top_level_folder The top-level folder (e.g. `rawdata`) to transfer files and folders within. @@ -453,6 +444,7 @@ def download_custom( (Optional). Whether to handle logging. This should always be True, unless logger is handled elsewhere (e.g. in a calling function). + """ if init_log: self._start_log("download-custom", local_vars=locals()) @@ -486,12 +478,10 @@ def upload_rawdata( overwrite_existing_files: OverwriteExistingFiles = "never", dry_run: bool = False, ): - """ - Upload files in the `rawdata` top level folder. + """Upload files in the `rawdata` top level folder. Parameters ---------- - overwrite_existing_files If "never" files on target will never be overwritten by source. If "always" files on target will be overwritten by source if @@ -503,6 +493,7 @@ def upload_rawdata( perform a dry-run of transfer. This will output as if file transfer was taking place, but no files will be moved. Useful to check which files will be moved on data transfer. + """ self._transfer_top_level_folder( "upload", @@ -518,12 +509,10 @@ def upload_derivatives( overwrite_existing_files: OverwriteExistingFiles = "never", dry_run: bool = False, ): - """ - Upload files in the `derivatives` top level folder. + """Upload files in the `derivatives` top level folder. Parameters ---------- - overwrite_existing_files If "never" files on target will never be overwritten by source. If "always" files on target will be overwritten by source if @@ -535,6 +524,7 @@ def upload_derivatives( perform a dry-run of transfer. This will output as if file transfer was taking place, but no files will be moved. Useful to check which files will be moved on data transfer. + """ self._transfer_top_level_folder( "upload", @@ -550,12 +540,10 @@ def download_rawdata( overwrite_existing_files: OverwriteExistingFiles = "never", dry_run: bool = False, ): - """ - Download files in the `rawdata` top level folder. + """Download files in the `rawdata` top level folder. Parameters ---------- - overwrite_existing_files If "never" files on target will never be overwritten by source. If "always" files on target will be overwritten by source if @@ -567,6 +555,7 @@ def download_rawdata( perform a dry-run of transfer. This will output as if file transfer was taking place, but no files will be moved. Useful to check which files will be moved on data transfer. + """ self._transfer_top_level_folder( "download", @@ -582,12 +571,10 @@ def download_derivatives( overwrite_existing_files: OverwriteExistingFiles = "never", dry_run: bool = False, ): - """ - Download files in the `derivatives` top level folder. + """Download files in the `derivatives` top level folder. Parameters ---------- - overwrite_existing_files If "never" files on target will never be overwritten by source. If "always" files on target will be overwritten by source if @@ -599,6 +586,7 @@ def download_derivatives( perform a dry-run of transfer. This will output as if file transfer was taking place, but no files will be moved. Useful to check which files will be moved on data transfer. + """ self._transfer_top_level_folder( "download", @@ -614,14 +602,12 @@ def upload_entire_project( overwrite_existing_files: OverwriteExistingFiles = "never", dry_run: bool = False, ) -> None: - """ - Upload the entire project (from 'local' to 'central'), + """Upload the entire project (from 'local' to 'central'), i.e. including every top level folder (e.g. 'rawdata', 'derivatives', 'code', 'analysis'). Parameters ---------- - overwrite_existing_files If "never" files on target will never be overwritten by source. If "always" files on target will be overwritten by source if @@ -633,6 +619,7 @@ def upload_entire_project( perform a dry-run of transfer. This will output as if file transfer was taking place, but no files will be moved. Useful to check which files will be moved on data transfer. + """ self._start_log("upload-entire-project", local_vars=locals()) self._transfer_entire_project( @@ -647,14 +634,12 @@ def download_entire_project( overwrite_existing_files: OverwriteExistingFiles = "never", dry_run: bool = False, ) -> None: - """ - Download the entire project (from 'central' to 'local'), + """Download the entire project (from 'central' to 'local'), i.e. including every top level folder (e.g. 'rawdata', 'derivatives', 'code', 'analysis'). Parameters ---------- - overwrite_existing_files If "never" files on target will never be overwritten by source. If "always" files on target will be overwritten by source if @@ -666,6 +651,7 @@ def download_entire_project( perform a dry-run of transfer. This will output as if file transfer was taking place, but no files will be moved. Useful to check which files will be moved on data transfer. + """ self._start_log("download-entire-project", local_vars=locals()) self._transfer_entire_project( @@ -681,8 +667,7 @@ def upload_specific_folder_or_file( overwrite_existing_files: OverwriteExistingFiles = "never", dry_run: bool = False, ) -> None: - """ - Upload a specific file or folder. If transferring + """Upload a specific file or folder. If transferring a single file, the path including the filename is required (see 'filepath' input). If a folder, wildcards "*" or "**" must be used to transfer @@ -691,7 +676,6 @@ def upload_specific_folder_or_file( Parameters ---------- - filepath a string containing the full filepath. @@ -706,6 +690,7 @@ def upload_specific_folder_or_file( perform a dry-run of transfer. This will output as if file transfer was taking place, but no files will be moved. Useful to check which files will be moved on data transfer. + """ self._start_log("upload-specific-folder-or-file", local_vars=locals()) @@ -723,8 +708,7 @@ def download_specific_folder_or_file( overwrite_existing_files: OverwriteExistingFiles = "never", dry_run: bool = False, ) -> None: - """ - Download a specific file or folder. If transferring + """Download a specific file or folder. If transferring a single file, the path including the filename is required (see 'filepath' input). If a folder, wildcards "*" or "**" must be used to transfer @@ -733,7 +717,6 @@ def download_specific_folder_or_file( Parameters ---------- - filepath a string containing the full filepath. @@ -748,6 +731,7 @@ def download_specific_folder_or_file( perform a dry-run of transfer. This will output as if file transfer was taking place, but no files will be moved. Useful to check which files will be moved on data transfer. + """ self._start_log( "download-specific-folder-or-file", local_vars=locals() @@ -767,8 +751,7 @@ def _transfer_top_level_folder( dry_run: bool = False, init_log: bool = True, ): - """ - Core function to upload / download files within a + """Core function to upload / download files within a particular top-level-folder. e.g. `upload_rawdata().` """ if init_log: @@ -798,9 +781,7 @@ def _transfer_top_level_folder( def _transfer_specific_file_or_folder( self, upload_or_download, filepath, overwrite_existing_files, dry_run ): - """ - Core function for upload/download_specific_folder_or_file(). - """ + """Core function for upload/download_specific_folder_or_file().""" if isinstance(filepath, str): filepath = Path(filepath) @@ -841,8 +822,7 @@ def _transfer_specific_file_or_folder( @requires_ssh_configs @check_is_not_local_project def setup_ssh_connection(self) -> None: - """ - Setup a connection to the central server using SSH. + """Setup a connection to the central server using SSH. Assumes the central_host_id and central_host_username are set in configs (see make_config_file() and update_config_file()) @@ -873,17 +853,16 @@ def setup_ssh_connection(self) -> None: @requires_ssh_configs @check_is_not_local_project def write_public_key(self, filepath: str) -> None: - """ - By default, the SSH private key only is stored, in + """By default, the SSH private key only is stored, in the datashuttle configs folder. Use this function to save the public key. Parameters ---------- - filepath full filepath (inc filename) to write the public key to. + """ key: paramiko.RSAKey key = paramiko.RSAKey.from_private_key_file( @@ -906,8 +885,7 @@ def make_config_file( central_host_id: Optional[str] = None, central_host_username: Optional[str] = None, ) -> None: - """ - Initialise the configurations for datashuttle to use on the + """Initialise the configurations for datashuttle to use on the local machine. Once initialised, these settings will be used each time the datashuttle is opened. This method can also be used to completely overwrite existing configs. @@ -921,7 +899,6 @@ def make_config_file( Parameters ---------- - local_path path to project folder on local machine @@ -946,6 +923,7 @@ def make_config_file( central_host_username username for which to log in to central host. e.g. "jziminski" + """ self._start_log( "make-config-file", @@ -1021,22 +999,17 @@ def update_config_file(self, **kwargs) -> None: @check_configs_set def get_local_path(self) -> Path: - """ - Get the projects local path. - """ + """Get the projects local path.""" return self.cfg["local_path"] @check_configs_set @check_is_not_local_project def get_central_path(self) -> Path: - """ - Get the project central path. - """ + """Get the project central path.""" return self.cfg["central_path"] def get_datashuttle_path(self) -> Path: - """ - Get the path to the local datashuttle + """Get the path to the local datashuttle folder where configs and other datashuttle files are stored. """ @@ -1044,9 +1017,7 @@ def get_datashuttle_path(self) -> Path: @check_configs_set def get_config_path(self) -> Path: - """ - Get the full path to the DataShuttle config file. - """ + """Get the full path to the DataShuttle config file.""" return self._config_path @check_configs_set @@ -1055,15 +1026,12 @@ def get_configs(self) -> Configs: @check_configs_set def get_logging_path(self) -> Path: - """ - Get the path where datashuttle logs are written. - """ + """Get the path where datashuttle logs are written.""" return self.cfg.logging_path @staticmethod def get_existing_projects() -> List[Path]: - """ - Get a list of existing project names found on the local machine. + """Get a list of existing project names found on the local machine. This is based on project folders in the "home / .datashuttle" folder that contain valid config.yaml files. """ @@ -1076,13 +1044,11 @@ def get_next_sub( return_with_prefix: bool = True, include_central: bool = False, ) -> str: - """ - Convenience function for get_next_sub_or_ses + """Convenience function for get_next_sub_or_ses to find the next subject number. Parameters ---------- - return_with_prefix If `True`, return with the "sub-" prefix. @@ -1090,6 +1056,7 @@ def get_next_sub( If `False, only get names from `local_path`, otherwise from `local_path` and `central_path`. If in local-project mode, this flag is ignored. + """ name_template = self.get_name_templates() name_template_regexp = ( @@ -1117,13 +1084,11 @@ def get_next_ses( return_with_prefix: bool = True, include_central: bool = False, ) -> str: - """ - Convenience function for get_next_sub_or_ses + """Convenience function for get_next_sub_or_ses to find the next session number. Parameters ---------- - top_level_folder "rawdata" or "derivatives" @@ -1137,6 +1102,7 @@ def get_next_ses( If `False, only get names from `local_path`, otherwise from `local_path` and `central_path`. If in local-project mode, this flag is ignored. + """ name_template = self.get_name_templates() name_template_regexp = ( @@ -1158,8 +1124,7 @@ def get_next_ses( @check_configs_set def is_local_project(self) -> bool: - """ - A project is 'local-only' if it has no `central_path` and `connection_method`. + """A project is 'local-only' if it has no `central_path` and `connection_method`. It can be used to make folders and validate, but not for transfer. """ return self.cfg.is_local_project() @@ -1168,22 +1133,20 @@ def is_local_project(self) -> bool: # ------------------------------------------------------------------------- def get_name_templates(self) -> Dict: - """ - Get the regexp templates used for validation. If + """Get the regexp templates used for validation. If the "on" key is set to `False`, template validation is not performed. Returns ------- - name_templates e.g. {"name_templates": {"on": False, "sub": None, "ses": None}} + """ settings = self._load_persistent_settings() return settings["name_templates"] def set_name_templates(self, new_name_templates: Dict) -> None: - """ - Update the persistent settings with new name templates. + """Update the persistent settings with new name templates. Name templates are regexp for that, when name_templates["on"] is set to `True`, "sub" and "ses" names are validated against @@ -1191,11 +1154,11 @@ def set_name_templates(self, new_name_templates: Dict) -> None: Parameters ---------- - new_name_templates e.g. {"name_templates": {"on": False, "sub": None, "ses": None}} where "sub" or "ses" can be a regexp that subject and session names respectively are validated against. + """ self._update_persistent_setting("name_templates", new_name_templates) @@ -1205,9 +1168,7 @@ def set_name_templates(self, new_name_templates: Dict) -> None: @check_configs_set def show_configs(self) -> None: - """ - Print the current configs to the terminal. - """ + """Print the current configs to the terminal.""" utils.print_message_to_user(self._get_json_dumps_config()) # ------------------------------------------------------------------------- @@ -1222,14 +1183,12 @@ def validate_project( include_central: bool = False, strict_mode: bool = False, ) -> List[str]: - """ - Perform validation on the project. This checks the subject + """Perform validation on the project. This checks the subject and session level folders to ensure there are no NeuroBlueprint formatting issues. Parameters ---------- - top_level_folder Folder to check, either "rawdata" or "derivatives". If ``None``, will check both folders. @@ -1250,6 +1209,7 @@ def validate_project( starting with sub- or ses- prefix are checked. In `Strict Mode`, any folder not prefixed with sub-, ses- or a valid datatype will raise a validation issue. + """ utils.print_message_to_user( f"Logs of the validation will be stored in: " @@ -1285,8 +1245,7 @@ def validate_project( @staticmethod def check_name_formatting(names: Union[str, list], prefix: Prefix) -> None: - """ - Pass list of names to check how these will be auto-formatted, + """Pass list of names to check how these will be auto-formatted, for example as when passed to create_folders() or upload_custom() or download() @@ -1295,13 +1254,13 @@ def check_name_formatting(names: Union[str, list], prefix: Prefix) -> None: Parameters ---------- - names A string or list of subject or session names. prefix The relevant subject or session prefix, e.g. "sub-" or "ses-" + """ if prefix not in ["sub", "ses"]: utils.log_and_raise_error( @@ -1325,19 +1284,17 @@ def _transfer_entire_project( overwrite_existing_files: OverwriteExistingFiles, dry_run: bool, ) -> None: - """ - Transfer (i.e. upload or download) the entire project (i.e. + """Transfer (i.e. upload or download) the entire project (i.e. every 'top level folder' (e.g. 'rawdata', 'derivatives'). Parameters ---------- - upload_or_download direction to transfer the data, either "upload" (from local to central) or "download" (from central to local). + """ for top_level_folder in canonical_folders.get_top_level_folders(): - utils.log_and_message(f"Transferring `{top_level_folder}`") self._transfer_top_level_folder( @@ -1355,14 +1312,12 @@ def _start_log( store_in_temp_folder: bool = False, verbose: bool = True, ) -> None: - """ - Initialize the logger. This is typically called at + """Initialize the logger. This is typically called at the start of public methods to initialize logging for a specific function call. Parameters ---------- - command_name name of the command, for the log output files. @@ -1373,6 +1328,7 @@ def _start_log( store_in_temp_folder if `False`, existing logging path will be used (local project .datashuttle). + """ if local_vars is None: variables = None @@ -1392,8 +1348,7 @@ def _start_log( ds_logger.start(path_to_save, command_name, variables, verbose) def _move_logs_from_temp_folder(self) -> None: - """ - Logs are stored within the project folder. Although + """Logs are stored within the project folder. Although in some instances, when setting configs, we do not know what the project folder is. In this case, make the logs in a temp folder in the .datashuttle config folder, @@ -1430,8 +1385,7 @@ def _error_on_base_project_name(self, project_name): ) def _log_successful_config_change(self, message: bool = False) -> None: - """ - Log the entire config at the time of config change. + """Log the entire config at the time of config change. If messaged, just message "update successful" rather than print the entire configs as it becomes confusing. """ @@ -1443,8 +1397,7 @@ def _log_successful_config_change(self, message: bool = False) -> None: ) def _get_json_dumps_config(self) -> str: - """ - Get the config dictionary formatted as json.dumps() + """Get the config dictionary formatted as json.dumps() which allows well formatted printing. """ copy_dict = copy.deepcopy(self.cfg.data) @@ -1452,8 +1405,7 @@ def _get_json_dumps_config(self) -> str: return json.dumps(copy_dict, indent=4) def _make_project_metadata_if_does_not_exist(self) -> None: - """ - Within the project local_path is also a .datashuttle + """Within the project local_path is also a .datashuttle folder that contains additional information, e.g. logs. """ folders.create_folders(self.cfg.project_metadata_path, log=False) @@ -1477,25 +1429,23 @@ def _setup_rclone_central_local_filesystem_config(self) -> None: def _update_persistent_setting( self, setting_name: str, setting_value: Any ) -> None: - """ - Load settings that are stored persistently across datashuttle + """Load settings that are stored persistently across datashuttle sessions. These are stored in yaml dumped to dictionary. Parameters ---------- - setting_name dictionary key of the persistent setting to change setting_value value to change the persistent setting to + """ settings = self._load_persistent_settings() if setting_name not in settings: utils.log_and_raise_error( - f"Setting key {setting_name} not found in " - f"settings dictionary", + f"Setting key {setting_name} not found in settings dictionary", KeyError, ) @@ -1504,29 +1454,25 @@ def _update_persistent_setting( self._save_persistent_settings(settings) def _init_persistent_settings(self) -> None: - """ - Initialise the default persistent settings + """Initialise the default persistent settings and save to file. """ settings = canonical_configs.get_persistent_settings_defaults() self._save_persistent_settings(settings) def _save_persistent_settings(self, settings: Dict) -> None: - """ - Save the settings dict to file as .yaml - """ + """Save the settings dict to file as .yaml""" with open(self._persistent_settings_path, "w") as settings_file: yaml.dump(settings, settings_file, sort_keys=False) def _load_persistent_settings(self) -> Dict: - """ - Load settings that are stored persistently across + """Load settings that are stored persistently across datashuttle sessions. """ if not self._persistent_settings_path.is_file(): self._init_persistent_settings() - with open(self._persistent_settings_path, "r") as settings_file: + with open(self._persistent_settings_path) as settings_file: settings = yaml.full_load(settings_file) self._update_settings_with_new_canonical_keys(settings) @@ -1534,8 +1480,7 @@ def _load_persistent_settings(self) -> Dict: return settings def _update_settings_with_new_canonical_keys(self, settings: Dict): - """ - Perform a check on the keys within persistent settings. + """Perform a check on the keys within persistent settings. If they do not exist, persistent settings is from older version and the new keys need adding. If changing keys within the top level (e.g. a dict entry in @@ -1565,9 +1510,7 @@ def _update_settings_with_new_canonical_keys(self, settings: Dict): ) def _check_top_level_folder(self, top_level_folder): - """ - Raise an error if ``top_level_folder`` not correct. - """ + """Raise an error if ``top_level_folder`` not correct.""" canonical_top_level_folders = canonical_folders.get_top_level_folders() if top_level_folder not in canonical_top_level_folders: diff --git a/datashuttle/datashuttle_functions.py b/datashuttle/datashuttle_functions.py index 8c220b536..250fd8bd6 100644 --- a/datashuttle/datashuttle_functions.py +++ b/datashuttle/datashuttle_functions.py @@ -28,14 +28,12 @@ def quick_validate_project( display_mode: DisplayMode = "warn", name_templates: Optional[Dict] = None, ) -> List[str]: - """ - Perform validation on the project. This checks the subject + """Perform validation on the project. This checks the subject and session level folders to ensure there are not NeuroBlueprint formatting issues. Parameters ---------- - project_path Path to the project to validate. Must include the project name, and hold a "rawdata" or "derivatives" folder. @@ -52,6 +50,7 @@ def quick_validate_project( A dictionary of templates for subject and session name to validate against. See ``DataShuttle.set_name_templates()`` for details. + """ project_path = Path(project_path) @@ -91,8 +90,7 @@ def quick_validate_project( def _format_top_level_folder( top_level_folder: TopLevelFolder | None, ) -> List[TopLevelFolder]: - """ - Take a `top_level_folder` ("rawdata" or "derivatives" str) and + """Take a `top_level_folder` ("rawdata" or "derivatives" str) and convert to list, if `None`, convert it to a list of both possible top-level folders. """ diff --git a/datashuttle/tui/app.py b/datashuttle/tui/app.py index ce00f7434..fb342c445 100644 --- a/datashuttle/tui/app.py +++ b/datashuttle/tui/app.py @@ -31,8 +31,7 @@ class TuiApp(App, inherit_bindings=False): # type: ignore - """ - The main app page for the DataShuttle TUI. + """The main app page for the DataShuttle TUI. This class acts as a base class from which all windows (select existing project, make new project, settings and @@ -67,8 +66,7 @@ def set_dark_mode(self, dark_mode: bool) -> None: self.theme = "textual-dark" if dark_mode else "textual-light" def on_button_pressed(self, event: Button.Pressed) -> None: - """ - Raise the relevant screen after button press. `push_screen` + """Raise the relevant screen after button press. `push_screen` second argument is a callback function returned after screen closes. """ if event.button.id == "mainwindow_existing_project_button": @@ -104,8 +102,7 @@ def show_modal_error_dialog(self, message: str) -> None: self.push_screen(modal_dialogs.MessageBox(message, border_color="red")) def handle_open_filesystem_browser(self, path_: Path) -> None: - """ - Open the system file browser to the path with the `showinfm` + """Open the system file browser to the path with the `showinfm` package, performing checks that the path exists prior to opening. """ if not path_.exists(): @@ -166,8 +163,7 @@ def rename_file_or_folder(self, path_, new_name): # Global Settings --------------------------------------------------------- def load_global_settings(self) -> Dict: - """ - Load the 'global settings' for the TUI that determine + """Load the 'global settings' for the TUI that determine project-independent settings that are persistent across sessions. These are stored in the canonical .datashuttle folder (see `get_global_settings_path`). @@ -178,15 +174,13 @@ def load_global_settings(self) -> Dict: global_settings = self.get_default_global_settings() self.save_global_settings(global_settings) else: - with open(settings_path, "r") as file: + with open(settings_path) as file: global_settings = yaml.full_load(file) return global_settings def get_global_settings_path(self) -> Path: - """ - The canonical path for the TUI's global settings. - """ + """The canonical path for the TUI's global settings.""" path_ = canonical_folders.get_datashuttle_path() return path_ / "global_tui_settings.yaml" @@ -206,8 +200,7 @@ def save_global_settings(self, global_settings: Dict) -> None: yaml.dump(global_settings, file, sort_keys=False) def copy_to_clipboard(self, value): - """ - Centralized function to copy to clipboard. + """Centralized function to copy to clipboard. This may fail under some circumstances (e.g., in headless mode on an HPC). """ try: diff --git a/datashuttle/tui/configs.py b/datashuttle/tui/configs.py index c16b1a425..27a701d24 100644 --- a/datashuttle/tui/configs.py +++ b/datashuttle/tui/configs.py @@ -30,8 +30,7 @@ class ConfigsContent(Container): - """ - This screen holds widgets and logic for setting datashuttle configs. + """This screen holds widgets and logic for setting datashuttle configs. It is used in `NewProjectPage` to instantiate a new project and initialise configs, or in `TabbedContent` to update an existing project's configs. @@ -60,8 +59,7 @@ def __init__( self.config_ssh_widgets: List[Any] = [] def compose(self) -> ComposeResult: - """ - `self.config_ssh_widgets` are SSH-setup related widgets + """`self.config_ssh_widgets` are SSH-setup related widgets that are only required when the user selects the SSH connection method. These are displayed / hidden based on the `connection_method` @@ -169,8 +167,7 @@ def compose(self) -> ComposeResult: yield Container(*config_screen_widgets, id="configs_container") def on_mount(self) -> None: - """ - When we have mounted the widgets, the following logic depends on whether + """When we have mounted the widgets, the following logic depends on whether we are setting up a new project (`self.project is `None`) or have an instantiated project. @@ -185,13 +182,13 @@ def on_mount(self) -> None: if self.interface: self.fill_widgets_with_project_configs() else: - self.query_one("#configs_local_filesystem_radiobutton").value = ( - True - ) + self.query_one( + "#configs_local_filesystem_radiobutton" + ).value = True self.switch_ssh_widgets_display(display_ssh=False) - self.query_one("#configs_setup_ssh_connection_button").visible = ( - False - ) + self.query_one( + "#configs_setup_ssh_connection_button" + ).visible = False # Setup tooltips if not self.interface: @@ -222,8 +219,7 @@ def on_mount(self) -> None: self.query_one(id).tooltip = get_tooltip(id) def on_radio_set_changed(self, event: RadioSet.Changed) -> None: - """ - Update the displayed SSH widgets when the `connection_method` + """Update the displayed SSH widgets when the `connection_method` radiobuttons are changed. When SSH is set, ssh config-setters are shown. Otherwise, these @@ -242,23 +238,22 @@ def on_radio_set_changed(self, event: RadioSet.Changed) -> None: if label == "No connection (local only)": self.query_one("#configs_central_path_input").value = "" self.query_one("#configs_central_path_input").disabled = True - self.query_one("#configs_central_path_select_button").disabled = ( - True - ) + self.query_one( + "#configs_central_path_select_button" + ).disabled = True display_ssh = False else: self.query_one("#configs_central_path_input").disabled = False - self.query_one("#configs_central_path_select_button").disabled = ( - False - ) + self.query_one( + "#configs_central_path_select_button" + ).disabled = False display_ssh = True if label == "SSH" else False self.switch_ssh_widgets_display(display_ssh) self.set_central_path_input_tooltip(display_ssh) def set_central_path_input_tooltip(self, display_ssh: bool) -> None: - """ - Use a different tooltip depending on whether connection method + """Use a different tooltip depending on whether connection method is ssh or local filesystem. """ id = "#configs_central_path_input" @@ -292,44 +287,42 @@ def get_platform_dependent_example_paths( return example_path def switch_ssh_widgets_display(self, display_ssh: bool) -> None: - """ - Show or hide SSH-related configs based on whether the current + """Show or hide SSH-related configs based on whether the current `connection_method` widget is "ssh" or "local_filesystem". Parameters ---------- - display_ssh If `True`, display the SSH-related widgets. + """ for widget in self.config_ssh_widgets: widget.display = display_ssh - self.query_one("#configs_central_path_select_button").display = ( - not display_ssh - ) + self.query_one( + "#configs_central_path_select_button" + ).display = not display_ssh if self.interface is None: - self.query_one("#configs_setup_ssh_connection_button").visible = ( - False - ) + self.query_one( + "#configs_setup_ssh_connection_button" + ).visible = False else: - self.query_one("#configs_setup_ssh_connection_button").visible = ( - display_ssh - ) + self.query_one( + "#configs_setup_ssh_connection_button" + ).visible = display_ssh if not self.query_one("#configs_central_path_input").value: if display_ssh: placeholder = f"e.g. {self.get_platform_dependent_example_paths('central', ssh=True)}" else: placeholder = f"e.g. {self.get_platform_dependent_example_paths('central', ssh=False)}" - self.query_one("#configs_central_path_input").placeholder = ( - placeholder - ) + self.query_one( + "#configs_central_path_input" + ).placeholder = placeholder def on_button_pressed(self, event: Button.Pressed) -> None: - """ - Enables the Create Folders button to read out current input values + """Enables the Create Folders button to read out current input values and use these to call project.create_folders(). """ if event.button.id == "configs_save_configs_button": @@ -366,36 +359,33 @@ def on_button_pressed(self, event: Button.Pressed) -> None: def handle_input_fill_from_select_directory( self, path_: Path, local_or_central: Literal["local", "central"] ) -> None: - """ - Update the `local` or `central` path inputs after + """Update the `local` or `central` path inputs after `SelectDirectoryTreeScreen` returns a path. Parameters ---------- - path_ The path returned from `SelectDirectoryTreeScreen`. If `False`, the screen exited with no directory selected. local_or_central The Input to fill with the path. + """ if path_ is False: return if local_or_central == "local": - self.query_one("#configs_local_path_input").value = ( - path_.as_posix() - ) + self.query_one( + "#configs_local_path_input" + ).value = path_.as_posix() elif local_or_central == "central": - self.query_one("#configs_central_path_input").value = ( - path_.as_posix() - ) + self.query_one( + "#configs_central_path_input" + ).value = path_.as_posix() def setup_ssh_connection(self) -> None: - """ - Set up the `SetupSshScreen` screen, - """ + """Set up the `SetupSshScreen` screen,""" assert self.interface is not None, "type narrow flexible `interface`" if not self.widget_configs_match_saved_configs(): @@ -410,8 +400,7 @@ def setup_ssh_connection(self) -> None: ) def widget_configs_match_saved_configs(self): - """ - Check that the configs currently stored in the widgets + """Check that the configs currently stored in the widgets on the screen match those stored in the app. This check is to avoid user starting to set up SSH with unexpected settings. It is a little fiddly as the Input for local @@ -433,8 +422,7 @@ def widget_configs_match_saved_configs(self): return True def setup_configs_for_a_new_project(self) -> None: - """ - If a project does not exist, we are in NewProjectScreen. + """If a project does not exist, we are in NewProjectScreen. We need to instantiate a new project based on the project name, create configs based on the current widget settings, and display any errors to the user, along with confirmation and the @@ -453,17 +441,15 @@ def setup_configs_for_a_new_project(self) -> None: success, output = interface.setup_new_project(project_name, cfg_kwargs) if success: - self.interface = interface - self.query_one("#configs_go_to_project_screen_button").visible = ( - True - ) + self.query_one( + "#configs_go_to_project_screen_button" + ).visible = True # Could not find a neater way to combine the push screen # while initiating the callback in one case but not the other. if cfg_kwargs["connection_method"] == "ssh": - self.query_one( "#configs_setup_ssh_connection_button" ).visible = True @@ -495,8 +481,7 @@ def setup_configs_for_a_new_project(self) -> None: self.parent_class.mainwindow.show_modal_error_dialog(output) def setup_configs_for_an_existing_project(self) -> None: - """ - If the project already exists, we are on the TabbedContent + """If the project already exists, we are on the TabbedContent screen. We need to get the configs to set from the current widget values and display the set values (or an error if there was a problem during setup) to the user. @@ -524,8 +509,7 @@ def setup_configs_for_an_existing_project(self) -> None: self.parent_class.mainwindow.show_modal_error_dialog(output) def fill_widgets_with_project_configs(self) -> None: - """ - If a configured project already exists, we want to fill the + """If a configured project already exists, we want to fill the widgets with the current project configs. This in some instances requires recasting to a new type of changing the value. @@ -587,8 +571,7 @@ def fill_widgets_with_project_configs(self) -> None: input.value = value def get_datashuttle_inputs_from_widgets(self) -> Dict: - """ - Get the configs to pass to `make_config_file()` from + """Get the configs to pass to `make_config_file()` from the current TUI settings. """ cfg_kwargs: Dict[str, Any] = {} diff --git a/datashuttle/tui/custom_widgets.py b/datashuttle/tui/custom_widgets.py index 5f5559e2e..7f2b2e12b 100644 --- a/datashuttle/tui/custom_widgets.py +++ b/datashuttle/tui/custom_widgets.py @@ -1,8 +1,8 @@ from __future__ import annotations +from collections.abc import Iterable from typing import ( TYPE_CHECKING, - Iterable, List, Optional, Tuple, @@ -38,8 +38,7 @@ # ClickableInput # -------------------------------------------------------------------------------------- class ClickableInput(Input): - """ - An input widget which emits a `ClickableInput.Clicked` + """An input widget which emits a `ClickableInput.Clicked` signal when clicked, containing the input name `input` and mouse button index `button`. """ @@ -86,10 +85,9 @@ def on_key(self, event: events.Key) -> None: class CustomDirectoryTree(DirectoryTree): - """ - Base class for directory tree with some customised additions: - - filter out top-level folders that are not canonical - - add additional keyboard shortcuts defined in `on_key`. + """Base class for directory tree with some customised additions: + - filter out top-level folders that are not canonical + - add additional keyboard shortcuts defined in `on_key`. """ @dataclass @@ -105,15 +103,13 @@ def __init__( self.mainwindow = mainwindow def filter_paths(self, paths: Iterable[Path]) -> Iterable[Path]: - """ - Filter out all hidden folders and files from DirectoryTree + """Filter out all hidden folders and files from DirectoryTree display. """ return [path for path in paths if not path.name.startswith(".")] def on_key(self, event: events.Key) -> None: - """ - Handle key presses on the CustomDirectoryTree. Depending on the keys pressed, + """Handle key presses on the CustomDirectoryTree. Depending on the keys pressed, copy the path under the cursor, refresh the directorytree or emit a DirectoryTreeSpecialKeyPress event. """ @@ -143,8 +139,7 @@ def on_key(self, event: events.Key) -> None: def _render_line( self, y: int, x1: int, x2: int, base_style: Style ) -> Strip: - """ - This function is overridden from textual's `Tree` class to stop + """This function is overridden from textual's `Tree` class to stop CSS styling on hovering and clicking which was distracting / changed the default color used for transfer status, respectively. @@ -200,6 +195,7 @@ def get_guides(style: Style) -> tuple[str, str, str, str]: Returns: Strings for space, vertical, terminator and cross. + """ lines: tuple[ Iterable[str], Iterable[str], Iterable[str], Iterable[str] @@ -287,8 +283,7 @@ def get_guides(style: Style) -> tuple[str, str, str, str]: class TreeAndInputTab(TabPane): - """ - A parent class that defined common methods for screens with + """A parent class that defined common methods for screens with a directory tree and sub / session inputs, .e. the Create tab and the Transfer tab. """ @@ -296,8 +291,7 @@ class TreeAndInputTab(TabPane): def handle_fill_input_from_directorytree( self, sub_input_key: str, ses_input_key: str, event: events.Key ) -> None: - """ - When a CustomDirectoryTree key is pressed, we typically + """When a CustomDirectoryTree key is pressed, we typically want to perform an action that involves an Input. These are coordinated here. Note that the 'copy' and 'refresh' features of the tree is handled at the level of the @@ -315,7 +309,6 @@ def handle_fill_input_from_directorytree( Parameters ---------- - sub_input_key The textual widget id for the subject input (prefixed with #) @@ -325,6 +318,7 @@ def handle_fill_input_from_directorytree( event A DirectoryTreeSpecialKeyPress event triggered from the CustomDirectoryTree. + """ if event.key == "ctrl+a": self.append_sub_or_ses_name_to_input( @@ -342,8 +336,7 @@ def handle_fill_input_from_directorytree( def insert_sub_or_ses_name_to_input( self, sub_input_key: str, ses_input_key: str, name: str ) -> None: - """ - see `handle_directorytree_key_pressed` for `sub_input_key` and + """See `handle_directorytree_key_pressed` for `sub_input_key` and `ses_input_key`. name @@ -357,9 +350,7 @@ def insert_sub_or_ses_name_to_input( def append_sub_or_ses_name_to_input( self, sub_input_key: str, ses_input_key: str, name: str ) -> None: - """ - see `insert_sub_or_ses_name_to_input`. - """ + """See `insert_sub_or_ses_name_to_input`.""" if name.startswith("sub-"): if not self.query_one(sub_input_key).value: self.query_one(sub_input_key).value = name @@ -375,9 +366,7 @@ def append_sub_or_ses_name_to_input( def get_sub_ses_names_and_datatype( self, sub_input_key: str, ses_input_key: str ) -> Tuple[List[str], List[str], List[str]]: - """ - see `handle_fill_input_from_directorytree` for parameters. - """ + """See `handle_fill_input_from_directorytree` for parameters.""" sub_names = self.query_one(sub_input_key).as_names_list() ses_names = self.query_one(ses_input_key).as_names_list() datatype = self.query_one("DatatypeCheckboxes").selected_datatypes() @@ -386,8 +375,7 @@ def get_sub_ses_names_and_datatype( class TopLevelFolderSelect(Select): - """ - A Select widget for display and updating of top-level-folders. The + """A Select widget for display and updating of top-level-folders. The Create tab and transfer tabs (custom, top-level-folder) all have top level folder selects that perform the same function. This widget unifies these in a single place. @@ -398,7 +386,6 @@ class TopLevelFolderSelect(Select): Parameters ---------- - existing_only If `True`, only top level folders that actually exist in the project are displayed. Otherwise, all possible canonical @@ -406,6 +393,7 @@ class TopLevelFolderSelect(Select): id Textualize widget id + """ def __init__(self, interface: Interface, id: str) -> None: @@ -440,8 +428,7 @@ def __init__(self, interface: Interface, id: str) -> None: ) def get_top_level_folder(self, init: bool = False) -> str: - """ - Get the top level folder from `persistent_settings`, + """Get the top level folder from `persistent_settings`, performing a confidence-check that it matches the textual display. """ top_level_folder = self.interface.tui_settings[ @@ -449,28 +436,24 @@ def get_top_level_folder(self, init: bool = False) -> str: ][self.settings_key] if not init: - assert ( - top_level_folder == self.get_displayed_top_level_folder() - ), "config and widget should never be out of sync." + assert top_level_folder == self.get_displayed_top_level_folder(), ( + "config and widget should never be out of sync." + ) return top_level_folder def get_displayed_top_level_folder(self) -> str: - """ - Get the top level folder that is currently selected + """Get the top level folder that is currently selected on the select widget. """ assert self.value in canonical_folders.get_top_level_folders() return self.value def on_select_changed(self, event: Select.Changed) -> None: - """ - When the select is changed, update the linked persistent setting. - """ + """When the select is changed, update the linked persistent setting.""" top_level_folder = event.value if event.value != Select.BLANK: - self.interface.save_tui_settings( top_level_folder, "top_level_folder_select", self.settings_key ) diff --git a/datashuttle/tui/interface.py b/datashuttle/tui/interface.py index fd3e7b2c4..8ede9cb0e 100644 --- a/datashuttle/tui/interface.py +++ b/datashuttle/tui/interface.py @@ -15,8 +15,7 @@ class Interface: - """ - An interface class between the TUI and datashuttle API. Takes input + """An interface class between the TUI and datashuttle API. Takes input to all datashuttle functions as passed from the TUI, outputs success status (True or False) and optional data, in the case of False. @@ -31,21 +30,19 @@ class Interface: """ def __init__(self) -> None: - self.project: DataShuttle self.name_templates: Dict = {} self.tui_settings: Dict = {} def select_existing_project(self, project_name: str) -> InterfaceOutput: - """ - Load an existing project into `self.project`. + """Load an existing project into `self.project`. Parameters ---------- - project_name The name of the datashuttle project to load. Must already exist. + """ try: project = DataShuttle(project_name, print_startup_message=False) @@ -58,17 +55,16 @@ def select_existing_project(self, project_name: str) -> InterfaceOutput: def setup_new_project( self, project_name: str, cfg_kwargs: Dict ) -> InterfaceOutput: - """ - Set up a new project and load into `self.project`. + """Set up a new project and load into `self.project`. Parameters ---------- - project_name Name of the project to set up. cfg_kwargs The configurations to set the new project to. + """ try: project = DataShuttle(project_name, print_startup_message=False) @@ -85,15 +81,14 @@ def setup_new_project( def set_configs_on_existing_project( self, cfg_kwargs: Dict ) -> InterfaceOutput: - """ - Update the settings on an existing project. Only the settings + """Update the settings on an existing project. Only the settings passed in `cfg_kwargs` are updated. Parameters ---------- - cfg_kwargs The configs and new values to update. + """ try: self.project.update_config_file(**cfg_kwargs) @@ -108,12 +103,10 @@ def create_folders( ses_names: Optional[List[str]], datatype: List[str], ) -> InterfaceOutput: - """ - Create folders through datashuttle. + """Create folders through datashuttle. Parameters ---------- - sub_names A list of un-formatted / unvalidated subject names to create. @@ -122,6 +115,7 @@ def create_folders( datatype A list of canonical datatype names to create. + """ top_level_folder = self.tui_settings["top_level_folder_select"][ "create_tab" @@ -144,8 +138,7 @@ def create_folders( def validate_names( self, sub_names: List[str], ses_names: Optional[List[str]] ) -> InterfaceOutput: - """ - Validate a list of subject / session names. This is used + """Validate a list of subject / session names. This is used to populate the Input tooltips with validation errors. Uses a central `_format_and_validate_names()` that is also called during folder creation itself, to ensure these a @@ -153,12 +146,12 @@ def validate_names( Parameters ---------- - sub_names List of subject names to format. ses_names List of session names to format. + """ top_level_folder = self.tui_settings["top_level_folder_select"][ "create_tab" @@ -185,15 +178,14 @@ def validate_names( # ---------------------------------------------------------------------------------- def transfer_entire_project(self, upload: bool) -> InterfaceOutput: - """ - Transfer the entire project (all canonical top-level folders). + """Transfer the entire project (all canonical top-level folders). Parameters ---------- - upload Upload from local to central if `True`, otherwise download from central to remote. + """ try: if upload: @@ -216,12 +208,10 @@ def transfer_entire_project(self, upload: bool) -> InterfaceOutput: def transfer_top_level_only( self, selected_top_level_folder: str, upload: bool ) -> InterfaceOutput: - """ - Transfer all files within a selected top level folder. + """Transfer all files within a selected top level folder. Parameters ---------- - selected_top_level_folder The top level folder selected in the TUI for this transfer window. @@ -266,12 +256,10 @@ def transfer_custom_selection( datatype: List[str], upload: bool, ) -> InterfaceOutput: - """ - Transfer a custom selection of subjects / sessions / datatypes. + """Transfer a custom selection of subjects / sessions / datatypes. Parameters ---------- - selected_top_level_folder The top level folder selected in the TUI for this transfer window. @@ -287,6 +275,7 @@ def transfer_custom_selection( upload Upload from local to central if `True`, otherwise download from central to remote. + """ try: if upload: @@ -314,8 +303,7 @@ def transfer_custom_selection( # ---------------------------------------------------------------------------------- def get_name_templates(self) -> Dict: - """ - Get the `name_templates` defining templates to validate + """Get the `name_templates` defining templates to validate against. These are stored in a variable to avoid constantly reading these values from disk where they are stored in `persistent_settings`. It is critical this variable @@ -328,8 +316,7 @@ def get_name_templates(self) -> Dict: return self.name_templates def set_name_templates(self, name_templates: Dict) -> InterfaceOutput: - """ - Set the `name_templates` here and on disk. See `get_name_templates` + """Set the `name_templates` here and on disk. See `get_name_templates` for more information. """ try: @@ -341,8 +328,7 @@ def set_name_templates(self, name_templates: Dict) -> InterfaceOutput: return False, str(e) def get_tui_settings(self) -> Dict: - """ - Get the "tui" field of `persistent_settings`. Similar to + """Get the "tui" field of `persistent_settings`. Similar to `get_name_templates`, there are held on the class to avoid constantly reading from disk. """ @@ -354,12 +340,10 @@ def get_tui_settings(self) -> Dict: def save_tui_settings( self, value: Any, key: str, key_2: Optional[str] = None ) -> None: - """ - Update the "tui" field of the `persistent_settings` on disk. + """Update the "tui" field of the `persistent_settings` on disk. Parameters ---------- - value Value to set the `persistent_settings` tui field to @@ -370,6 +354,7 @@ def save_tui_settings( key_2 Optionals second level of the dictionary to update. e.g. "create_tab" + """ if key_2 is None: self.tui_settings[key] = value @@ -388,8 +373,7 @@ def get_configs(self) -> Configs: return self.project.cfg def get_textual_compatible_project_configs(self) -> Configs: - """ - Datashuttle configs keeps paths saved as pathlib.Path + """Datashuttle configs keeps paths saved as pathlib.Path objects. In some cases textual requires str representation. This method returns datashuttle configs with all paths that are Path converted to str. @@ -414,7 +398,6 @@ def get_next_sub( def get_next_ses( self, top_level_folder: TopLevelFolder, sub: str ) -> InterfaceOutput: - try: next_ses = self.project.get_next_ses( top_level_folder, diff --git a/datashuttle/tui/screens/create_folder_settings.py b/datashuttle/tui/screens/create_folder_settings.py index b8228439f..26ae336ec 100644 --- a/datashuttle/tui/screens/create_folder_settings.py +++ b/datashuttle/tui/screens/create_folder_settings.py @@ -3,7 +3,6 @@ from typing import TYPE_CHECKING, Dict, Optional if TYPE_CHECKING: - from textual.app import ComposeResult from datashuttle.tui.app import App @@ -28,8 +27,7 @@ class CreateFoldersSettingsScreen(ModalScreen): - """ - This screen handles setting datashuttle's `name_template`'s, as well + """This screen handles setting datashuttle's `name_template`'s, as well as the top-level-folder select and option to bypass all validation. Name Templates @@ -46,10 +44,10 @@ class CreateFoldersSettingsScreen(ModalScreen): Attributes ---------- - Because the Input for `name_templates` is shared between subject and session, the values are held in the `input_values` attribute. These are loaded from `persistent_settings` on init. + """ TITLE = "Create Folders Settings" @@ -164,8 +162,7 @@ def switch_template_container_disabled(self) -> None: self.query_one("#template_inner_container").disabled = not is_on def fill_input_from_template(self) -> None: - """ - Fill the `name_templates` Input, that is shared + """Fill the `name_templates` Input, that is shared between subject and session, depending on the current radioset value. """ @@ -179,8 +176,7 @@ def fill_input_from_template(self) -> None: input.value = value def on_button_pressed(self, event: Button.Pressed) -> None: - """ - On close, update the `name_templates` stored in + """On close, update the `name_templates` stored in `persistent_settings` with those set on the TUI. Setting may error if templates are turned on but @@ -208,9 +204,7 @@ def make_name_templates_from_widgets(self) -> Dict: } def on_checkbox_changed(self, event: Checkbox.Changed) -> None: - """ - Turn `name_templates` on or off and update the TUI accordingly. - """ + """Turn `name_templates` on or off and update the TUI accordingly.""" is_on = event.value if event.checkbox.id == "template_settings_validation_on_checkbox": @@ -232,13 +226,12 @@ def on_checkbox_changed(self, event: Checkbox.Changed) -> None: disable_container = not self.query_one( "#template_settings_validation_on_checkbox" ).value - self.query_one("#template_inner_container").disabled = ( - disable_container - ) + self.query_one( + "#template_inner_container" + ).disabled = disable_container def on_radio_set_changed(self, event: RadioSet.Changed) -> None: - """ - Update the displayed SSH widgets when the `connection_method` + """Update the displayed SSH widgets when the `connection_method` radiobuttons are changed. """ label = str(event.pressed.label) diff --git a/datashuttle/tui/screens/datatypes.py b/datashuttle/tui/screens/datatypes.py index aa731761d..78899342e 100644 --- a/datashuttle/tui/screens/datatypes.py +++ b/datashuttle/tui/screens/datatypes.py @@ -3,7 +3,6 @@ from typing import TYPE_CHECKING, List, Literal, Optional if TYPE_CHECKING: - from textual.app import ComposeResult from datashuttle.tui.interface import Interface @@ -65,8 +64,7 @@ class DisplayedDatatypesScreen(ModalScreen): - """ - Screen to select the which datatype checkboxes to show on the Create / Transfer tabs. + """Screen to select the which datatype checkboxes to show on the Create / Transfer tabs. Display a SessionList widget which all canonical broad and narrow-type datatypes. When selected, this will update DatatypeCheckboxes (coordinates @@ -83,6 +81,7 @@ class DisplayedDatatypesScreen(ModalScreen): checkboxes are so close together. Testing indicate that when writing to file after each click, syncing could get messed up and the wrong checkboxes displayed on the window. + """ def __init__( @@ -102,8 +101,7 @@ def __init__( ) def compose(self) -> ComposeResult: - """ - Collect the datatypes names and status from + """Collect the datatypes names and status from the persistent settings and display. """ selections = [] @@ -146,8 +144,7 @@ def on_mount(self): # assert False, f"{dir(self.query_one('#displayed_datatypes_selection_list'))}" def on_button_pressed(self, event): - """ - When 'Save' is pressed, the configs copied on this class + """When 'Save' is pressed, the configs copied on this class are updated back onto the interface configs, and written to disk. Otherwise, close the screen without saving. """ @@ -163,8 +160,7 @@ def on_button_pressed(self, event): def on_selection_list_selection_toggled( self, event: SelectionList.SelectionMessage.SelectionToggled ): - """ - When a selection is toggled, update the configs with + """When a selection is toggled, update the configs with the 'displayed' status and save to disk. """ datatype_name = event.selection.prompt.plain @@ -182,13 +178,11 @@ def on_selection_list_selection_toggled( class DatatypeCheckboxes(Static): - """ - Dynamically-populated checkbox widget for convenient datatype + """Dynamically-populated checkbox widget for convenient datatype selection during folder creation. Parameters ---------- - settings_key 'create' if datatype checkboxes for the create tab, 'transfer' for the transfer tab. Transfer tab includes @@ -196,7 +190,6 @@ class DatatypeCheckboxes(Static): Attributes ---------- - datatype_config a Dictionary containing datatype as key (e.g. "ephys", "behav") and values are `bool` indicating whether the checkbox is on / off. @@ -210,6 +203,7 @@ class DatatypeCheckboxes(Static): however because this screen persists through the lifetime of the app there is no clear time point in which to save the checkbox status. Therefore, the configs are updated (written to disk) on each click. + """ def __init__( @@ -242,8 +236,7 @@ def compose(self) -> ComposeResult: @on(Checkbox.Changed) def on_checkbox_changed(self) -> None: - """ - When a checkbox is changed, update the `self.datatype_config` + """When a checkbox is changed, update the `self.datatype_config` to contain new boolean values for each datatype. Also update the stored `persistent_settings`. """ @@ -266,8 +259,7 @@ def on_mount(self) -> None: ).tooltip = tooltips[datatype] def selected_datatypes(self) -> List[str]: - """ - Get the names of the datatype options for which the + """Get the names of the datatype options for which the checkboxes are switched on. """ selected_datatypes = [ diff --git a/datashuttle/tui/screens/get_help.py b/datashuttle/tui/screens/get_help.py index e177b7d42..622f9474b 100644 --- a/datashuttle/tui/screens/get_help.py +++ b/datashuttle/tui/screens/get_help.py @@ -47,7 +47,6 @@ def action_link_zulip(self): webbrowser.open(links.get_link_zulip()) def compose(self) -> ComposeResult: - yield Container( Static(self.text, id="get_help_label"), Button("Main Menu", id="all_main_menu_buttons"), diff --git a/datashuttle/tui/screens/modal_dialogs.py b/datashuttle/tui/screens/modal_dialogs.py index 4d801e823..9625c0318 100644 --- a/datashuttle/tui/screens/modal_dialogs.py +++ b/datashuttle/tui/screens/modal_dialogs.py @@ -23,8 +23,7 @@ class MessageBox(ModalScreen): - """ - A screen for rendering error messages. + """A screen for rendering error messages. message The message to display in the message box @@ -70,8 +69,7 @@ def on_button_pressed(self) -> None: class ConfirmAndAwaitTransferPopup(ModalScreen): - """ - A popup screen for confirming, awaiting and finishing a Transfer. + """A popup screen for confirming, awaiting and finishing a Transfer. When users select Transfer, this screen pops up to a) allow users to confirm transfer b) display a `LoadingIndicator` while the transfer runs in a separate worker c) indicate the transfer is finished. @@ -117,7 +115,6 @@ def on_button_pressed(self, event: Button.Pressed) -> None: async def handle_transfer_and_update_ui_when_complete(self) -> None: """Runs the data transfer worker and updates the UI on completion""" - data_transfer_worker = self.transfer_func() await data_transfer_worker.wait() success, output = data_transfer_worker.result @@ -138,21 +135,20 @@ async def handle_transfer_and_update_ui_when_complete(self) -> None: class SelectDirectoryTreeScreen(ModalScreen): - """ - A modal screen that includes a DirectoryTree to browse + """A modal screen that includes a DirectoryTree to browse and select folders. If a folder is double-clicked, the path to the folder is returned through 'dismiss' callback mechanism. Parameters ---------- - mainwindow Textual main app screen path_ Path to use as the DirectoryTree root, if `None` set to the system user home. + """ def __init__(self, mainwindow: App, path_: Optional[Path] = None) -> None: @@ -166,7 +162,6 @@ def __init__(self, mainwindow: App, path_: Optional[Path] = None) -> None: self.prev_click_time = 0 def compose(self) -> ComposeResult: - label_message = ( "Select (double click) a folder with the same name as the project.\n" "If the project folder does not exist, select the parent folder and it will be created." diff --git a/datashuttle/tui/screens/new_project.py b/datashuttle/tui/screens/new_project.py index ea7c0826c..769629ff3 100644 --- a/datashuttle/tui/screens/new_project.py +++ b/datashuttle/tui/screens/new_project.py @@ -14,8 +14,7 @@ class NewProjectScreen(Screen): - """ - Screen for setting up a new datashuttle project, by + """Screen for setting up a new datashuttle project, by inputting the desired configs. This uses the ConfigsContent window to display and set the configs. @@ -30,9 +29,9 @@ class NewProjectScreen(Screen): Parameters ---------- - mainwindow The main TUI app + """ TITLE = "Make New Project" diff --git a/datashuttle/tui/screens/project_manager.py b/datashuttle/tui/screens/project_manager.py index 7d6343632..ebe0168e9 100644 --- a/datashuttle/tui/screens/project_manager.py +++ b/datashuttle/tui/screens/project_manager.py @@ -22,8 +22,7 @@ class ProjectManagerScreen(Screen): - """ - Screen containing the Create, Transfer and Configs tabs. This is + """Screen containing the Create, Transfer and Configs tabs. This is the primary screen within which the user interacts with a pre-configured project. @@ -85,8 +84,7 @@ def compose(self) -> ComposeResult: ) def on_button_pressed(self, event: Button.Pressed) -> None: - """ - Dismisses the TabScreen (and returns to the main menu) once + """Dismisses the TabScreen (and returns to the main menu) once the 'Main Menu' button is pressed. """ if event.button.id == "all_main_menu_buttons": @@ -95,8 +93,7 @@ def on_button_pressed(self, event: Button.Pressed) -> None: def on_tabbed_content_tab_activated( self, event: TabbedContent.TabActivated ) -> None: - """ - Refresh the directorytree for create or transfer tabs whenever + """Refresh the directorytree for create or transfer tabs whenever the tabbedcontent is switched to one of these tabs. This is also triggered on mount, leading to it being reloaded @@ -125,8 +122,7 @@ def update_active_tab_tree(self): self.query_one(f"#{active_tab_id}").reload_directorytree() def on_configs_content_configs_saved(self) -> None: - """ - When configs are saved, we may switch between a 'full' project + """When configs are saved, we may switch between a 'full' project and a 'local only' project (no `central_path` or `connection_method` set). In such a case we need to refresh the ProjectManager screen to add / remove the transfer tab. @@ -148,7 +144,6 @@ def on_configs_content_configs_saved(self) -> None: ) if old_project_type == project_type: - if project_type == "full": self.query_one( "#tabscreen_transfer_tab" @@ -167,8 +162,7 @@ def on_configs_content_configs_saved(self) -> None: ) def wrap_dismiss(self, _): - """ - Need to wrap dismiss as cannot include it directly + """Need to wrap dismiss as cannot include it directly in push_screen callback, or even wrapped in lambda. """ self.dismiss() diff --git a/datashuttle/tui/screens/project_selector.py b/datashuttle/tui/screens/project_selector.py index 289990e5c..3bd923570 100644 --- a/datashuttle/tui/screens/project_selector.py +++ b/datashuttle/tui/screens/project_selector.py @@ -18,8 +18,7 @@ class ProjectSelectorScreen(Screen): - """ - The project selection screen. Finds and displays DataShuttle + """The project selection screen. Finds and displays DataShuttle projects present on the local system. `self.dismiss()` returns an initialised project if initialisation @@ -28,7 +27,6 @@ class ProjectSelectorScreen(Screen): Parameters ---------- - mainwindow The main TUI app, functions on which are used to coordinate screen display. @@ -55,7 +53,6 @@ def compose(self) -> ComposeResult: def on_button_pressed(self, event: Button.Pressed) -> None: if event.button.id in self.project_names: - project_name = event.button.id interface = Interface() diff --git a/datashuttle/tui/screens/settings.py b/datashuttle/tui/screens/settings.py index 9f14d9ab4..0634ea041 100644 --- a/datashuttle/tui/screens/settings.py +++ b/datashuttle/tui/screens/settings.py @@ -20,8 +20,7 @@ class SettingsScreen(ModalScreen): - """ - Screen accessible from the main window that contains + """Screen accessible from the main window that contains 'global' settings for the TUI. 'Global' settings are non-project specific settings (e.g. dark mode) and are handled independently of the main datashuttle API. diff --git a/datashuttle/tui/screens/setup_ssh.py b/datashuttle/tui/screens/setup_ssh.py index 8d1721a13..7502caadb 100644 --- a/datashuttle/tui/screens/setup_ssh.py +++ b/datashuttle/tui/screens/setup_ssh.py @@ -18,8 +18,7 @@ class SetupSshScreen(ModalScreen): - """ - This dialog windows handles the TUI equivalent of API's + """This dialog windows handles the TUI equivalent of API's setup_ssh_connection(). This asks to confirm the central hostkey, and takes password to setup SSH key pair. @@ -42,7 +41,7 @@ def compose(self) -> ComposeResult: yield Container( Horizontal( Static( - "Ready to setup setup SSH. " "Press OK to proceed.", + "Ready to setup setup SSH. Press OK to proceed.", id="messagebox_message_label", ), id="messagebox_message_container", @@ -60,8 +59,7 @@ def on_mount(self) -> None: self.query_one("#setup_ssh_password_input").visible = False def on_button_pressed(self, event: Button.pressed) -> None: - """ - When each stage is successfully progressed by clicking the "ok" button, + """When each stage is successfully progressed by clicking the "ok" button, `self.stage` is iterated by 1. For saving and excepting hostkey, if there is a problem (error or user declines) the 'OK' button is frozen so it is not possible to proceed. For accepting password @@ -84,8 +82,7 @@ def on_button_pressed(self, event: Button.pressed) -> None: self.dismiss() def ask_user_to_accept_hostkeys(self) -> None: - """ - The central server is identified by a hostkey. + """The central server is identified by a hostkey. Get this hostkey and present it to user, clicking 'OK' is they are happy. If there is an error, block process (because it most likely is necessary to edit the central host id) and @@ -116,8 +113,7 @@ def ask_user_to_accept_hostkeys(self) -> None: self.stage += 1 def save_hostkeys_and_prompt_password_input(self) -> None: - """ - Once the hostkey is accepted, get the user password + """Once the hostkey is accepted, get the user password for the central server. When 'OK' is pressed we go straight to 'use_password_to_setup_ssh_key_pairs'. """ @@ -141,8 +137,7 @@ def save_hostkeys_and_prompt_password_input(self) -> None: self.stage += 1 def use_password_to_setup_ssh_key_pairs(self) -> None: - """ - Get the user password for the central server. If correct, + """Get the user password for the central server. If correct, SSH key pair is setup and 'OK' button changed to 'Finish'. Otherwise, continue allowing failed password attempts. """ diff --git a/datashuttle/tui/tabs/create_folders.py b/datashuttle/tui/tabs/create_folders.py index 1e3ef7de8..7b98eb6db 100644 --- a/datashuttle/tui/tabs/create_folders.py +++ b/datashuttle/tui/tabs/create_folders.py @@ -35,9 +35,7 @@ class CreateFoldersTab(TreeAndInputTab): - """ - Create new project files formatted according to the NeuroBlueprint specification. - """ + """Create new project files formatted according to the NeuroBlueprint specification.""" def __init__(self, mainwindow: App, interface: Interface) -> None: super(CreateFoldersTab, self).__init__( @@ -110,8 +108,7 @@ def on_mount(self) -> None: self.query_one(id).tooltip = get_tooltip(id) def on_button_pressed(self, event: Button.Pressed) -> None: - """ - Enables the Create Folders button to read out current input values + """Enables the Create Folders button to read out current input values and use these to call project.create_folders(). `unused_bool` is necessary to get dismiss to call @@ -121,7 +118,6 @@ def on_button_pressed(self, event: Button.Pressed) -> None: self.create_folders() elif event.button.id == "create_folders_displayed_datatypes_button": - self.mainwindow.push_screen( DisplayedDatatypesScreen("create", self.interface), self.refresh_after_datatypes_changed, @@ -141,8 +137,7 @@ async def refresh_after_datatypes_changed(self, ignore): def on_clickable_input_clicked( self, event: ClickableInput.Clicked ) -> None: - """ - Handled a double click on the custom ClickableInput widget, + """Handled a double click on the custom ClickableInput widget, which indicates the input should be filled with a suggested value. Determine if we have the subject or session input, and @@ -161,8 +156,7 @@ def on_clickable_input_clicked( def on_custom_directory_tree_directory_tree_special_key_press( self, event: CustomDirectoryTree.DirectoryTreeSpecialKeyPress ): - """ - Handle a key press on the directory tree, which can refresh the + """Handle a key press on the directory tree, which can refresh the directorytree or fill / append subject/session folder name to the relevant input widget. """ @@ -180,8 +174,7 @@ def on_custom_directory_tree_directory_tree_special_key_press( self.mainwindow.prompt_rename_file_or_folder(event.node_path) def fill_input_with_template(self, prefix: Prefix, input_id: str) -> None: - """ - Given the `name_template`, fill the sub or ses + """Given the `name_template`, fill the sub or ses Input with the template (based on `prefix`). If `self.templates` is off, then just suggest "sub-" or "ses-". """ @@ -203,8 +196,7 @@ def templates_on(self, prefix: Prefix) -> bool: # ---------------------------------------------------------------------------------- def revalidate_inputs(self, all_prefixes: List[str]) -> None: - """ - Revalidate and style both subject and session + """Revalidate and style both subject and session inputs based on their value. """ input_names = { @@ -218,8 +210,7 @@ def revalidate_inputs(self, all_prefixes: List[str]) -> None: self.query_one(key).validate(value=value) def update_input_tooltip(self, message: List[str], prefix: Prefix) -> None: - """ - Update the value of a subject or session tooltip, which + """Update the value of a subject or session tooltip, which indicates the validation status of the input value. """ id = ( @@ -238,8 +229,7 @@ def update_input_tooltip(self, message: List[str], prefix: Prefix) -> None: # ---------------------------------------------------------------------------------- def create_folders(self) -> None: - """ - Create project folders based on current widget input + """Create project folders based on current widget input through the datashuttle API. """ ses_names: Optional[List[str]] @@ -261,8 +251,7 @@ def create_folders(self) -> None: self.mainwindow.show_modal_error_dialog(output) def reload_directorytree(self) -> None: - """ - This reloads the directorytree and also updates validation. + """This reloads the directorytree and also updates validation. Not now a good method name but done for consistency with other tab refresh methods. """ @@ -275,8 +264,7 @@ def reload_directorytree(self) -> None: def fill_input_with_next_sub_or_ses_template( self, prefix: Prefix, input_id: str ) -> None: - """ - This fills a sub / ses Input with a suggested name based on the + """This fills a sub / ses Input with a suggested name based on the next subject / session in the project (local). If `name_templates` are set, then the sub- or ses- first key @@ -286,12 +274,12 @@ def fill_input_with_next_sub_or_ses_template( Parameters ---------- - prefix Whether to fill the subject or session Input input_id The textual input name to update. + """ top_level_folder = self.interface.tui_settings[ "top_level_folder_select" @@ -345,8 +333,7 @@ def fill_input_with_next_sub_or_ses_template( input.value = fill_value def run_local_validation(self, prefix: Prefix): - """ - Run validation of the values stored in the + """Run validation of the values stored in the sub / ses Input according to the passed prefix using core datashuttle functions. @@ -367,9 +354,9 @@ def run_local_validation(self, prefix: Prefix): Parameters ---------- - prefix Whether to run validation on the subject or session Input + """ sub_names = self.query_one( "#create_folders_subject_input" @@ -397,7 +384,5 @@ def run_local_validation(self, prefix: Prefix): return True, f"Formatted names: {names}" def update_directorytree_root(self, new_root_path: Path) -> None: - """ - Will automatically refresh the tree through the reactive attribute `path`. - """ + """Will automatically refresh the tree through the reactive attribute `path`.""" self.query_one("#create_folders_directorytree").path = new_root_path diff --git a/datashuttle/tui/tabs/logging.py b/datashuttle/tui/tabs/logging.py index fb45c8269..f35c7a987 100644 --- a/datashuttle/tui/tabs/logging.py +++ b/datashuttle/tui/tabs/logging.py @@ -16,7 +16,7 @@ class RichLogScreen(ModalScreen): def __init__(self, log_file): super(RichLogScreen, self).__init__() - with open(log_file, "r") as file: + with open(log_file) as file: self.log_contents = "".join(file.readlines()) def compose(self): diff --git a/datashuttle/tui/tabs/transfer.py b/datashuttle/tui/tabs/transfer.py index 4603c6674..b50a14b08 100644 --- a/datashuttle/tui/tabs/transfer.py +++ b/datashuttle/tui/tabs/transfer.py @@ -43,15 +43,13 @@ class TransferTab(TreeAndInputTab): - """ - This tab handles the upload / download of files between local + """This tab handles the upload / download of files between local and central folders. It contains a TransferDirectoryTree that displays the transfer status of the files in the local folder, and calls underlying datashuttle transfer functions. Parameters ---------- - title The title of the tab @@ -66,7 +64,6 @@ class TransferTab(TreeAndInputTab): Attributes ---------- - show_legend Convenience attribute linked to a global setting exists that turns off / on styling of directorytree nodes based on transfer status. ` @@ -76,6 +73,7 @@ class TransferTab(TreeAndInputTab): ]` When on, the legend must be hidden. + """ def __init__( @@ -210,7 +208,6 @@ def compose(self) -> ComposeResult: yield Label("â­• Legend", id="transfer_legend") def on_mount(self) -> None: - for id in [ "#transfer_directorytree", "#transfer_switch_container", @@ -261,8 +258,7 @@ def on_checkbox_changed(self, event: Checkbox.Changed) -> None: # ---------------------------------------------------------------------------------- def switch_transfer_widgets_display(self) -> None: - """ - Show or hide transfer parameters based on whether the transfer mode + """Show or hide transfer parameters based on whether the transfer mode currently selected in `transfer_radioset`. """ for widget in self.transfer_all_widgets: @@ -279,8 +275,7 @@ def switch_transfer_widgets_display(self) -> None: ).value def on_radio_set_changed(self, event: RadioSet.Changed) -> None: - """ - Update the displayed transfer parameter widgets when the + """Update the displayed transfer parameter widgets when the `transfer_radioset` radiobuttons are changed. """ label = str(event.pressed.label) @@ -288,8 +283,7 @@ def on_radio_set_changed(self, event: RadioSet.Changed) -> None: self.switch_transfer_widgets_display() def on_button_pressed(self, event: Button.Pressed) -> None: - """ - If the Transfer button is clicked, opens a modal dialog + """If the Transfer button is clicked, opens a modal dialog to confirm that the user wishes to transfer their data (in the direction selected). If "Yes" is selected, `self.transfer_data` (see below) is run. @@ -347,8 +341,7 @@ def reload_directorytree(self) -> None: self.query_one("#transfer_directorytree").update_transfer_tree() def update_directorytree_root(self, new_root_path: Path) -> None: - """ - This will automatically refresh the tree through the + """This will automatically refresh the tree through the reactive variable `path`. """ self.query_one("#transfer_directorytree").path = new_root_path @@ -358,8 +351,7 @@ def update_directorytree_root(self, new_root_path: Path) -> None: @work(exclusive=True, thread=True) def transfer_data(self) -> Worker[InterfaceOutput]: - """ - A threaded worker to transfer data + """A threaded worker to transfer data This function transfers data based on the config provided by the radio buttons such as a) the data to be transferred (all / top-level-folders / custom) b) the @@ -375,7 +367,6 @@ def transfer_data(self) -> Worker[InterfaceOutput]: success, output = self.interface.transfer_entire_project(upload) elif self.query_one("#transfer_toplevel_radiobutton").value: - selected_top_level_folder = self.query_one( "#transfer_toplevel_select" ).get_top_level_folder() @@ -385,7 +376,6 @@ def transfer_data(self) -> Worker[InterfaceOutput]: ) elif self.query_one("#transfer_custom_radiobutton").value: - selected_top_level_folder = self.query_one( "#transfer_custom_select" ).get_top_level_folder() diff --git a/datashuttle/tui/tabs/transfer_status_tree.py b/datashuttle/tui/tabs/transfer_status_tree.py index dbcaa205c..1d4410fc7 100644 --- a/datashuttle/tui/tabs/transfer_status_tree.py +++ b/datashuttle/tui/tabs/transfer_status_tree.py @@ -3,7 +3,6 @@ from typing import TYPE_CHECKING, Dict, Optional if TYPE_CHECKING: - from rich.style import Style from textual.widgets._directory_tree import DirEntry @@ -24,23 +23,21 @@ class TransferStatusTree(CustomDirectoryTree): - """ - A directorytree in which the nodes are styled depending on their + """A directorytree in which the nodes are styled depending on their transfer status. e.g. indicates whether files are changed between local or central, or appear in local only. Attributes ---------- - Keep the local path as a string, linked to project.cfg["local_path"], so that no conversion to string is necessary in `format_transfer_label` which is called many times. + """ def __init__( self, mainwindow: App, interface: Interface, id: Optional[str] = None ): - self.interface = interface self.local_path_str = self.interface.get_configs()[ "local_path" @@ -55,8 +52,7 @@ def on_mount(self) -> None: self.update_transfer_tree(init=True) def update_transfer_tree(self, init: bool = False) -> None: - """ - Updates tree styling to reflect the current TUI state + """Updates tree styling to reflect the current TUI state and project transfer status. """ self.local_path_str = self.interface.get_configs()[ @@ -72,9 +68,7 @@ def update_transfer_tree(self, init: bool = False) -> None: self.reload() def update_local_transfer_paths(self) -> None: - """ - Compiles a list of all project files and paths. - """ + """Compiles a list of all project files and paths.""" paths_list = [] for top_level_folder in canonical_folders.get_top_level_folders(): @@ -88,9 +82,7 @@ def update_local_transfer_paths(self) -> None: self.transfer_paths = paths_list def update_transfer_diffs(self) -> None: - """ - Updates the transfer diffs used to style the DirectoryTree. - """ + """Updates the transfer diffs used to style the DirectoryTree.""" self.transfer_diffs = get_local_and_central_file_differences( self.interface.get_configs(), top_level_folders_to_check=["rawdata", "derivatives"], @@ -102,8 +94,7 @@ def update_transfer_diffs(self) -> None: def render_label( self, node: TreeNode[DirEntry], base_style: Style, style: Style ) -> Text: - """ - Extends the `DirectoryTree.render_label()` method to allow + """Extends the `DirectoryTree.render_label()` method to allow custom styling of file nodes according to their transfer status. """ node_label = node._label.copy() @@ -145,8 +136,7 @@ def render_label( return text def format_transfer_label(self, node_label, node_path) -> None: - """ - Takes nodes being formatted using `render_label` and applies custom + """Takes nodes being formatted using `render_label` and applies custom formatting according to the node's transfer status. """ node_relative_path = node_path.as_posix().replace( diff --git a/datashuttle/tui/tooltips.py b/datashuttle/tui/tooltips.py index 61cb9f70d..21a6558f7 100644 --- a/datashuttle/tui/tooltips.py +++ b/datashuttle/tui/tooltips.py @@ -1,6 +1,5 @@ def get_tooltip(id: str) -> str: - """ - Master function to get tooltips for all widgets, + """Master function to get tooltips for all widgets, based on their widget (textual) id. """ # Configs diff --git a/datashuttle/tui/utils/tui_decorators.py b/datashuttle/tui/utils/tui_decorators.py index f2a96d41f..6304c9054 100644 --- a/datashuttle/tui/utils/tui_decorators.py +++ b/datashuttle/tui/utils/tui_decorators.py @@ -9,8 +9,7 @@ def require_double_click(func): - """ - A decorator that calls the decorated function + """A decorator that calls the decorated function on a double click, otherwise will not do anything. Requires the first argument (`self` on the class) to diff --git a/datashuttle/tui/utils/tui_validators.py b/datashuttle/tui/utils/tui_validators.py index 792fc1c5e..1605c63ba 100644 --- a/datashuttle/tui/utils/tui_validators.py +++ b/datashuttle/tui/utils/tui_validators.py @@ -1,6 +1,4 @@ -""" -Tools for live validation of user inputs in the DataShuttle TUI. -""" +"""Tools for live validation of user inputs in the DataShuttle TUI.""" from __future__ import annotations @@ -15,8 +13,7 @@ class NeuroBlueprintValidator(Validator): def __init__(self, prefix: Prefix, parent: CreateFoldersTab) -> None: - """ - Custom Validator() class that takes + """Custom Validator() class that takes sub / ses prefix as input. Runs validation of the name against the project and propagates any error message through the Input tooltip. @@ -26,8 +23,7 @@ def __init__(self, prefix: Prefix, parent: CreateFoldersTab) -> None: self.prefix = prefix def validate(self, name: str) -> ValidationResult: - """ - Run validation and update the tooltip with the error, + """Run validation and update the tooltip with the error, if no error then the formatted sub / ses name is displayed. This is set on an Input widget. """ diff --git a/datashuttle/tui_launcher.py b/datashuttle/tui_launcher.py index fb2e05ff9..f98f4b0be 100644 --- a/datashuttle/tui_launcher.py +++ b/datashuttle/tui_launcher.py @@ -29,9 +29,7 @@ def main() -> None: - """ - Launch the datashuttle tui. - """ + """Launch the datashuttle tui.""" args = parser.parse_args() if args.launch == "launch": diff --git a/datashuttle/utils/data_transfer.py b/datashuttle/utils/data_transfer.py index a39e6377f..10562f300 100644 --- a/datashuttle/utils/data_transfer.py +++ b/datashuttle/utils/data_transfer.py @@ -12,8 +12,7 @@ class TransferData: - """ - Class to perform data transfers. This works by first building + """Class to perform data transfers. This works by first building a large list of all files to transfer. Then, rclone is called once with this list to perform the transfer. @@ -23,7 +22,6 @@ class TransferData: Parameters ---------- - cfg datashuttle configs UserDict. @@ -58,6 +56,7 @@ class TransferData: log if `True`, log and print the transfer output. + """ def __init__( @@ -112,8 +111,7 @@ def __init__( # ------------------------------------------------------------------------- def build_a_list_of_all_files_and_folders_to_transfer(self) -> List[str]: - """ - Build a list of every file to transfer based on the user-passed + """Build a list of every file to transfer based on the user-passed arguments. This cycles through every subject, session and datatype and adds the outputs to three lists: @@ -123,9 +121,9 @@ def build_a_list_of_all_files_and_folders_to_transfer(self) -> List[str]: Returns ------- - include_list A list of paths to pass to rclone's `--include` flag. + """ # Find sub names to transfer processed_sub_names = self.get_processed_names(self.sub_names) @@ -186,8 +184,7 @@ def build_a_list_of_all_files_and_folders_to_transfer(self) -> List[str]: def make_include_arg( self, list_of_paths: List[str], recursive: bool = True ) -> List[str]: - """ - Format the list of paths to rclone's required + """Format the list of paths to rclone's required `--include` flag format. """ if not any(list_of_paths): @@ -212,8 +209,7 @@ def include_arg(ele: str) -> str: def update_list_with_non_sub_top_level_folders( self, extra_folder_names: List[str], extra_filenames: List[str] ) -> None: - """ - Search the subject level for all files and folders in the + """Search the subject level for all files and folders in the top-level-folder. Split the output based onto files / folders within "sub-" prefixed folders or not. """ @@ -240,8 +236,7 @@ def update_list_with_non_ses_sub_level_folders( extra_filenames: List[str], sub: str, ) -> None: - """ - For the subject, get a list of files / folders that are + """For the subject, get a list of files / folders that are not within "ses-" prefixed folders. """ sub_level_folders: List[str] @@ -277,8 +272,7 @@ def update_list_with_non_dtype_ses_level_folders( sub: str, ses: str, ) -> None: - """ - For a specific subject and session, get a list of files / folders + """For a specific subject and session, get a list of files / folders that are not in canonical datashuttle datatype folders. """ ses_level_folders: List[str] @@ -322,8 +316,7 @@ def update_list_with_dtype_paths( sub: str, ses: Optional[str] = None, ) -> None: - """ - Given a particular subject and session, get a list of all + """Given a particular subject and session, get a list of all canonical datatype folders. """ datatype = list(filter(lambda x: x != "all_non_datatype", datatype)) @@ -360,8 +353,7 @@ def to_list(self, names: Union[str, List[str]]) -> List[str]: def check_input_arguments( self, ) -> None: - """ - Check the sub / session names passed. The checking here + """Check the sub / session names passed. The checking here is stricter than for create_folders / formatting.check_and_format_names because we want to ensure that a) non-datatype arguments are not passed at the wrong input (e.g. all_non_ses as a subject name). @@ -373,8 +365,8 @@ def check_input_arguments( Parameters ---------- - see update_list_with_dtype_paths() + """ if len(self.sub_names) > 1 and any( [name in ["all", "all_sub"] for name in self.sub_names] @@ -422,8 +414,7 @@ def get_processed_names( names_checked: List[str], sub: Optional[str] = None, ) -> List[str]: - """ - Process the list of subject session names. + """Process the list of subject session names. If they are pre-defined (e.g. ["sub-001", "sub-002"]) they will be checked and formatted as per formatting.check_and_format_names() and @@ -436,7 +427,6 @@ def get_processed_names( Parameters ---------- - see transfer_sub_ses_data() """ @@ -478,8 +468,7 @@ def get_processed_names( return processed_names def transfer_non_datatype(self, datatype_checked: List[str]) -> bool: - """ - Convenience function, bool if all non-datatype folders + """Convenience function, bool if all non-datatype folders are to be transferred """ return any( diff --git a/datashuttle/utils/decorators.py b/datashuttle/utils/decorators.py index cacf54914..99dcde8ed 100644 --- a/datashuttle/utils/decorators.py +++ b/datashuttle/utils/decorators.py @@ -5,8 +5,7 @@ def requires_ssh_configs(func): - """ - Decorator to check file is loaded. Used on Mainwindow class + """Decorator to check file is loaded. Used on Mainwindow class methods only as first arg is assumed to be self (containing cfgs) """ @@ -29,8 +28,7 @@ def wrapper(*args, **kwargs): def check_configs_set(func): - """ - Check that configs have been loaded (i.e. + """Check that configs have been loaded (i.e. project.cfg is not None) before the func is run. """ @@ -50,8 +48,7 @@ def wrapper(*args, **kwargs): def check_is_not_local_project(func): - """ - Decorator to check that the project is not + """Decorator to check that the project is not a local project. If it is, raise. This decorator should be placed above methods which diff --git a/datashuttle/utils/ds_logger.py b/datashuttle/utils/ds_logger.py index 3d0ac7b69..353fc0886 100644 --- a/datashuttle/utils/ds_logger.py +++ b/datashuttle/utils/ds_logger.py @@ -38,9 +38,7 @@ def start( variables: Optional[List[Any]], verbose: bool = True, ) -> None: - """ - Call fancylog to initialise logging. - """ + """Call fancylog to initialise logging.""" filename = get_logging_filename(command_name) fancylog.start_logging( @@ -60,8 +58,7 @@ def start( def get_logging_filename(command_name: str) -> str: - """ - Get the filename to which the log will be saved. This + """Get the filename to which the log will be saved. This starts with ISO8601-formatted datetime, so logs are stored in datetime order. """ @@ -70,26 +67,24 @@ def get_logging_filename(command_name: str) -> str: def log_names(list_of_headers: List[Any], list_of_names: List[Any]) -> None: - """ - Log a list of subject or session names. + """Log a list of subject or session names. Parameters ---------- - list_of_headers a list of titles that the names will be printed under, e.g. "sub_names", "ses_names" list_of_names list of names to print to log + """ for header, names in zip(list_of_headers, list_of_names): utils.log(f"{header}: {names}") def wrap_variables_for_fancylog(local_vars: dict, cfg: Configs) -> List: - """ - Wrap the locals from the original function call to log + """Wrap the locals from the original function call to log and the datashuttle.cfg in a wrapper class with __dict__ attribute for fancylog writing. @@ -110,9 +105,7 @@ def __init__(self, local_vars_, cfg_): def close_log_filehandler() -> None: - """ - Remove handlers from all loggers. - """ + """Remove handlers from all loggers.""" logger = get_logger() logger.debug("Finished logging.") handlers = logger.handlers[:] diff --git a/datashuttle/utils/folder_class.py b/datashuttle/utils/folder_class.py index 6ada726c2..8b85856de 100644 --- a/datashuttle/utils/folder_class.py +++ b/datashuttle/utils/folder_class.py @@ -1,6 +1,5 @@ class Folder: - """ - Folder class used to contain details of canonical + """Folder class used to contain details of canonical folders in the project folder tree. see configs.canonical_folders.py for details. diff --git a/datashuttle/utils/folders.py b/datashuttle/utils/folders.py index 33d470088..f7d86c4b7 100644 --- a/datashuttle/utils/folders.py +++ b/datashuttle/utils/folders.py @@ -36,8 +36,7 @@ def create_folder_trees( datatype: Union[List[str], str], log: bool = True, ) -> Dict[str, List[Path]]: - """ - Entry method to make a full folder tree. It will + """Entry method to make a full folder tree. It will iterate through all passed subjects, then sessions, then subfolders within a datatype folder. This permits flexible creation of folders (e.g. @@ -48,13 +47,13 @@ def create_folder_trees( Parameters ---------- - sub_names, ses_names, datatype see create_folders() log whether to log or not. If True, logging must already be initialised. + """ datatype_passed = datatype not in [[""], ""] @@ -123,14 +122,12 @@ def make_datatype_folders( save_paths: Dict, log: bool = True, ): - """ - Make datatype folder (e.g. behav) at the sub or ses + """Make datatype folder (e.g. behav) at the sub or ses level. Checks folder_class.Folders attributes, whether the datatype is used and at the current level. Parameters ---------- - cfg datashuttle configs @@ -154,12 +151,12 @@ def make_datatype_folders( log whether to log on or not (if True, logging must already be initialised). + """ datatype_items = cfg.get_datatype_as_dict_items(datatype) for datatype_key, datatype_folder in datatype_items: # type: ignore if datatype_folder.level == level: - datatype_name = datatype_folder.name datatype_path = sub_or_ses_level_path / datatype_name @@ -177,19 +174,18 @@ def make_datatype_folders( def create_folders(paths: Union[Path, List[Path]], log: bool = True) -> None: - """ - For path or list of paths, make them if + """For path or list of paths, make them if they do not already exist. Parameters ---------- - paths Path or list of Paths to create log if True, log all made folders. This requires the logger to already be initialised. + """ if isinstance(paths, Path): paths = [paths] @@ -217,8 +213,7 @@ def search_project_for_sub_or_ses_names( include_central: bool, return_full_path: bool = False, ) -> Dict: - """ - If sub is None, the top-level level folder will be + """If sub is None, the top-level level folder will be searched (i.e. for subjects). The search string "sub-*" is suggested in this case. Otherwise, the subject, level folder for the specified subject will be searched. The search_str "ses-*" is suggested in this case. @@ -228,7 +223,6 @@ def search_project_for_sub_or_ses_names( will be searched for on central, showing a confusing 'folder not found' message. """ - # Search local and central for folders that begin with "sub-*" local_foldernames, _ = search_sub_or_ses_level( cfg, @@ -270,15 +264,14 @@ def items_from_datatype_input( sub: str, ses: Optional[str] = None, ) -> Union[ItemsView, zip]: - """ - Get the list of datatypes to transfer, either + """Get the list of datatypes to transfer, either directly from user input, or by searching what is available if "all" is passed. Parameters ---------- - see _transfer_datatype() for parameters. + """ base_folder = cfg.get_base_folder(local_or_central, top_level_folder) @@ -310,8 +303,7 @@ def search_for_datatype_folders( sub: str, ses: Optional[str] = None, ) -> zip: - """ - Search a subject or session folder specifically + """Search a subject or session folder specifically for datatypes. First searches for all folders / files in the folder, and then returns any folders that match datatype name. @@ -323,6 +315,7 @@ def search_for_datatype_folders( ------- Find the datatype files and return in a format that mirrors dict.items() + """ search_results = search_sub_or_ses_level( cfg, base_folder, local_or_central, sub, ses @@ -339,8 +332,7 @@ def process_glob_to_find_datatype_folders( folder_names: list, datatype_folders: dict, ) -> zip: - """ - Process the results of glob on a sub or session level, + """Process the results of glob on a sub or session level, which could contain any kind of folder / file. see project.search_sub_or_ses_level() for inputs. @@ -349,6 +341,7 @@ def process_glob_to_find_datatype_folders( ------- Find the datatype files and return in a format that mirrors dict.items() + """ ses_folder_keys = [] ses_folder_values = [] @@ -377,8 +370,7 @@ def search_for_wildcards( all_names: List[str], sub: Optional[str] = None, ) -> List[str]: - """ - Handle wildcard flag in upload or download. + """Handle wildcard flag in upload or download. All names in name are searched for @*@ string, and replaced with single * for glob syntax. If sub is passed, it is @@ -393,7 +385,6 @@ def search_for_wildcards( Parameters ---------- - project initialised datashuttle project @@ -457,13 +448,11 @@ def search_sub_or_ses_level( verbose: bool = True, return_full_path: bool = False, ) -> Tuple[List[str] | List[Path], List[str]]: - """ - Search project folder at the subject or session level. + """Search project folder at the subject or session level. Only returns folders Parameters ---------- - cfg datashuttle project cfg. Currently, this is used as a holder for ssh configs to avoid too many @@ -490,11 +479,11 @@ def search_sub_or_ses_level( verbose If `True`, if a search folder cannot be found, a message will be printed with the un-found path. + """ if ses and not sub: utils.log_and_raise_error( - "cannot pass session to " - "search_sub_or_ses_level() without subject", + "cannot pass session to search_sub_or_ses_level() without subject", ValueError, ) @@ -524,13 +513,11 @@ def search_for_folders( verbose: bool = True, return_full_path: bool = False, ) -> Tuple[List[Any], List[Any]]: - """ - Wrapper to determine the method used to search for search + """Wrapper to determine the method used to search for search prefix folders in the search path. Parameters ---------- - local_or_central "local" or "central" @@ -543,6 +530,7 @@ def search_for_folders( verbose If `True`, when a search folder cannot be found, a message will be printed with the missing path. + """ if local_or_central == "central" and cfg["connection_method"] == "ssh": all_folder_names, all_filenames = ssh.search_ssh_central_for_folders( @@ -570,8 +558,7 @@ def search_for_folders( def search_filesystem_path_for_folders( search_path_with_prefix: Path, return_full_path: bool = False ) -> Tuple[List[Path | str], List[Path | str]]: - """ - Use glob to search the full search path (including prefix) with glob. + """Use glob to search the full search path (including prefix) with glob. Files are filtered out of results, returning folders only. """ all_folder_names = [] @@ -581,7 +568,6 @@ def search_filesystem_path_for_folders( sorter_files_and_folders = sorted(all_files_and_folders) for file_or_folder_str in sorter_files_and_folders: - file_or_folder = Path(file_or_folder_str) if file_or_folder.is_dir(): diff --git a/datashuttle/utils/formatting.py b/datashuttle/utils/formatting.py index 12e9a6027..1cea80fa8 100644 --- a/datashuttle/utils/formatting.py +++ b/datashuttle/utils/formatting.py @@ -22,8 +22,7 @@ def check_and_format_names( name_templates: Optional[Dict] = None, bypass_validation: bool = False, ) -> List[str]: - """ - Format a list of subject or session names, e.g. + """Format a list of subject or session names, e.g. by ensuring all have sub- or ses- prefix, checking for tags, that names do not include spaces and that there are not duplicates. @@ -38,7 +37,6 @@ def check_and_format_names( Parameters ---------- - names str or list containing sub or ses names (e.g. to create folders) @@ -53,6 +51,7 @@ def check_and_format_names( bypass_validation If `True`, NeuroBlueprint validation will be performed on the passed names. + """ if isinstance(names, str): names = [names] @@ -79,8 +78,7 @@ def check_and_format_names( def format_names(names: List, prefix: Prefix) -> List[str]: - """ - Check a single or list of input session or subject names. + """Check a single or list of input session or subject names. First check the type is correct, next prepend the prefix sub- or ses- to entries that do not have the relevant prefix. @@ -88,13 +86,13 @@ def format_names(names: List, prefix: Prefix) -> List[str]: with required inputs e.g. date, time Parameters - ----------- - + ---------- names str or list containing sub or ses names (e.g. to make folders) prefix "sub" or "ses" - this defines the prefix checks. + """ assert prefix in ["sub", "ses"], "`prefix` must be 'sub' or 'ses'." @@ -117,8 +115,7 @@ def format_names(names: List, prefix: Prefix) -> List[str]: def update_names_with_range_to_flag( names: List[str], prefix: str ) -> List[str]: - """ - Given a list of names, check if they contain the @TO@ keyword. + """Given a list of names, check if they contain the @TO@ keyword. If so, expand to a range of names. Names including the @TO@ keyword must be in the form prefix-num1@num2. The maximum number of leading zeros are used to pad the output @@ -135,7 +132,9 @@ def update_names_with_range_to_flag( if tags("to") in name: check_name_with_to_tag_is_formatted_correctly(name, prefix) - prefix_tag = re.search(f"{prefix}-[0-9]+{tags('to')}[0-9]+", name)[0] # type: ignore + prefix_tag = re.search(f"{prefix}-[0-9]+{tags('to')}[0-9]+", name)[ + 0 + ] # type: ignore tag_number = prefix_tag.split(f"{prefix}-")[1] name_start_str, name_end_str = name.split(tag_number) @@ -172,8 +171,7 @@ def update_names_with_range_to_flag( def check_name_with_to_tag_is_formatted_correctly( name: str, prefix: str ) -> None: - """ - Check the input string is formatted with the @TO@ key + """Check the input string is formatted with the @TO@ key as expected. """ first_key_value_pair = name.split("_")[0] @@ -191,8 +189,7 @@ def check_name_with_to_tag_is_formatted_correctly( def make_list_of_zero_padded_names_across_range( left_number: str, right_number: str, name_start_str: str, name_end_str: str ) -> List[str]: - """ - Numbers formatted with the @TO@ keyword need to have + """Numbers formatted with the @TO@ keyword need to have standardised leading zeros on the output. Here we take the maximum number of leading zeros and apply for all numbers in the range. Note int() will strip @@ -200,7 +197,6 @@ def make_list_of_zero_padded_names_across_range( Parameters ---------- - left_number left (start) number from the range, e.g. "001" @@ -213,6 +209,7 @@ def make_list_of_zero_padded_names_across_range( name_end_str rest of the name after the flag, i.e. all other key-value pairs. + """ max_leading_zeros = max( utils.num_leading_zeros(left_number), @@ -237,8 +234,7 @@ def make_list_of_zero_padded_names_across_range( def update_names_with_datetime(names: List[str]) -> None: - """ - Replace @DATE@ and @DATETIME@ flag with date and datetime respectively. + """Replace @DATE@ and @DATETIME@ flag with date and datetime respectively. Format using key-value pair for bids, i.e. date-20221223_time- """ @@ -261,8 +257,7 @@ def replace_date_time_tags_in_name( date_with_key: str, time_with_key: str, ): - """ - For all names in the list, do the replacement of tags + """For all names in the list, do the replacement of tags with their final values. """ for i, name in enumerate(names): @@ -295,8 +290,7 @@ def format_datetime(date: str, time_: str) -> str: def add_underscore_before_after_if_not_there(string: str, key: str) -> str: - """ - If names are passed with @DATE@, @TIME@, or @DATETIME@ + """If names are passed with @DATE@, @TIME@, or @DATETIME@ but not surrounded by underscores, check and insert if required. e.g. sub-001@DATE@ becomes sub-001_@DATE@ or sub-001@DATEid-101 becomes sub-001_@DATE_id-101 @@ -307,9 +301,9 @@ def add_underscore_before_after_if_not_there(string: str, key: str) -> str: # Handle left edge if string[key_start_idx - 1] != "_": string_split = string.split(key) # assumes key only in string once - assert ( - len(string_split) == 2 - ), f"{key} must not appear in string more than once." + assert len(string_split) == 2, ( + f"{key} must not appear in string more than once." + ) string = f"{string_split[0]}_{key}{string_split[1]}" @@ -325,8 +319,7 @@ def add_underscore_before_after_if_not_there(string: str, key: str) -> str: def add_missing_prefixes_to_names( all_names: Union[List[str], str], prefix: str ) -> List[str]: - """ - Make sure all elements in the list of names are + """Make sure all elements in the list of names are prefixed with the prefix, typically "sub-" or "ses-" Use expanded list for readability diff --git a/datashuttle/utils/getters.py b/datashuttle/utils/getters.py index 8e1e8cd95..ac648f54f 100644 --- a/datashuttle/utils/getters.py +++ b/datashuttle/utils/getters.py @@ -37,8 +37,7 @@ def get_next_sub_or_ses( default_num_value_digits: int = 3, name_template_regexp: Optional[str] = None, ) -> str: - """ - Suggest the next available subject or session number. This function will + """Suggest the next available subject or session number. This function will search the local repository, and the central repository, for all subject or session folders (subject or session depending on inputs). @@ -50,7 +49,6 @@ def get_next_sub_or_ses( Parameters ---------- - cfg datashuttle configs class @@ -83,6 +81,7 @@ def get_next_sub_or_ses( ------- suggested_new_num the new suggested sub / ses. + """ prefix: Prefix @@ -123,8 +122,7 @@ def get_max_sub_or_ses_num_and_value_length( default_num_value_digits: Optional[int] = None, name_template_regexp: Optional[str] = None, ) -> Tuple[int, int]: - """ - Given a list of BIDS-style folder names, find the maximum subject or + """Given a list of BIDS-style folder names, find the maximum subject or session value (sub or ses depending on `prefix`). Also, find the number of value digits across the project, so a new suggested number can be formatted consistency. If the list is empty, set the value @@ -132,7 +130,6 @@ def get_max_sub_or_ses_num_and_value_length( Parameters ---------- - all_folders A list of BIDS-style formatted folder names. @@ -140,7 +137,6 @@ def get_max_sub_or_ses_num_and_value_length( Returns ------- - max_existing_num The largest number sub / ses value in the past list. @@ -153,10 +149,9 @@ def get_max_sub_or_ses_num_and_value_length( """ if len(all_folders) == 0: - - assert isinstance( - default_num_value_digits, int - ), "`default_num_value_digits` must be int`" + assert isinstance(default_num_value_digits, int), ( + "`default_num_value_digits` must be int`" + ) max_existing_num = 0 @@ -214,8 +209,7 @@ def get_max_sub_or_ses_num_and_value_length( def get_num_value_digits_from_project( all_values_str: List[str], prefix: Prefix ) -> int: - """ - Find the number of digits for the sub or ses key within the project. + """Find the number of digits for the sub or ses key within the project. `all_values_str` is a list of all the sub or ses values from within the project. """ @@ -235,8 +229,7 @@ def get_num_value_digits_from_project( def get_num_value_digits_from_regexp( prefix: Prefix, name_template_regexp: str ) -> Union[Literal[False], int]: - """ - Given a name template regexp, find the number of values for the + """Given a name template regexp, find the number of values for the sub or ses key. These will be fixed with "\d" (digit) or ".?" (wildcard). If there is length-unspecific wildcard (.*) in the sub key, then skip. In practice, there should never really be a .* in the sub or ses @@ -262,8 +255,7 @@ def get_num_value_digits_from_regexp( def get_existing_project_paths() -> List[Path]: - """ - Return full path and names of datashuttle projects on + """Return full path and names of datashuttle projects on this local machine. A project is determined by a project folder in the home / .datashuttle folder that contains a config.yaml file. Returns in order of most recently modified @@ -301,8 +293,7 @@ def get_all_sub_and_ses_paths( top_level_folder: TopLevelFolder, include_central: bool, ) -> Dict: - """ - Get a list of every subject and session name in the + """Get a list of every subject and session name in the local and central project folders. Local and central names are combined into a single list, separately for subject and sessions. @@ -311,7 +302,6 @@ def get_all_sub_and_ses_paths( Parameters ---------- - cfg datashuttle Configs @@ -321,6 +311,7 @@ def get_all_sub_and_ses_paths( include_central If `False, only get names from `local_path`, otherwise from `local_path` and `central_path`. + """ sub_folder_paths = folders.search_project_for_sub_or_ses_names( cfg, @@ -340,7 +331,6 @@ def get_all_sub_and_ses_paths( all_ses_folder_paths = {} for sub_path in all_sub_folder_paths: - sub = sub_path.name ses_folder_paths = folders.search_project_for_sub_or_ses_names( diff --git a/datashuttle/utils/rclone.py b/datashuttle/utils/rclone.py index 0462af2b7..a0bd2772c 100644 --- a/datashuttle/utils/rclone.py +++ b/datashuttle/utils/rclone.py @@ -9,18 +9,17 @@ def call_rclone(command: str, pipe_std: bool = False) -> CompletedProcess: - """ - Call rclone with the specified command. Current mode is double-verbose. + """Call rclone with the specified command. Current mode is double-verbose. Return the completed process from subprocess. Parameters ---------- - command Rclone command to be run pipe_std if True, do not output anything to stdout. + """ command = "rclone " + command if pipe_std: @@ -42,8 +41,7 @@ def setup_rclone_config_for_local_filesystem( rclone_config_name: str, log: bool = True, ): - """ - RClone sets remote targets in a config file that are + """RClone sets remote targets in a config file that are used at transfer. For local filesystem, this is essentially a placeholder and that is not linked to a particular filepath. It just tells rclone to use the local filesystem - then we @@ -57,13 +55,13 @@ def setup_rclone_config_for_local_filesystem( Parameters ---------- - rclone_config_name canonical config name, generated by datashuttle.cfg.get_rclone_config_name() log whether to log, if True logger must already be initialised. + """ call_rclone(f"config create {rclone_config_name} local", pipe_std=True) @@ -77,14 +75,12 @@ def setup_rclone_config_for_ssh( ssh_key_path: Path, log: bool = True, ): - """ - RClone sets remote targets in a config file that are + """RClone sets remote targets in a config file that are used at transfer. For SSH, this must contain the central path, username and ssh key. The relative path is supplied at transfer time. Parameters ---------- - cfg datashuttle configs UserDict. @@ -98,6 +94,7 @@ def setup_rclone_config_for_ssh( log whether to log, if True logger must already be initialised. + """ call_rclone( f"config create " @@ -117,15 +114,12 @@ def setup_rclone_config_for_ssh( def log_rclone_config_output(): output = call_rclone("config file", pipe_std=True) utils.log( - f"Successfully created rclone config. " - f"{output.stdout.decode('utf-8')}" + f"Successfully created rclone config. {output.stdout.decode('utf-8')}" ) def check_rclone_with_default_call() -> bool: - """ - Check to see whether rclone is installed. - """ + """Check to see whether rclone is installed.""" try: output = call_rclone("-h", pipe_std=True) except FileNotFoundError: @@ -134,8 +128,7 @@ def check_rclone_with_default_call() -> bool: def prompt_rclone_download_if_does_not_exist() -> None: - """ - Check that rclone is installed. If it does not + """Check that rclone is installed. If it does not (e.g. first time using datashuttle) then download. """ if not check_rclone_with_default_call(): @@ -158,12 +151,10 @@ def transfer_data( include_list: List[str], rclone_options: Dict, ) -> subprocess.CompletedProcess: - """ - Transfer data by making a call to Rclone. + """Transfer data by making a call to Rclone. Parameters ---------- - cfg datashuttle configs @@ -180,6 +171,7 @@ def transfer_data( rclone_options A list of options to pass to Rclone's copy function. see `cfg.make_rclone_transfer_options()`. + """ assert upload_or_download in [ "upload", @@ -217,8 +209,7 @@ def get_local_and_central_file_differences( cfg: Configs, top_level_folders_to_check: List[TopLevelFolder], ) -> Dict: - """ - Convert the output of rclone's check (with `--combine`) flag + """Convert the output of rclone's check (with `--combine`) flag to a dictionary separating each case. Rclone output comes as a list of files, separated by newlines, @@ -227,18 +218,17 @@ def get_local_and_central_file_differences( Parameters ---------- - top_level_folders_to_check List of top-level folders to check. Returns ------- - parsed_output A dictionary where the keys are the cases (e.g. "same" across local and central) and the values are lists of paths that fall into these cases. Note the paths are relative to the "rawdata" folder. + """ convert_symbols = { "=": "same", @@ -252,7 +242,6 @@ def get_local_and_central_file_differences( parsed_output = {val: [] for val in convert_symbols.values()} for top_level_folder in top_level_folders_to_check: - rclone_output = perform_rclone_check(cfg, top_level_folder) # type: ignore split_rclone_output = rclone_output.split("\n") @@ -273,8 +262,7 @@ def get_local_and_central_file_differences( def assert_rclone_check_output_is_as_expected(result, symbol, convert_symbols): - """ - Ensure the output of Rclone check is as expected. Currently, the "error" + """Ensure the output of Rclone check is as expected. Currently, the "error" case is untested and a test case is required. Once the test case is obtained this should most likely be moved to tests. """ @@ -293,8 +281,7 @@ def assert_rclone_check_output_is_as_expected(result, symbol, convert_symbols): def perform_rclone_check( cfg: Configs, top_level_folder: TopLevelFolder ) -> str: - """ - Use Rclone's `check` command to build a list of files that + """Use Rclone's `check` command to build a list of files that are the same ("="), different ("*"), found in local only ("+") or central only ("-"). The output is formatted as " \n". """ @@ -306,7 +293,7 @@ def perform_rclone_check( ).parent.as_posix() output = call_rclone( - f'{rclone_args("check")} ' + f"{rclone_args('check')} " f'"{local_filepath}" ' f'"{cfg.get_rclone_config_name()}:{central_filepath}"' f" --combined -", @@ -319,9 +306,7 @@ def perform_rclone_check( def handle_rclone_arguments( rclone_options: Dict, include_list: List[str] ) -> str: - """ - Construct the extra arguments to pass to RClone, - """ + """Construct the extra arguments to pass to RClone,""" extra_arguments_list = [] extra_arguments_list += ["-" + rclone_options["transfer_verbosity"]] @@ -351,9 +336,7 @@ def handle_rclone_arguments( def rclone_args(name: str) -> str: - """ - Central function to hold rclone commands - """ + """Central function to hold rclone commands""" valid_names = [ "dry_run", "copy", diff --git a/datashuttle/utils/ssh.py b/datashuttle/utils/ssh.py index eb8025bd2..b1cd9d2b0 100644 --- a/datashuttle/utils/ssh.py +++ b/datashuttle/utils/ssh.py @@ -48,9 +48,7 @@ def connect_client_core( def add_public_key_to_central_authorized_keys( cfg: Configs, password: str, log=True ) -> None: - """ - Append the public part of key to central server ~/.ssh/authorized_keys. - """ + """Append the public part of key to central server ~/.ssh/authorized_keys.""" generate_and_write_ssh_key(cfg.ssh_key_path) key = paramiko.RSAKey.from_private_key_file(cfg.ssh_key_path.as_posix()) @@ -78,8 +76,7 @@ def generate_and_write_ssh_key(ssh_key_path: Path) -> None: def get_remote_server_key(central_host_id: str): - """ - Get the remove server host key for validation before + """Get the remove server host key for validation before connection. """ transport: paramiko.Transport @@ -106,8 +103,7 @@ def setup_ssh_key( cfg: Configs, log: bool = True, ) -> None: - """ - Set up an SSH private / public key pair with + """Set up an SSH private / public key pair with central server. First, a private key is generated and saved in the .datashuttle config path. Next a connection requiring input @@ -115,8 +111,7 @@ def setup_ssh_key( added to ~/.ssh/authorized_keys. Parameters - ----------- - + ---------- ssh_key_path path to the ssh private key @@ -130,6 +125,7 @@ def setup_ssh_key( log log if True, logger must already be initialised. + """ if not sys.stdin.isatty(): proceed = input( @@ -175,8 +171,7 @@ def connect_client_with_logging( password: Optional[str] = None, message_on_sucessful_connection: bool = True, ) -> None: - """ - Connect client to central server using paramiko. + """Connect client to central server using paramiko. Accept either password or path to private key, but not both. Paramiko does not support pathlib. """ @@ -184,7 +179,7 @@ def connect_client_with_logging( connect_client_core(client, cfg, password) if message_on_sucessful_connection: utils.print_message_to_user( - f"Connection to { cfg['central_host_id']} made successfully." + f"Connection to {cfg['central_host_id']} made successfully." ) except Exception: @@ -203,8 +198,7 @@ def connect_client_with_logging( def verify_ssh_central_host( central_host_id: str, hostkeys_path: Path, log: bool = True ) -> bool: - """ - Similar to connecting with other SSH manager e.g. putty, + """Similar to connecting with other SSH manager e.g. putty, get the server key and present when connecting for manual validation. """ @@ -250,13 +244,11 @@ def search_ssh_central_for_folders( verbose: bool = True, return_full_path: bool = False, ) -> Tuple[List[Any], List[Any]]: - """ - Search for the search prefix in the search path over SSH. + """Search for the search prefix in the search path over SSH. Returns the list of matching folders, files are filtered out. Parameters - ----------- - + ---------- search_path path to search for folders in @@ -269,6 +261,7 @@ def search_ssh_central_for_folders( verbose If `True`, if a search folder cannot be found, a message will be printed with the un-found path. + """ client: paramiko.SSHClient with paramiko.SSHClient() as client: @@ -296,13 +289,11 @@ def get_list_of_folder_names_over_sftp( verbose: bool = True, return_full_path: bool = False, ) -> Tuple[List[Any], List[Any]]: - """ - Use paramiko's sftp to search a path + """Use paramiko's sftp to search a path over ssh for folders. Return the folder names. Parameters ---------- - stfp connected paramiko stfp object (see search_ssh_central_for_folders()) @@ -317,12 +308,12 @@ def get_list_of_folder_names_over_sftp( verbose If `True`, if a search folder cannot be found, a message will be printed with the un-found path. + """ all_folder_names = [] all_filenames = [] try: for file_or_folder in sftp.listdir_attr(search_path.as_posix()): - if file_or_folder.st_mode is not None and fnmatch.fnmatch( file_or_folder.filename, search_prefix ): diff --git a/datashuttle/utils/utils.py b/datashuttle/utils/utils.py index 87a39e5c9..18b98f913 100644 --- a/datashuttle/utils/utils.py +++ b/datashuttle/utils/utils.py @@ -19,8 +19,7 @@ def log(message: str) -> None: - """ - Log the message to the main initialised + """Log the message to the main initialised logger. """ if ds_logger.logging_is_active(): @@ -29,8 +28,7 @@ def log(message: str) -> None: def log_and_message(message: str, use_rich: bool = False) -> None: - """ - Log the message and send it to user. + """Log the message and send it to user. use_rich : is True, use rich's print() function """ log(message) @@ -38,9 +36,7 @@ def log_and_message(message: str, use_rich: bool = False) -> None: def log_and_raise_error(message: str, exception: Any) -> None: - """ - Log the message before raising the same message as an error. - """ + """Log the message before raising the same message as an error.""" if ds_logger.logging_is_active(): logger = ds_logger.get_logger() logger.error(f"\n\n{' '.join(traceback.format_stack(limit=5))}") @@ -57,8 +53,7 @@ def warn(message: str, log: bool) -> None: def raise_error(message: str, exception) -> None: - """ - Centralized way to raise an error. The logger is closed + """Centralized way to raise an error. The logger is closed to ensure it is not still running if a function call raises an exception in a python environment. """ @@ -69,8 +64,7 @@ def raise_error(message: str, exception) -> None: def print_message_to_user( message: Union[str, list], use_rich: bool = False ) -> None: - """ - Centralised way to send message. + """Centralised way to send message. use_rich : use rich's print() function. """ if use_rich: @@ -80,9 +74,7 @@ def print_message_to_user( def get_user_input(message: str) -> str: - """ - Centralised way to get user input - """ + """Centralised way to get user input""" input_ = input(message) return input_ @@ -125,8 +117,7 @@ def get_values_from_bids_formatted_name( return_as_int: bool = False, sort: bool = False, ) -> Union[List[int], List[str]]: - """ - Find the values associated with a key from a list of all + """Find the values associated with a key from a list of all BIDS-formatted file / folder names. This is typically used to find sub / ses values. @@ -135,10 +126,10 @@ def get_values_from_bids_formatted_name( This function does not raise through datashuttle because we don't want to turn off logging, as some times these exceptions are caught and skipped. + """ all_values = [] for name in all_names: - if key not in name: raise NeuroBlueprintError( f"The key {key} is not found in {name}", KeyError @@ -177,8 +168,7 @@ def sub_or_ses_value_to_int(value: str) -> int: def get_value_from_key_regexp(name: str, key: str) -> List[str]: - """ - Find the value related to the key in a + """Find the value related to the key in a BIDS-style key-value pair name. e.g. sub-001_ses-312 would find 312 for key "ses". @@ -197,8 +187,7 @@ def integers_are_consecutive(list_of_ints: List[int]) -> bool: def diff(x: List) -> List: - """ - slow, custom differentiator for small inputs, to avoid + """slow, custom differentiator for small inputs, to avoid adding numpy as a dependency. """ return [x[i + 1] - x[i] for i in range(len(x) - 1)] @@ -213,14 +202,10 @@ def num_leading_zeros(string: str) -> int: def all_unique(list_: List) -> bool: - """ - Check that all values in a list are different. - """ + """Check that all values in a list are different.""" return len(list_) == len(set(list_)) def all_identical(list_: List) -> bool: - """ - Check that all values in a list are identical. - """ + """Check that all values in a list are identical.""" return len(set(list_)) == 1 diff --git a/datashuttle/utils/validation.py b/datashuttle/utils/validation.py index 05f58f6f3..99c798fb6 100644 --- a/datashuttle/utils/validation.py +++ b/datashuttle/utils/validation.py @@ -86,8 +86,7 @@ def get_datetime_error(key, name: str, strfmt: str, path_: Path | None) -> str: def get_template_error(name: str, regexp: str, path_: Path | None) -> str: - """ - The missing full-stop at the end is intentional, to avoid + """The missing full-stop at the end is intentional, to avoid confusion when reading the regexp. """ return handle_path( @@ -137,13 +136,11 @@ def validate_list_of_names( name_templates: Optional[Dict] = None, check_value_lengths: bool = True, ) -> List[str]: - """ - Validate a list of subject or session names, ensuring + """Validate a list of subject or session names, ensuring they are formatted as per NeuroBlueprint. Parameters ---------- - path_or_name_list A list of pathlib.Path to NeuroBlueprint-formatted folders to validate @@ -156,6 +153,7 @@ def validate_list_of_names( check_value_lengths If `True`, check that the prefix- value lengths are consistent across the passed list. + """ if len(path_or_name_list) == 0: return [] @@ -164,7 +162,6 @@ def validate_list_of_names( # First, just validate each name individually for path_or_name in path_or_name_list: - path_, name = get_path_and_name(path_or_name) error_messages += prefix_is_duplicate_or_has_bad_values( @@ -190,7 +187,6 @@ def validate_list_of_names( ) for path_or_name in stripped_path_or_names_list: - path_, name = get_path_and_name(path_or_name) error_messages += new_name_duplicates_existing( @@ -208,8 +204,7 @@ def validate_list_of_names( def prefix_is_duplicate_or_has_bad_values( name: str, prefix: Prefix, path_: Path | None ) -> List[str]: - """ - Check that the prefix (sub- or ses-) is found only + """Check that the prefix (sub- or ses-) is found only once in the name and that its value can be converted to integer. """ @@ -233,8 +228,7 @@ def new_name_duplicates_existing( existing_path_or_name_list: List[Path] | List[str], prefix: Prefix, ) -> List[str]: - """ - Check that a subject or session value does not duplicate + """Check that a subject or session value does not duplicate an existing value. The only case this is allowed is when the names match exactly. @@ -251,7 +245,6 @@ def new_name_duplicates_existing( error_messages = [] for exist_path_or_name in existing_path_or_name_list: - exist_path, exist_name = get_path_and_name(exist_path_or_name) exist_name_id = utils.get_values_from_bids_formatted_name( @@ -274,8 +267,7 @@ def names_dont_match_templates( prefix: Prefix, name_templates: Optional[Dict] = None, ) -> List[str]: - """ - Test a list of subject or session names against + """Test a list of subject or session names against the respective `name_templates`, a regexp template. """ if name_templates is None: @@ -298,8 +290,7 @@ def names_dont_match_templates( def get_path_and_name(path_or_name: Path | str) -> Tuple[Optional[Path], str]: - """ - Convenience function to get the folder name + """Convenience function to get the folder name from something that is either a Path (pointing to the folder) or a str of the folder name itself. """ @@ -310,8 +301,7 @@ def get_path_and_name(path_or_name: Path | str) -> Tuple[Optional[Path], str]: def replace_tags_in_regexp(regexp: str) -> str: - """ - Before validation, all tags in the names are converted to + """Before validation, all tags in the names are converted to their final values (e.g. @DATE@ -> _date-). We also want to allow template to be formatted like `sub-\d\d_@DATE@` as it is convenient for auto-completion in the TUI. @@ -336,8 +326,7 @@ def replace_tags_in_regexp(regexp: str) -> str: def name_begins_with_bad_key( name: str, prefix: Prefix, path_: Path | None ) -> List[str]: - """ - Check that a list of NeuroBlueprint names begin + """Check that a list of NeuroBlueprint names begin with the required prefix (sub- or ses-). """ if name[:4] != f"{prefix}-": @@ -349,8 +338,7 @@ def name_begins_with_bad_key( def names_include_special_characters( name: str, path_: Path | None ) -> List[str]: - """ - Check that a list of NeuroBlueprint formatted + """Check that a list of NeuroBlueprint formatted names do not contain special characters (i.e. characters that are not integers, letters, dash or underscore). """ @@ -367,8 +355,7 @@ def name_has_special_character(name: str) -> bool: def dashes_and_underscore_alternate_incorrectly( name: str, path_: Path | None ) -> List[str]: - """ - Check a list of NeuroBlueprint formatted names + """Check a list of NeuroBlueprint formatted names have the "-" and "-" ordered correctly. Names should be key-value pairs separated by underscores e.g. sub-001_ses-001. @@ -386,7 +373,7 @@ def dashes_and_underscore_alternate_incorrectly( or dashes_underscores[0] != 1 # first must be - or dashes_underscores[-1] != 1 # last must be - or underscore_dash_not_interleaved - or (name[-1] in discrim.keys()) # name cannot end with - or _ + or (name[-1] in discrim) # name cannot end with - or _ ): return [get_name_format_error(name, path_)] else: @@ -397,8 +384,7 @@ def value_lengths_are_inconsistent( path_or_names_list: List[Path] | List[str] | List[Path | str], prefix: Prefix, ) -> List[str]: - """ - Given a list of NeuroBlueprint-formatted subject or session + """Given a list of NeuroBlueprint-formatted subject or session names, determine if there are inconsistent value lengths for the sub or ses key. e.g. ["sub-01", "sub-001"] is an error. """ @@ -429,9 +415,7 @@ def datetime_are_iso_format( name: str, path_: Path | None, ) -> List[str]: - """ - Check formatting for date-, time-, or datetime- tags. - """ + """Check formatting for date-, time-, or datetime- tags.""" formats = { "datetime": "%Y%m%dT%H%M%S", "time": "%H%M%S", @@ -466,8 +450,7 @@ def datetime_are_iso_format( def raise_display_mode( message: str, display_mode: DisplayMode, log: bool ) -> None: - """ - Show a message by raising an error, displaying warning, or printing. + """Show a message by raising an error, displaying warning, or printing. Optionally log with the current datashuttle logger. """ if display_mode == "error": @@ -501,12 +484,10 @@ def validate_project( name_templates: Optional[Dict] = None, strict_mode: bool = False, ) -> List[str]: - """ - Validate all subject and session folders within a project. + """Validate all subject and session folders within a project. Parameters - ----------- - + ---------- cfg datashuttle Configs class. @@ -534,6 +515,7 @@ def validate_project( starting with sub- or ses- prefix are checked. In `Strict Mode`, any folder not prefixed with sub-, ses- or a valid datatype will raise a validation issue. + """ error_messages = [] @@ -541,7 +523,6 @@ def validate_project( error_messages += check_high_level_project_structure(cfg, include_central) for top_level_folder in top_level_folder_list: - if strict_mode: error_messages += check_strict_mode( cfg, top_level_folder, include_central @@ -568,7 +549,6 @@ def validate_project( # Check all names as well as duplicates per-subject for ses_paths in folder_paths["ses"].values(): - error_messages += validate_list_of_names( ses_paths, "ses", @@ -604,8 +584,7 @@ def validate_names_against_project( log: bool = True, name_templates: Optional[Dict] = None, ) -> None: - """ - Given a list of subject and (optionally) session names, + """Given a list of subject and (optionally) session names, check that these names are formatted consistently with the rest of the project. Used for creating folders. @@ -615,7 +594,6 @@ def validate_names_against_project( Parameters ---------- - cfg datashuttle Configs class. @@ -645,6 +623,7 @@ def validate_names_against_project( name_templates A `name_template` dictionary to validate against. See `set_name_templates()`. + """ error_messages = [] @@ -662,7 +641,6 @@ def validate_names_against_project( ) if folder_paths["sub"]: - # Strip any totally invalid names which we can't extract # the sub integer value for the following checks valid_sub_names = strip_uncheckable_names(sub_names, "sub") @@ -690,14 +668,12 @@ def validate_names_against_project( # Now we need to check the sessions. if ses_names is not None and any(ses_names): - # First, validate the list of passed session names error_messages += validate_list_of_names( ses_names, "ses", name_templates=name_templates ) if folder_paths["sub"]: - # Next, we need to check that the passed session names # do not duplicate existing session names and # that do not create inconsistent ses- lengths across the project. @@ -708,7 +684,6 @@ def validate_names_against_project( # are allowed across different subjects (but not within a single sub). for new_sub in sub_names: if new_sub in folder_paths["ses"]: - valid_ses_in_sub = strip_uncheckable_names( folder_paths["ses"][new_sub], "ses", @@ -746,8 +721,7 @@ def validate_names_against_project( def check_high_level_project_structure( cfg: Configs, include_central: bool ) -> List[str]: - """ - Perform basic validation checks on the project structure, + """Perform basic validation checks on the project structure, that the project folder name is valid, and that the project folder contains either a "rawdata" or "derivatives" folder. @@ -810,8 +784,7 @@ def check_high_level_project_structure( def check_strict_mode( cfg: Configs, top_level_folder: TopLevelFolder, include_central: bool ) -> List[str]: - """ - `strict_mode` does not allow any non-NeuroBlueprint folder to exist + """`strict_mode` does not allow any non-NeuroBlueprint folder to exist in the project outside the datatype folder. NeuroBlueprint folders are top-level folder or folder with sub-, ses- or datatype. @@ -841,7 +814,6 @@ def check_strict_mode( ) for sub_level_path in sub_level_folder_paths["local"]: - # Check all folders found in a top-level folder are # sub- prefixed folders. sub_level_name = sub_level_path.name @@ -861,7 +833,6 @@ def check_strict_mode( ) for ses_level_path in ses_level_folder_paths["local"]: - # For each sub- prefixed folder, check that all folders within # the subject folder are ses- prefixed folders. ses_level_name = ses_level_path.name @@ -884,7 +855,6 @@ def check_strict_mode( canonical_datatypes = canonical_configs.get_datatypes() for datatype_level_path in search_results: - # For each ses- prefixed folder, check that # only valid datatypes are included within it. datatype_level_name = datatype_level_path.name @@ -916,8 +886,7 @@ def strip_uncheckable_names( path_or_names_list: List[Path] | List[str], prefix: Prefix, ) -> List[Path] | List[str]: - """ - Convenience function to remove any name in which + """Convenience function to remove any name in which the `prefix` value (sub or ses typically) cannot be converted into an integer. This is necessary as some validation steps (e.g. checking duplicate names) requires @@ -927,7 +896,6 @@ def strip_uncheckable_names( new_list = [] for path_or_name in path_or_names_list: - path_, name = get_path_and_name(path_or_name) try: @@ -953,8 +921,7 @@ def strip_uncheckable_names( def check_datatypes_are_valid( datatype: Union[List[str], str], allow_all: bool = False ) -> str | None: - """ - Check a datatype of list of datatypes is a valid + """Check a datatype of list of datatypes is a valid NeuroBlueprint datatype. """ datatype_folders = canonical_folders.get_datatype_folders() diff --git a/tests/conftest.py b/tests/conftest.py index 2203b1025..50c27d849 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,5 +1,4 @@ -""" -Test configs, used for setting up SSH tests. +"""Test configs, used for setting up SSH tests. Before running these tests, it is necessary to setup an SSH key. This can be done through datashuttle diff --git a/tests/ssh_test_utils.py b/tests/ssh_test_utils.py index 0838669f3..bf5bf2c74 100644 --- a/tests/ssh_test_utils.py +++ b/tests/ssh_test_utils.py @@ -7,8 +7,7 @@ def setup_project_for_ssh( project, central_path, central_host_id, central_host_username ): - """ - Set up the project configs to use SSH connection + """Set up the project configs to use SSH connection to central """ project.update_config_file( @@ -26,8 +25,7 @@ def setup_project_for_ssh( def setup_mock_input(input_): - """ - This is very similar to pytest monkeypatch but + """This is very similar to pytest monkeypatch but using that was giving me very strange output, monkeypatch.setattr('builtins.input', lambda _: "n") i.e. pdb went deep into some unrelated code stack @@ -38,16 +36,12 @@ def setup_mock_input(input_): def restore_mock_input(orig_builtin): - """ - orig_builtin: the copied, original builtins.input - """ + """orig_builtin: the copied, original builtins.input""" builtins.input = orig_builtin def setup_hostkeys(project): - """ - Convenience function to verify the server hostkey. - """ + """Convenience function to verify the server hostkey.""" orig_builtin = setup_mock_input(input_="y") ssh.verify_ssh_central_host( project.cfg["central_host_id"], project.cfg.hostkeys_path, log=True diff --git a/tests/test_utils.py b/tests/test_utils.py index ad908160e..46d0b23ea 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -27,8 +27,7 @@ def setup_project_default_configs( local_path=False, central_path=False, ): - """ - Set up a fresh project to test on + """Set up a fresh project to test on local_path / central_path: provide the config paths to set """ @@ -76,8 +75,7 @@ def make_project_paths(config_dict): def glob_basenames(search_path, recursive=False, exclude=None): - """ - Use glob to search but strip the full path, including + """Use glob to search but strip the full path, including only the base name (lowest level). """ paths_ = glob.glob(search_path, recursive=recursive) @@ -131,8 +129,7 @@ def delete_project_if_it_exists(project_name): def setup_project_fixture(tmp_path, test_project_name, project_type="full"): - """ - Set up a project, either in full mode or local-only mode. This is + """Set up a project, either in full mode or local-only mode. This is very similar to the `BaseTest` fixture but is designed for use in other fixtures that require additional boilerplate e.g. logging. """ @@ -184,8 +181,7 @@ def get_test_config_arguments_dict( set_as_defaults=False, required_arguments_only=False, ): - """ - Retrieve configs, either the required configs + """Retrieve configs, either the required configs (for project.make_config_file()), all configs (default) or non-default configs. Note that default configs here are the expected default arguments in project.make_config_file(). @@ -227,8 +223,7 @@ def get_test_config_arguments_dict( def get_all_broad_folders_used(value=True): - """ - The `folders_used` construct tells the tests which + """The `folders_used` construct tells the tests which folders were used (e.g. created or transferred) and which are not. This means the expected datatypes can be checked. @@ -253,8 +248,7 @@ def get_all_broad_folders_used(value=True): def check_folder_tree_is_correct( base_folder, subs, sessions, folder_used, created_folder_dict=None ): - """ - Automated test that folders are made based + """Automated test that folders are made based on the structure specified on project itself. Cycle through all datatypes (defined in @@ -316,8 +310,7 @@ def check_folder_tree_is_correct( def check_and_cd_folder(path_): - """ - Check a folder exists and CD to it if it does. + """Check a folder exists and CD to it if it does. Use the pytest -s flag to print all tested paths """ @@ -331,8 +324,7 @@ def check_datatype_sub_ses_uploaded_correctly( subs_to_upload=None, ses_to_upload=None, ): - """ - Iterate through the project (datatype > ses > sub) and + """Iterate through the project (datatype > ses > sub) and check that the folders at each level match those that are expected (passed in datatype / sub / ses to upload). Folders are searched with wildcard glob. @@ -368,8 +360,7 @@ def check_datatype_sub_ses_uploaded_correctly( def make_and_check_local_project_folders( project, top_level_folder, subs, sessions, datatype, datatypes_used=None ): - """ - Make a local project folder tree with the specified datatype, + """Make a local project folder tree with the specified datatype, subs, sessions and check it is made successfully. Since empty folders are not transferred, it is necessary @@ -422,8 +413,7 @@ def check_project_configs( project, *kwargs, ): - """ - Core function for checking the config against + """Core function for checking the config against provided configs (kwargs). Open the config.yaml file and check the config values stored there, and in project.cfg, against the provided configs. @@ -444,7 +434,7 @@ def check_project_configs( def check_config_file(config_path, *kwargs): """""" - with open(config_path, "r") as config_file: + with open(config_path) as config_file: config_yaml = yaml.full_load(config_file) for name, value in kwargs[0].items(): @@ -461,9 +451,9 @@ def get_top_level_folder_path( ): """""" - assert ( - folder_name in canonical_folders.get_top_level_folders() - ), "folder_name must be canonical e.g. rawdata" + assert folder_name in canonical_folders.get_top_level_folders(), ( + "folder_name must be canonical e.g. rawdata" + ) if local_or_central == "local": base_path = project.cfg["local_path"] @@ -480,8 +470,7 @@ def handle_upload_or_download( top_level_folder=None, swap_last_folder_only=False, ): - """ - To keep things consistent and avoid the pain of writing + """To keep things consistent and avoid the pain of writing files over SSH, to test download just swap the central and local server (so things are still transferred from local machine to central, but using the download function). @@ -537,8 +526,7 @@ def get_transfer_func( def swap_local_and_central_paths(project, swap_last_folder_only=False): - """ - When testing upload vs. download, the most convenient way + """When testing upload vs. download, the most convenient way to test download is to swap the paths. In this case, we 'download' from local to central. It much simplifies creating the folders to transfer (which are created locally), and is fully required @@ -584,25 +572,21 @@ def swap_local_and_central_paths(project, swap_last_folder_only=False): def get_default_sub_sessions_to_test(): - """ - Canonical subs / sessions for these tests - """ + """Canonical subs / sessions for these tests""" subs = ["sub-001", "sub-002", "sub-003"] sessions = ["ses-001_datetime-20220516T135022", "ses-002", "ses-003"] return subs, sessions def move_some_keys_to_end_of_dict(config): - """ - Need to move connection method to the end + """Need to move connection method to the end so ssh opts are already set before it is changed. """ config["connection_method"] = config.pop("connection_method") def clear_capsys(capsys): - """ - read from capsys clears it, so new + """Read from capsys clears it, so new print statements are clearer to read. """ capsys.readouterr() @@ -619,14 +603,13 @@ def write_file(path_, contents="", append=False): def read_file(path_): - with open(path_, "r") as file: + with open(path_) as file: contents = file.readlines() return contents def set_datashuttle_loggers(disable): - """ - Turn off or on datashuttle logs, if these are + """Turn off or on datashuttle logs, if these are on when testing with pytest they will be propagated to pytest's output, making it difficult to read. @@ -643,8 +626,7 @@ def set_datashuttle_loggers(disable): def check_working_top_level_folder_only_exists( folder_name, base_path_to_check, subs, sessions, folders_used=None ): - """ - Check that the folder tree made in the 'folder_name' + """Check that the folder tree made in the 'folder_name' (e.g. 'rawdata') top level folder is correct. Additionally, check that no other top-level folders exist. This is to ensure that folders made / transferred from one top-level folder @@ -672,11 +654,11 @@ def read_log_file(logging_path): log_filepath = list(glob.glob(str(logging_path / "*.log"))) assert len(log_filepath) == 1, ( - f"there should only be one log " f"in log output path {logging_path}" + f"there should only be one log in log output path {logging_path}" ) log_filepath = log_filepath[0] - with open(log_filepath, "r") as file: + with open(log_filepath) as file: log = file.read() return log @@ -684,7 +666,7 @@ def read_log_file(logging_path): def delete_log_files(logging_path): ds_logger.close_log_filehandler() - for log in glob.glob((str(logging_path / "*.log"))): + for log in glob.glob(str(logging_path / "*.log")): os.remove(log) diff --git a/tests/tests_integration/_test_configs.py b/tests/tests_integration/_test_configs.py index feff2900d..3d617a072 100644 --- a/tests/tests_integration/_test_configs.py +++ b/tests/tests_integration/_test_configs.py @@ -15,25 +15,20 @@ class TestConfigs(BaseTest): @pytest.fixture(scope="function") def non_existent_path(self, tmp_path): - """ - Return a path that does not exist. - """ + """Return a path that does not exist.""" non_existent_path = tmp_path / "does_not_exist" assert not non_existent_path.is_dir() return non_existent_path @pytest.fixture(scope="function") def existent_path(self, tmp_path): - """ - Return a path that exists. - """ + """Return a path that exists.""" existent_path = tmp_path / "exists" os.makedirs(existent_path, exist_ok=True) return existent_path def test_warning_on_startup(self, no_cfg_project): - """ - When no configs have been set, a warning should be shown that + """When no configs have been set, a warning should be shown that the config has not been initialized. Need to download Rclone first to ensure input() is not called. """ @@ -60,8 +55,7 @@ def test_warning_on_startup(self, no_cfg_project): ) @pytest.mark.parametrize("path_type", ["local_path", "central_path"]) def test_bad_path_syntax(self, project, bad_pattern, path_type, tmp_path): - """ - "~", "." and "../" syntax is not supported because + """ "~", "." and "../" syntax is not supported because it does not work with rclone. Theoretically it could be supported by checking for "." etc. and filling in manually, but it does not seem robust. @@ -95,8 +89,7 @@ def test_bad_path_syntax(self, project, bad_pattern, path_type, tmp_path): assert "must contain the full folder path with no " in str(e.value) def test_no_ssh_options_set_on_make_config_file(self, no_cfg_project): - """ - Check that program will assert if not all ssh options + """Check that program will assert if not all ssh options are set on make_config_file """ with pytest.raises(ConfigError) as e: @@ -116,8 +109,7 @@ def test_no_ssh_options_set_on_make_config_file(self, no_cfg_project): # ------------------------------------------------------------- def test_required_configs(self, no_cfg_project, tmp_path): - """ - Set the required arguments of the config (local_path, central_path, + """Set the required arguments of the config (local_path, central_path, connection_method and check they are set correctly in both the no_cfg_project.cfg dict and config.yaml file. """ @@ -133,8 +125,7 @@ def test_required_configs(self, no_cfg_project, tmp_path): ) def test_config_defaults(self, no_cfg_project, tmp_path): - """ - Check the default configs are set as expected + """Check the default configs are set as expected (see get_test_config_arguments_dict()) for tested defaults. """ required_options = test_utils.get_test_config_arguments_dict( @@ -150,8 +141,7 @@ def test_config_defaults(self, no_cfg_project, tmp_path): test_utils.check_configs(no_cfg_project, default_options) def test_non_default_configs(self, no_cfg_project, tmp_path): - """ - Set the configs to non-default options, make the + """Set the configs to non-default options, make the config file and check file and no_cfg_project.cfg are set correctly. """ changed_configs = test_utils.get_test_config_arguments_dict( @@ -167,8 +157,7 @@ def test_non_default_configs(self, no_cfg_project, tmp_path): # ------------------------------------------------------------- def test_update_config_file__(self, no_cfg_project, tmp_path): - """ - Set the configs as default, and then update them to + """Set the configs as default, and then update them to new configs and check they are updated properly. Then, update only a subset (back to the defaults) and @@ -213,8 +202,7 @@ def test_update_config_file__(self, no_cfg_project, tmp_path): test_utils.check_configs(project, default_configs) def test_existing_projects(self, monkeypatch, tmp_path): - """ - Test existing projects are correctly found based on whether + """Test existing projects are correctly found based on whether they exist in the home directory and contain a config.yaml. By default, datashuttle saves project folders to @@ -266,8 +254,7 @@ def patch_get_datashuttle_path(): # -------------------------------------------------------------------------------------------------------------------- def check_config_reopen_and_check_config_again(self, project, *kwargs): - """ - Check the config file and project.cfg against provided kwargs, + """Check the config file and project.cfg against provided kwargs, delete the project and set up the project again, checking everything is loaded correctly. """ diff --git a/tests/tests_integration/base.py b/tests/tests_integration/base.py index b04816cba..c3ea76d20 100644 --- a/tests/tests_integration/base.py +++ b/tests/tests_integration/base.py @@ -10,11 +10,9 @@ class BaseTest: - @pytest.fixture(scope="function") def no_cfg_project(test): - """ - Fixture that creates an empty project. Ignore the warning + """Fixture that creates an empty project. Ignore the warning that no configs are setup yet. """ test_utils.delete_project_if_it_exists(TEST_PROJECT_NAME) @@ -27,8 +25,7 @@ def no_cfg_project(test): @pytest.fixture(scope="function") def project(self, tmp_path, request): - """ - Set up a project with default configs to use for testing. + """Set up a project with default configs to use for testing. This fixture uses indirect parameterization to test both 'full' and 'local-only' (no `central_path` or `connection_method`). The @@ -64,8 +61,7 @@ def project(self, tmp_path, request): @pytest.fixture(scope="function") def clean_project_name(self): - """ - Create an empty project, but ensure no + """Create an empty project, but ensure no configs already exists, and delete created configs after test. """ diff --git a/tests/tests_integration/test_create_folders.py b/tests/tests_integration/test_create_folders.py index e50623776..0a69b0695 100644 --- a/tests/tests_integration/test_create_folders.py +++ b/tests/tests_integration/test_create_folders.py @@ -13,11 +13,9 @@ class TestCreateFolders(BaseTest): - @pytest.mark.parametrize("project", ["local", "full"], indirect=True) def test_generate_folders_default_ses(self, project): - """ - Make a subject folders with full tree. Don't specify + """Make a subject folders with full tree. Don't specify session name (it will default to no sessions). Check that the folder tree is created correctly. Pass @@ -37,8 +35,7 @@ def test_generate_folders_default_ses(self, project): @pytest.mark.parametrize("project", ["local", "full"], indirect=True) def test_explicitly_session_list(self, project): - """ - Perform an alternative test where the output is tested explicitly. + """Perform an alternative test where the output is tested explicitly. This is some redundancy to ensure tests are working correctly and make explicit the expected folder tree. @@ -83,8 +80,7 @@ def test_explicitly_session_list(self, project): def test_every_broad_datatype_passed( self, project, behav, ephys, funcimg, anat ): - """ - Check every combination of data type used and ensure only the + """Check every combination of data type used and ensure only the correct ones are made. NOTE: This test could be refactored to reduce code reuse. @@ -121,8 +117,7 @@ def test_every_broad_datatype_passed( @pytest.mark.parametrize("project", ["local", "full"], indirect=True) def test_custom_folder_names(self, project, monkeypatch): - """ - Change folder names to custom (non-default) and + """Change folder names to custom (non-default) and ensure they are made correctly. """ new_name_datafolders = canonical_folders.get_datatype_folders() @@ -178,8 +173,7 @@ def new_name_func(): ) @pytest.mark.parametrize("project", ["local", "full"], indirect=True) def test_datatypes_subsection(self, project, files_to_test): - """ - Check that combinations of datatypes passed to make file folder + """Check that combinations of datatypes passed to make file folder make the correct combination of datatypes. Note this will fail when new top level folders are added, and should be @@ -207,8 +201,7 @@ def test_datatypes_subsection(self, project, files_to_test): @pytest.mark.parametrize("project", ["local", "full"], indirect=True) def test_date_flags_in_session(self, project): - """ - Check that @DATE@ is converted into current date + """Check that @DATE@ is converted into current date in generated folder names """ date, time_ = self.get_formatted_date_and_time() @@ -230,8 +223,7 @@ def test_date_flags_in_session(self, project): @pytest.mark.parametrize("project", ["local", "full"], indirect=True) def test_datetime_flag_in_session(self, project): - """ - Check that @DATETIME@ is converted to datetime + """Check that @DATETIME@ is converted to datetime in generated folder names """ date, time_ = self.get_formatted_date_and_time() @@ -257,8 +249,7 @@ def test_datetime_flag_in_session(self, project): @pytest.mark.parametrize("project", ["local", "full"], indirect=True) def test_created_paths_dict_sub_or_ses_only(self, project): - """ - Test that the `created_folders` dictionary returned by + """Test that the `created_folders` dictionary returned by `create_folders` correctly splits paths when only subject or session is passed. The `datatype` case is tested in `test_utils.check_folder_tree_is_correct()`. @@ -292,8 +283,7 @@ def test_created_paths_dict_sub_or_ses_only(self, project): ) @pytest.mark.parametrize("project", ["local", "full"], indirect=True) def test_all_top_level_folders(self, project, top_level_folder): - """ - Check that when switching the top level folder (e.g. rawdata, derivatives) + """Check that when switching the top level folder (e.g. rawdata, derivatives) new folders are made in the correct folder. """ subs = ["sub-001", "sub-002"] @@ -319,8 +309,7 @@ def test_all_top_level_folders(self, project, top_level_folder): @pytest.mark.parametrize("top_level_folder", ["rawdata", "derivatives"]) @pytest.mark.parametrize("return_with_prefix", [True, False]) def test_get_next_sub(self, project, return_with_prefix, top_level_folder): - """ - Test that the next subject number is suggested correctly. + """Test that the next subject number is suggested correctly. This takes the union of subjects available in the local and central repository. As such test the case where either are empty, or when they have different subjects in. @@ -373,8 +362,7 @@ def test_get_next_sub(self, project, return_with_prefix, top_level_folder): @pytest.mark.parametrize("top_level_folder", ["rawdata", "derivatives"]) @pytest.mark.parametrize("return_with_prefix", [True, False]) def test_get_next_ses(self, project, return_with_prefix, top_level_folder): - """ - Almost identical to test_get_next_sub() but with calls + """Almost identical to test_get_next_sub() but with calls for searching sessions. This could be combined with above but reduces readability, so leave with some duplication. @@ -443,8 +431,7 @@ def test_get_next_ses(self, project, return_with_prefix, top_level_folder): @pytest.mark.parametrize("project", ["local", "full"], indirect=True) def test_get_next_sub_and_ses_name_template(self, project): - """ - In the case where a name template exists, these getters should use the + """In the case where a name template exists, these getters should use the number of digits on the template (even if these are different within the project!). """ diff --git a/tests/tests_integration/test_datatypes.py b/tests/tests_integration/test_datatypes.py index 1aec45480..10720e90b 100644 --- a/tests/tests_integration/test_datatypes.py +++ b/tests/tests_integration/test_datatypes.py @@ -8,15 +8,13 @@ class TestDatatypes(BaseTest): - """ - Tests for creating folders and transfer (very similar to other tests) + """Tests for creating folders and transfer (very similar to other tests) however which test the creation and transfer of narrow datatypes. Other tests used broad datatypes. """ def test_create_narrow_datatypes(self, project): - """ - Create all narrow datatype folders and check + """Create all narrow datatype folders and check they are created as expected. """ # Make folder tree including all narrow datatypes @@ -41,8 +39,7 @@ def test_create_narrow_datatypes(self, project): ) def get_narrow_only_datatypes_used(self, used=True): - """ - This is similar to test_utils.get_all_broad_folders_used + """This is similar to test_utils.get_all_broad_folders_used but for narrow datatypes. """ return { @@ -55,8 +52,7 @@ def test_transfer_datatypes( project, upload_or_download, ): - """ - Create a project with narrow datatypes and check these + """Create a project with narrow datatypes and check these folders are transferred as expected. """ subs, sessions = test_utils.get_default_sub_sessions_to_test() diff --git a/tests/tests_integration/test_filesystem_transfer.py b/tests/tests_integration/test_filesystem_transfer.py index 8cbff488f..98e31a629 100644 --- a/tests/tests_integration/test_filesystem_transfer.py +++ b/tests/tests_integration/test_filesystem_transfer.py @@ -13,7 +13,6 @@ class TestFileTransfer(BaseTest): - @pytest.mark.parametrize( "top_level_folder", canonical_folders.get_top_level_folders() ) @@ -28,8 +27,7 @@ def test_transfer_empty_folder_structure( upload_or_download, transfer_method, ): - """ - First make a project (folders only) locally. + """First make a project (folders only) locally. Next upload this to the central path and check all folders are uploaded correctly. """ @@ -78,8 +76,7 @@ def test_transfer_across_top_level_folders( upload_or_download, transfer_method, ): - """ - For each possible top level folder (e.g. rawdata, derivatives) + """For each possible top level folder (e.g. rawdata, derivatives) (parametrized) create a folder tree in every top-level folder, then transfer using upload / download and upload_rawdata() / download_rawdata() that only the working top-level folder @@ -151,7 +148,6 @@ def test_transfer_all_top_level_folders(self, project, upload_or_download): transfer_function() for top_level_folder in canonical_folders.get_top_level_folders(): - test_utils.check_folder_tree_is_correct( os.path.join(base_path_to_check, top_level_folder), subs, @@ -177,8 +173,7 @@ def test_transfer_all_top_level_folders(self, project, upload_or_download): def test_transfer_empty_folder_specific_data( self, project, upload_or_download, datatype_to_transfer ): - """ - For the combination of datatype folders, make a folder + """For the combination of datatype folders, make a folder tree with all datatype folders then upload select ones, checking only the selected ones are uploaded. """ @@ -215,7 +210,7 @@ def test_transfer_empty_folder_specific_data( ["behav", "ephys", "funcimg", "anat"], ], ) - @pytest.mark.parametrize("upload_or_download", ["upload" "download"]) + @pytest.mark.parametrize("upload_or_download", ["uploaddownload"]) def test_transfer_empty_folder_specific_subs( self, project, @@ -223,8 +218,7 @@ def test_transfer_empty_folder_specific_subs( datatype_to_transfer, sub_idx_to_upload, ): - """ - Create a project folder tree with a set of subs, then + """Create a project folder tree with a set of subs, then take a subset of these subs and upload them. Check only the selected subs were uploaded. """ @@ -268,8 +262,7 @@ def test_transfer_empty_folder_specific_ses( sub_idx_to_upload, ses_idx_to_upload, ): - """ - Make a project with set subs and sessions. Then select a subset of the + """Make a project with set subs and sessions. Then select a subset of the sessions to upload. Check only the selected sessions were uploaded. """ subs, sessions = test_utils.get_default_sub_sessions_to_test() @@ -303,8 +296,7 @@ def test_transfer_empty_folder_specific_ses( def test_transfer_with_keyword_parameters( self, project, upload_or_download ): - """ - Test the @TO@ keyword is accepted properly when making a session and + """Test the @TO@ keyword is accepted properly when making a session and transferring it. First pass @TO@-formatted sub and sessions to create_folders. Then transfer the files (upload or download). @@ -350,8 +342,7 @@ def test_transfer_with_keyword_parameters( @pytest.mark.parametrize("upload_or_download", ["upload", "download"]) def test_wildcard_transfer(self, project, upload_or_download): - """ - Transfer a subset of define subject and session + """Transfer a subset of define subject and session and check only the expected folders are there. """ subs = ["sub-389", "sub-989", "sub-445"] @@ -395,8 +386,7 @@ def test_wildcard_transfer(self, project, upload_or_download): ] def test_deep_folder_structure(self, project): - """ - Just a quick test as all other tests only test files directly in the + """Just a quick test as all other tests only test files directly in the datatyp directly. Check that rlcone is setup to transfer multiple levels down from the datatype level. """ @@ -427,8 +417,7 @@ def test_rclone_options( dry_run, capsys, ): - """ - When verbosity is --vv, rclone itself will output + """When verbosity is --vv, rclone itself will output a list of all called arguments. Use this to check rclone is called with the arguments set in configs as expected. verbosity itself is tested in another method. @@ -475,8 +464,7 @@ def test_overwrite_same_size_earlier_to_later( top_level_folder, upload_or_download, ): - """ - Main test to check every parameterization for overwrite settings. + """Main test to check every parameterization for overwrite settings. It is such an important setting it is tested for all top level folder, transfer method, even though it makes for quite a confusing function. @@ -541,8 +529,7 @@ def test_overwrite_same_size_later_to_earlier( top_level_folder, upload_or_download, ): - """ - This functions is extremely similar to + """This functions is extremely similar to `test_overwrite_same_size_later_to_earlier()` but it is much easier to understand individually when they are split. @@ -588,8 +575,7 @@ def test_overwrite_same_size_later_to_earlier( def test_overwrite_different_size_different_times( self, project, overwrite_existing_files ): - """ - Quick additional test to confirm that "if_source_newer" will still + """Quick additional test to confirm that "if_source_newer" will still not transfer even if the older file is larger. This is the expected behaviour from rclone, this is confidence check on understanding. """ @@ -670,8 +656,7 @@ def setup_overwrite_file_tests( def test_dry_run( self, project, top_level_folder, transfer_method, upload_or_download ): - """ - Just do a quick functional test on dry-run that indeed nothing + """Just do a quick functional test on dry-run that indeed nothing is transferred across all top-level-folder / upload-download methods. """ @@ -704,8 +689,7 @@ def test_specific_file_or_folder( transfer_file, upload_or_download, ): - """ - Test upload_specific_folder_or_file() and download_specific_folder_or_file(). + """Test upload_specific_folder_or_file() and download_specific_folder_or_file(). Make a project with two different files (just to ensure non-target files are not transferred). Transfer diff --git a/tests/tests_integration/test_formatting.py b/tests/tests_integration/test_formatting.py index bf62c6a50..0b5db8338 100644 --- a/tests/tests_integration/test_formatting.py +++ b/tests/tests_integration/test_formatting.py @@ -11,8 +11,7 @@ class TestFormatting(BaseTest): "input", [1, {"test": "one"}, 1.0, ["1", "2", ["three"]]] ) def test_format_names_bad_input(self, input, prefix): - """ - Test that names passed in incorrect type + """Test that names passed in incorrect type (not str, list) raise appropriate error. """ with pytest.raises(TypeError) as e: @@ -22,8 +21,7 @@ def test_format_names_bad_input(self, input, prefix): @pytest.mark.parametrize("prefix", ["sub", "ses"]) def test_format_names_duplicate_ele(self, prefix): - """ - Test that appropriate error is raised when duplicate name + """Test that appropriate error is raised when duplicate name is passed to format_names(). """ with pytest.raises(NeuroBlueprintError) as e: @@ -37,8 +35,7 @@ def test_format_names_duplicate_ele(self, prefix): ) def test_format_names_prefix(self): - """ - Check that format_names correctly prefixes input + """Check that format_names correctly prefixes input with default sub or ses prefix. This is less useful now that ses/sub name dash and underscore order is more strictly checked. diff --git a/tests/tests_integration/test_local_only_mode.py b/tests/tests_integration/test_local_only_mode.py index 6df59233b..0db07a503 100644 --- a/tests/tests_integration/test_local_only_mode.py +++ b/tests/tests_integration/test_local_only_mode.py @@ -13,10 +13,8 @@ class TestLocalOnlyProject(BaseTest): - def test_bad_setup(self, tmp_path): - """ - Test setup without providing both central_path and connection + """Test setup without providing both central_path and connection method (distinguishing a full vs local-only project) """ local_path = tmp_path / "test_local" @@ -41,8 +39,7 @@ def test_bad_setup(self, tmp_path): @pytest.mark.parametrize("project", ["local"], indirect=True) def test_full_to_local_project(self, project): - """ - Make a full project a local-only project, and check the transfer + """Make a full project a local-only project, and check the transfer functionality is now restricted. """ project.update_config_file(central_path=None, connection_method=None) @@ -60,8 +57,7 @@ def test_full_to_local_project(self, project): @pytest.mark.parametrize("project", ["local"], indirect=True) def test_local_project_to_full(self, tmp_path, project): - """ - Test updating a local-only project to a full one + """Test updating a local-only project to a full one by adding the required configs (both must be set together) Perform a quick check that data transfer does not error out now that the project is full, and the configs are set as expected. @@ -88,8 +84,7 @@ def test_local_project_to_full(self, tmp_path, project): @pytest.mark.parametrize("project", ["local"], indirect=True) def test_local_to_full_project(self, project): - """ - Change a project from local-only to a normal project by updating + """Change a project from local-only to a normal project by updating the relevant configs. Smoke test that general functionality is maintained and that transfers work correctly. """ @@ -126,8 +121,7 @@ def test_local_to_full_project(self, project): @pytest.mark.parametrize("top_level_folder", ["rawdata", "derivatives"]) @pytest.mark.parametrize("project", ["full"], indirect=True) def test_get_next_sub_and_ses(self, project, top_level_folder): - """ - Make a 'full' project with subject and session > 1 in both local + """Make a 'full' project with subject and session > 1 in both local and central projects. Then, delete the local and run get next sub / ses, and make the project local-only. Call validation requesting to also check central path, which should be ignored as we are in local-only mode. diff --git a/tests/tests_integration/test_logging.py b/tests/tests_integration/test_logging.py index 4e4c3393b..fa193c5b7 100644 --- a/tests/tests_integration/test_logging.py +++ b/tests/tests_integration/test_logging.py @@ -18,12 +18,9 @@ class TestLogging: - @pytest.fixture(scope="function") def teardown_logger(self): - """ - Ensure the logger is deleted at the end of each test. - """ + """Ensure the logger is deleted at the end of each test.""" yield if "datashuttle" in logging.root.manager.loggerDict: logging.root.manager.loggerDict.pop("datashuttle") @@ -33,14 +30,11 @@ def teardown_logger(self): # ------------------------------------------------------------------------- def test_logger_name(self): - """ - Check the canonical logger name. - """ + """Check the canonical logger name.""" assert ds_logger.get_logger_name() == "datashuttle" def test_start_logging(self, tmp_path, teardown_logger): - """ - Test that the central `start` logging function + """Test that the central `start` logging function starts the named logger with the expected handlers. """ assert ds_logger.logging_is_active() is False @@ -57,9 +51,7 @@ def test_start_logging(self, tmp_path, teardown_logger): assert isinstance(logger.handlers[0], logging.FileHandler) def test_shutdown_logger(self, tmp_path, teardown_logger): - """ - Check the log handler remover indeed removes the handles. - """ + """Check the log handler remover indeed removes the handles.""" assert ds_logger.logging_is_active() is False ds_logger.start(tmp_path, "test-command", variables=[]) @@ -72,9 +64,7 @@ def test_shutdown_logger(self, tmp_path, teardown_logger): assert ds_logger.logging_is_active() is False def test_logging_an_error(self, project, teardown_logger): - """ - Check that errors are caught and logged properly. - """ + """Check that errors are caught and logged properly.""" with pytest.raises(NeuroBlueprintError): project.create_folders("rawdata", "sob-001") @@ -89,8 +79,7 @@ def test_logging_an_error(self, project, teardown_logger): @pytest.fixture(scope="function") def clean_project_name(self): - """ - Create an empty project, but ensure no + """Create an empty project, but ensure no configs already exists, and delete created configs after test. @@ -107,8 +96,7 @@ def clean_project_name(self): @pytest.fixture(scope="function") def project(self, tmp_path, clean_project_name, request): - """ - Set `up a project with default configs to use + """Set `up a project with default configs to use for testing. This fixture is distinct from the base.py fixture as requires additional logging setup / teardown. @@ -137,16 +125,15 @@ def project(self, tmp_path, clean_project_name, request): @pytest.mark.parametrize("project", ["local", "full"], indirect=True) def test_log_filename(self, project): - """ - Check the log filename is formatted correctly, for + """Check the log filename is formatted correctly, for `update_config_file`, an arbitrary command """ project.update_config_file(central_host_id="test_id") log_search = list(project.cfg.logging_path.glob("*.log")) - assert ( - len(log_search) == 1 - ), "should only be 1 log in this test environment." + assert len(log_search) == 1, ( + "should only be 1 log in this test environment." + ) log_filename = log_search[0].name regex = re.compile(r"\d{8}T\d{6}_update-config-file.log") @@ -254,8 +241,7 @@ def test_create_folders(self, project): def test_logs_upload_and_download( self, project, upload_or_download, transfer_method ): - """ - Set transfer verbosity and progress settings so + """Set transfer verbosity and progress settings so maximum output is produced to test against. """ subs = ["sub-11"] @@ -311,8 +297,7 @@ def test_logs_upload_and_download( def test_logs_upload_and_download_folder_or_file( self, project, upload_or_download ): - """ - Set transfer verbosity and progress settings so + """Set transfer verbosity and progress settings so maximum output is produced to test against. """ test_utils.make_and_check_local_project_folders( @@ -353,8 +338,7 @@ def test_logs_upload_and_download_folder_or_file( def test_temp_log_folder_moved_make_config_file( self, clean_project_name, tmp_path ): - """ - Check that + """Check that logs are moved to the passed `local_path` when `make_config_file()` is passed. """ @@ -379,8 +363,7 @@ def test_temp_log_folder_moved_make_config_file( assert "make-config-file" in project_path_logs[0] def test_clear_logging_path(self, clean_project_name, tmp_path): - """ - The temporary logging path holds logs which are all + """The temporary logging path holds logs which are all transferred to a new `local_path` when configs are updated. This should only ever be the most recent log action, and not others which may @@ -455,8 +438,7 @@ def test_logs_bad_create_folders_error(self, project): @pytest.mark.parametrize("project", ["local", "full"], indirect=True) def test_validate_project_logging(self, project): - """ - Test that `validate_project` logs errors + """Test that `validate_project` logs errors and warnings to file. """ # Make conflicting subject folders @@ -489,8 +471,7 @@ def test_validate_project_logging(self, project): @pytest.mark.parametrize("project", ["local", "full"], indirect=True) def test_validate_names_against_project_logging(self, project): - """ - Implicitly test `validate_names_against_project` called when + """Implicitly test `validate_names_against_project` called when `make_project_folders` is called, that it logs errors to file. Warnings are not tested. """ diff --git a/tests/tests_integration/test_settings.py b/tests/tests_integration/test_settings.py index b996a1d02..5d5834c92 100644 --- a/tests/tests_integration/test_settings.py +++ b/tests/tests_integration/test_settings.py @@ -11,11 +11,9 @@ class TestPersistentSettings(BaseTest): - @pytest.mark.parametrize("project", ["local", "full"], indirect=True) def test_persistent_settings_name_templates(self, project): - """ - Test the 'name_templates' option that is stored in persistent + """Test the 'name_templates' option that is stored in persistent settings and adds a regexp to validate subject and session names against. @@ -120,13 +118,11 @@ def test_persistent_settings_name_templates(self, project): @pytest.mark.parametrize("project", ["local", "full"], indirect=True) def test_persistent_settings_tui(self, project): - """ - Test persistent settings for the project that + """Test persistent settings for the project that determine display of the TUI. First check defaults are correct, change every one and save, then check they are correct on re-load. """ - # test all defaults settings = project._load_persistent_settings() tui_settings = settings["tui"] @@ -145,8 +141,7 @@ def test_persistent_settings_tui(self, project): @pytest.mark.parametrize("project", ["local", "full"], indirect=True) def test_bypass_validation(self, project): - """ - Check bypass validation which will allow folder + """Check bypass validation which will allow folder creation even when validation fails. Check it is off by default, turn on, check bad name can be created. Reload, turn off, check for error on attempting to create @@ -161,13 +156,12 @@ def test_bypass_validation(self, project): project.create_folders("rawdata", "sub-@@@") assert ( - "BAD_VALUE: The value for prefix sub in name sub-@@@ is not an integer." - == str(e.value) + str(e.value) + == "BAD_VALUE: The value for prefix sub in name sub-@@@ is not an integer." ) def get_settings_default(self): - """ - Hard-coded default settings that should mirror `canonical_configs` + """Hard-coded default settings that should mirror `canonical_configs` and should be changed whenever the canonical configs are changed. This is to protect against accidentally changing these configs. """ @@ -209,9 +203,7 @@ def get_settings_default(self): return default_settings def get_settings_changed(self): - """ - The default settings with every possible setting changed. - """ + """The default settings with every possible setting changed.""" changed_settings = { "create_checkboxes_on": {}, "transfer_checkboxes_on": { diff --git a/tests/tests_integration/test_ssh_file_transfer.py b/tests/tests_integration/test_ssh_file_transfer.py index a94a5b05a..4b4babce4 100644 --- a/tests/tests_integration/test_ssh_file_transfer.py +++ b/tests/tests_integration/test_ssh_file_transfer.py @@ -29,8 +29,7 @@ class TestFileTransfer: ], ) def pathtable_and_project(self, request, tmpdir_factory): - """ - Create a project for SSH testing. Setup + """Create a project for SSH testing. Setup the project as normal, and switch configs to use SSH connection. @@ -49,7 +48,7 @@ def pathtable_and_project(self, request, tmpdir_factory): items have been transferred. This is achieved by using "class" scope. - NOTES + Notes ----- - Pytest params - The `params` key sets the `params` attribute on the pytest `request` fixture. @@ -68,6 +67,7 @@ def pathtable_and_project(self, request, tmpdir_factory): a few seconds after SSH transfer. This makes the tests run very slowly. We can get rid of this limitation on linux. + """ testing_ssh = request.param tmp_path = tmpdir_factory.mktemp("test") @@ -165,8 +165,7 @@ def test_all_data_transfer_options( datatype, upload_or_download, ): - """ - Parse the arguments to filter the pathtable, getting + """Parse the arguments to filter the pathtable, getting the files expected to be transferred passed on the arguments Note files in sub/ses/datatype folders must be handled separately to those in non-sub, non-ses, non-datatype folders @@ -245,8 +244,7 @@ def test_all_data_transfer_options( # --------------------------------------------------------------------------------------------------------------- def query_table(self, pathtable, arguments): - """ - Search the table for arguments, return empty + """Search the table for arguments, return empty if arguments empty """ if any(arguments): @@ -256,8 +254,7 @@ def query_table(self, pathtable, arguments): return folders def parse_arguments(self, pathtable, list_of_names, field): - """ - Replicate datashuttle name formatting by parsing + """Replicate datashuttle name formatting by parsing "all" arguments and turning them into a list of all names, (subject or session), taken from the pathtable. """ @@ -276,8 +273,7 @@ def parse_arguments(self, pathtable, list_of_names, field): return list_of_names def make_pathtable_search_filter(self, sub_names, ses_names, datatype): - """ - Create a string of arguments to pass to pd.query() that will + """Create a string of arguments to pass to pd.query() that will create the table of only transferred sub, ses and datatype. Two arguments must be created, one of all sub / ses / datatypes diff --git a/tests/tests_integration/test_ssh_setup.py b/tests/tests_integration/test_ssh_setup.py index 45ad0f51d..f26b1d921 100644 --- a/tests/tests_integration/test_ssh_setup.py +++ b/tests/tests_integration/test_ssh_setup.py @@ -1,5 +1,4 @@ -""" -SSH configs are set in conftest.py . The password +"""SSH configs are set in conftest.py . The password should be stored in a file called test_ssh_password.txt located in the same folder as test_ssh.py """ @@ -16,8 +15,7 @@ class TestSSH: @pytest.fixture(scope="function") def project(test, tmp_path): - """ - Make a project as per usual, but now add + """Make a project as per usual, but now add in test ssh configurations """ tmp_path = tmp_path / "test with space" @@ -45,8 +43,7 @@ def project(test, tmp_path): def test_verify_ssh_central_host_do_not_accept( self, capsys, project, input_ ): - """ - Use the main function to test this. Test the sub-function + """Use the main function to test this. Test the sub-function when accepting, because this main function will also call setup ssh key pairs which we don't want to do yet @@ -64,8 +61,7 @@ def test_verify_ssh_central_host_do_not_accept( assert "Host not accepted. No connection made.\n" in captured.out def test_verify_ssh_central_host_accept(self, capsys, project): - """ - User is asked to accept the server hostkey. Mock this here + """User is asked to accept the server hostkey. Mock this here and check hostkey is successfully accepted and written to configs. """ test_utils.clear_capsys(capsys) @@ -81,20 +77,19 @@ def test_verify_ssh_central_host_accept(self, capsys, project): captured = capsys.readouterr() assert captured.out == "Host accepted.\n" - with open(project.cfg.hostkeys_path, "r") as file: + with open(project.cfg.hostkeys_path) as file: hostkey = file.readlines()[0] assert f"{project.cfg['central_host_id']} ssh-ed25519 " in hostkey def test_generate_and_write_ssh_key(self, project): - """ - Check ssh key for passwordless connection is written + """Check ssh key for passwordless connection is written to file """ path_to_save = project.cfg["local_path"] / "test" ssh.generate_and_write_ssh_key(path_to_save) - with open(path_to_save, "r") as file: + with open(path_to_save) as file: first_line = file.readlines()[0] assert first_line == "-----BEGIN RSA PRIVATE KEY-----\n" diff --git a/tests/tests_integration/test_transfer_checks.py b/tests/tests_integration/test_transfer_checks.py index 342ed6407..7a6f631f3 100644 --- a/tests/tests_integration/test_transfer_checks.py +++ b/tests/tests_integration/test_transfer_checks.py @@ -15,8 +15,7 @@ class TestTransferChecks(BaseTest): [["rawdata", "derivatives"], ["rawdata"], ["derivatives"]], ) def test_rclone_check(self, project, top_level_folders): - """ - Test rclone.get_local_and_central_file_differences(). This function + """Test rclone.get_local_and_central_file_differences(). This function returns a dictionary where values are list of paths and keys separate based on differences between local and central projects. diff --git a/tests/tests_integration/test_validation.py b/tests/tests_integration/test_validation.py index e24a9ecaa..71fd7d658 100644 --- a/tests/tests_integration/test_validation.py +++ b/tests/tests_integration/test_validation.py @@ -14,7 +14,6 @@ class TestValidation(BaseTest): - @pytest.mark.parametrize( "sub_name", ["sub-001", "sub-999_@DATE@", "sub-001_random-tag_another-tag"], @@ -34,8 +33,7 @@ class TestValidation(BaseTest): def test_warn_on_inconsistent_sub_value_lengths( self, project, sub_name, bad_sub_name ): - """ - This test checks that inconsistent sub value lengths are properly + """This test checks that inconsistent sub value lengths are properly detected across the project. This is performed with an assortment of possible filenames and leading zero conflicts. @@ -94,8 +92,7 @@ def test_warn_on_inconsistent_sub_value_lengths( def test_warn_on_inconsistent_ses_value_lengths( self, project, ses_name, bad_ses_name ): - """ - This function is exactly the same as + """This function is exactly the same as `test_warn_on_inconsistent_sub_value_lengths()` but operates at the session level. This is extreme code duplication, but factoring the main logic out got very messy and hard to follow. @@ -139,8 +136,7 @@ def test_warn_on_inconsistent_ses_value_lengths( @pytest.mark.parametrize("project", ["local", "full"], indirect=True) def test_warn_on_inconsistent_sub_and_ses_value_lengths(self, project): - """ - Test that warning is shown for both subject and session when + """Test that warning is shown for both subject and session when inconsistent zeros are found in both. """ os.makedirs( @@ -173,8 +169,7 @@ def check_inconsistent_sub_or_ses_value_length_warning( @pytest.mark.parametrize("project", ["local", "full"], indirect=True) def test_duplicate_ses_or_sub_key_value_pair(self, project): - """ - Test the check that if a duplicate key is attempt to be made + """Test the check that if a duplicate key is attempt to be made when making a folder e.g. sub-001 exists, then make sub-001_id-123. After this check, make a folder that can be made (e.g. sub-003) just to make sure it does not raise error. @@ -207,8 +202,7 @@ def test_duplicate_ses_or_sub_key_value_pair(self, project): @pytest.mark.parametrize("project", ["local", "full"], indirect=True) def test_duplicate_sub_and_ses_num_leading_zeros(self, project): - """ - Very similar to test_duplicate_ses_or_sub_key_value_pair(), + """Very similar to test_duplicate_ses_or_sub_key_value_pair(), but explicitly check that error is raised if the same number is used with different number of leading zeros. """ @@ -228,8 +222,7 @@ def test_duplicate_sub_and_ses_num_leading_zeros(self, project): @pytest.mark.parametrize("project", ["local", "full"], indirect=True) def test_duplicate_sub_when_creating_session(self, project): - """ - Check the unique case that a duplicate subject is + """Check the unique case that a duplicate subject is introduced when the session is made. """ project.create_folders("rawdata", "sub-001") @@ -273,8 +266,7 @@ def test_duplicate_sub_when_creating_session(self, project): assert "DUPLICATE_NAME" in str(e.value) def test_duplicate_ses_across_subjects(self, project): - """ - Quick test that duplicate session folders only raise + """Quick test that duplicate session folders only raise an error when they are in the same subject. """ project.create_folders("rawdata", "sub-001", "ses-001") @@ -293,8 +285,7 @@ def test_duplicate_ses_across_subjects(self, project): @pytest.mark.parametrize("project", ["local", "full"], indirect=True) def test_invalid_sub_and_ses_name(self, project): - """ - This is a slightly weird case, the name is successfully + """This is a slightly weird case, the name is successfully prefixed as 'sub-sub_100` but when the value if `sub-` is extracted, it is also "sub" and so an error is raised. """ @@ -302,16 +293,16 @@ def test_invalid_sub_and_ses_name(self, project): project.create_folders("rawdata", "sub_100") assert ( - "BAD_VALUE: The value for prefix sub in name sub-sub_100 is not an integer." - == str(e.value) + str(e.value) + == "BAD_VALUE: The value for prefix sub in name sub-sub_100 is not an integer." ) with pytest.raises(NeuroBlueprintError) as e: project.create_folders("rawdata", "sub-001", "ses_100") assert ( - "BAD_VALUE: The value for prefix ses in name ses-ses_100 is not an integer." - == str(e.value) + str(e.value) + == "BAD_VALUE: The value for prefix ses in name ses-ses_100 is not an integer." ) # ------------------------------------------------------------------------- @@ -319,8 +310,7 @@ def test_invalid_sub_and_ses_name(self, project): # ------------------------------------------------------------------------- def test_validate_project(self, project): - """ - Test the `validate_project` function over all it's arguments. + """Test the `validate_project` function over all it's arguments. Note not every validation case is tested exhaustively, these are tested in `test_validation_unit.py` elsewhere here. """ @@ -452,8 +442,7 @@ def test_output_paths_are_valid(self, project): def test_validate_names_against_project_with_bad_existing_names( self, project ): - """ - When using `validate_names_against_project()` there are + """When using `validate_names_against_project()` there are three possible classes of error: 1) error in the passed names. 2) an error already exists in the project. @@ -583,8 +572,7 @@ def test_validate_names_against_project_with_bad_existing_names( @pytest.mark.parametrize("project", ["local", "full"], indirect=True) def test_validate_names_against_project_interactions(self, project): - """ - Check that interactions between the list of names and existing + """Check that interactions between the list of names and existing project are caught. This includes duplicate subject / session names as well as inconsistent subject / session value lengths. """ @@ -706,8 +694,7 @@ def test_validate_names_against_project_interactions(self, project): @pytest.mark.parametrize("project", ["local", "full"], indirect=True) def test_tags_in_name_templates_pass_validation(self, project): - """ - It is useful to allow tags in the `name_templates` as it means + """It is useful to allow tags in the `name_templates` as it means auto-completion in the TUI can use tags for automatic name generation. Because all subject and session names are fully formatted (e.g. @DATE@ converted to actual dates) @@ -760,9 +747,7 @@ def test_tags_in_name_templates_pass_validation(self, project): assert "TEMPLATE: The name: ses-001_datex-20241212" in str(e.value) def test_name_templates_validate_project(self, project): - """ - TODO - """ + """TODO:""" name_templates = { "on": True, "sub": "sub-\d\d_id-\d.?", @@ -835,8 +820,7 @@ def test_quick_validation(self, mocker, project): assert kwargs["name_templates"] == {"on": False} def test_quick_validation_top_level_folder(self, project): - """ - Test that errors are raised as expected on + """Test that errors are raised as expected on bad project path input. """ with pytest.raises(FileNotFoundError) as e: @@ -924,8 +908,7 @@ def test_strict_mode_validation(self, project, top_level_folder): def test_check_high_level_project_structure( self, project, top_level_folder ): - """ - Check that local and central project names are properly formatted + """Check that local and central project names are properly formatted and that """ with pytest.warns(UserWarning) as w: diff --git a/tests/tests_tui/test_local_only_project.py b/tests/tests_tui/test_local_only_project.py index 0599ea144..8a06ccfa7 100644 --- a/tests/tests_tui/test_local_only_project.py +++ b/tests/tests_tui/test_local_only_project.py @@ -5,14 +5,12 @@ class TestTuiLocalOnlyProject(TuiBase): - @pytest.mark.asyncio async def test_local_only_make_project( self, empty_project_paths, ): - """ - Test a local-only project, where the only set config is `local_path`. + """Test a local-only project, where the only set config is `local_path`. Set up a local project, and check the 'Transfer' tab is disabled and set configs are propagated. """ @@ -21,7 +19,6 @@ async def test_local_only_make_project( app = TuiApp() async with app.run_test(size=self.tui_size()) as pilot: - await self.setup_and_check_local_only_project( pilot, project_name, local_path ) @@ -49,8 +46,7 @@ async def test_local_project_to_full( self, empty_project_paths, ): - """ - It is possible to switch between a 'local-only' project (`local_path` + """It is possible to switch between a 'local-only' project (`local_path` only set) and a full project with all configs set, where transfer is allowed. Here start as a local project then set configs so we become a full project. """ @@ -60,7 +56,6 @@ async def test_local_project_to_full( app = TuiApp() async with app.run_test(size=self.tui_size()) as pilot: - # Set up a local-only project await self.setup_and_check_local_only_project( pilot, project_name, local_path @@ -125,8 +120,7 @@ async def test_full_project_to_local( self, setup_project_paths, ): - """ - Very similar to `test_check_local_only_project_to_full()`, but + """Very similar to `test_check_local_only_project_to_full()`, but going from a full project to a local only. This still requires a refresh so the full transfer tab can be set to a placeholder. """ @@ -135,7 +129,6 @@ async def test_full_project_to_local( app = TuiApp() async with app.run_test(size=self.tui_size()) as pilot: - # Fixture generated a full project, switch to it's # project manager screen here. await self.check_and_click_onto_existing_project( @@ -187,8 +180,7 @@ async def test_full_project_to_local( async def setup_and_check_local_only_project( self, pilot, project_name, local_path ): - """ - Set up a local-only project by filling in the `local_path` and setting + """Set up a local-only project by filling in the `local_path` and setting the radio button to the no-connection option. """ # Move to configs window diff --git a/tests/tests_tui/test_tui_configs.py b/tests/tests_tui/test_tui_configs.py index da853f8bc..a9f4d8fe0 100644 --- a/tests/tests_tui/test_tui_configs.py +++ b/tests/tests_tui/test_tui_configs.py @@ -14,7 +14,6 @@ class TestTuiConfigs(TuiBase): - # ------------------------------------------------------------------------- # Test New Project Configs # ------------------------------------------------------------------------- @@ -26,8 +25,7 @@ async def test_make_new_project_configs( empty_project_paths, kwargs_set, ): - """ - Check the ConfigsContent when making a new project. This contains + """Check the ConfigsContent when making a new project. This contains many widgets shared with the ConfigsContent on the tab page, however also includes an additional information banner and input for the project name. @@ -67,7 +65,6 @@ async def test_make_new_project_configs( app = TuiApp() async with app.run_test(size=self.tui_size()) as pilot: - # Select a new project, check NewProjectScreen is # displayed correctly. await self.scroll_to_click_pause( @@ -158,8 +155,7 @@ async def test_make_new_project_configs( async def test_update_config_on_project_manager_screen( self, setup_project_paths ): - """ - Test the ConfigsContent on the project manager tab screen. + """Test the ConfigsContent on the project manager tab screen. The project is set up in the fixture, navigate to the project page. Check that the default configs are displayed. Change all the configs, save, and check these are updated on the config file and on the @@ -172,7 +168,6 @@ async def test_update_config_on_project_manager_screen( app = TuiApp() async with app.run_test(size=self.tui_size()) as pilot: - # Navigate to the existing project and click onto the # configs tab. await self.check_and_click_onto_existing_project( @@ -214,7 +209,7 @@ async def test_update_config_on_project_manager_screen( "central_host_username": "random_username", } - for key in new_kwargs.keys(): + for key in new_kwargs: # The purpose is to update to completely new configs assert new_kwargs[key] != project_cfg[key] @@ -267,8 +262,7 @@ async def test_update_config_on_project_manager_screen( @pytest.mark.asyncio async def test_configs_select_path(self, monkeypatch): - """ - Test the 'Select' buttons / DirectoryTree on the ConfigsContent. + """Test the 'Select' buttons / DirectoryTree on the ConfigsContent. These are used to select folders that are filled into the Input. Open the select dialog, select a folder, check the path is filled into the Input. There is one for both local @@ -281,7 +275,6 @@ async def test_configs_select_path(self, monkeypatch): app = TuiApp() async with app.run_test(size=self.tui_size()) as pilot: - # Select the page and ConfigsContent for setting up new project await self.scroll_to_click_pause( pilot, "#mainwindow_new_project_button" @@ -344,10 +337,8 @@ async def test_configs_select_path(self, monkeypatch): @pytest.mark.asyncio async def test_bad_configs_screen_input(self, empty_project_paths): - app = TuiApp() async with app.run_test(size=self.tui_size()) as pilot: - # Select a new project, check NewProjectScreen is displayed correctly. await self.scroll_to_click_pause( pilot, "#mainwindow_new_project_button" @@ -375,11 +366,9 @@ async def test_bad_configs_screen_input(self, empty_project_paths): async def check_configs_widgets_match_configs( self, configs_content, kwargs ): - """ - Check that the widgets of the TUI configs match those found + """Check that the widgets of the TUI configs match those found in `kwargs`. """ - # Local Path ---------------------------------------------------------- assert ( @@ -402,7 +391,6 @@ async def check_configs_widgets_match_configs( ) if kwargs["connection_method"] == "ssh": - # Central Host ID ------------------------------------------------- assert ( @@ -429,11 +417,9 @@ async def check_configs_widgets_match_configs( ) async def set_configs_content_widgets(self, pilot, kwargs): - """ - Given a dict of options that can be set on the configs TUI + """Given a dict of options that can be set on the configs TUI in kwargs, set all configs widgets according to kwargs. """ - # Local Path ---------------------------------------------------------- await self.fill_input( @@ -443,7 +429,6 @@ async def set_configs_content_widgets(self, pilot, kwargs): # Connection Method --------------------------------------------------- if kwargs["connection_method"] == "ssh": - await self.scroll_to_click_pause(pilot, "#configs_ssh_radiobutton") # Central Host ID ------------------------------------------------- @@ -471,8 +456,7 @@ async def set_configs_content_widgets(self, pilot, kwargs): async def check_new_project_configs( self, pilot, project_name, configs_content, kwargs ): - """ - Check the configs displayed on the TUI match those found in `kwargs`. + """Check the configs displayed on the TUI match those found in `kwargs`. Also, check the widgets unique to ConfigsContent on the configs selection for a new project. """ diff --git a/tests/tests_tui/test_tui_create_folders.py b/tests/tests_tui/test_tui_create_folders.py index 00a408f76..8fbc32a2a 100644 --- a/tests/tests_tui/test_tui_create_folders.py +++ b/tests/tests_tui/test_tui_create_folders.py @@ -13,7 +13,6 @@ class TestTuiCreateFolders(TuiBase): - # ------------------------------------------------------------------------- # General test Create Folders # ------------------------------------------------------------------------- @@ -23,9 +22,7 @@ class TestTuiCreateFolders(TuiBase): async def test_create_folders_sub_and_ses( self, setup_project_paths, test_multi_input ): - """ - Basic test that folders are created as expected through the TUI. - """ + """Basic test that folders are created as expected through the TUI.""" # Define folders to create if test_multi_input: sub_text = "sub-001, sub-002" @@ -42,7 +39,6 @@ async def test_create_folders_sub_and_ses( app = TuiApp() async with app.run_test(size=self.tui_size()) as pilot: - # Set up the TUI on the 'create' tab, filling the # input with the subject and session folders to create. await self.check_and_click_onto_existing_project( @@ -87,8 +83,7 @@ async def test_create_folders_sub_and_ses( @pytest.mark.asyncio async def test_create_folders_formatted_names(self, setup_project_paths): - """ - Test preview tooltips and create folders with _@DATE@ formatting. + """Test preview tooltips and create folders with _@DATE@ formatting. The @TO@ key is not tested. """ tmp_config_path, tmp_path, project_name = setup_project_paths.values() @@ -181,15 +176,13 @@ async def test_create_folders_formatted_names(self, setup_project_paths): async def test_create_folders_bad_validation_tooltips( self, setup_project_paths ): - """ - Check that correct tooltips are displayed when + """Check that correct tooltips are displayed when various invalid subject or session names are provided. """ tmp_config_path, tmp_path, project_name = setup_project_paths.values() app = TuiApp() async with app.run_test(size=self.tui_size()) as pilot: - await self.check_and_click_onto_existing_project( pilot, project_name ) @@ -259,8 +252,7 @@ async def test_create_folders_bad_validation_tooltips( async def test_validation_error_and_bypass_validation( self, setup_project_paths ): - """ - Test validation and bypass validation options by + """Test validation and bypass validation options by first trying to create an invalid folder name, and checking an error displays. Next, turn on 'bypass validation' and check the folders are created despite being invalid. @@ -269,7 +261,6 @@ async def test_validation_error_and_bypass_validation( app = TuiApp() async with app.run_test(size=self.tui_size()) as pilot: - await self.check_and_click_onto_existing_project( pilot, project_name ) @@ -338,15 +329,13 @@ async def test_validation_error_and_bypass_validation( async def test_name_template_next_sub_or_ses_and_validation( self, setup_project_paths ): - """ - Test validation and double-click for next sub / ses + """Test validation and double-click for next sub / ses values when 'name templates' is set in the 'Settings' window. """ tmp_config_path, tmp_path, project_name = setup_project_paths.values() app = TuiApp() async with app.run_test(size=self.tui_size()) as pilot: - await self.check_and_click_onto_existing_project( pilot, project_name ) @@ -468,15 +457,13 @@ async def test_name_template_next_sub_or_ses_and_validation( @pytest.mark.asyncio async def test_get_next_sub_and_ses_no_template(self, setup_project_paths): - """ - Test the double click on Input correctly fills with the + """Test the double click on Input correctly fills with the next sub or ses (or prefix only when CTRL is pressed). """ tmp_config_path, tmp_path, project_name = setup_project_paths.values() app = TuiApp() async with app.run_test(size=self.tui_size()) as pilot: - await self.setup_existing_project_create_tab_filled_sub_and_ses( pilot, project_name, create_folders=True ) @@ -532,15 +519,13 @@ async def test_get_next_sub_and_ses_no_template(self, setup_project_paths): async def test_create_folders_settings_top_level_folder( self, setup_project_paths ): - """ - Check the folders are created in the correct top level + """Check the folders are created in the correct top level folder when this is changed in the 'Settings' screen. """ tmp_config_path, tmp_path, project_name = setup_project_paths.values() app = TuiApp() async with app.run_test(size=self.tui_size()) as pilot: - # Open the CreateFoldersSettingsScreen await self.setup_existing_project_create_tab_filled_sub_and_ses( pilot, project_name, create_folders=False @@ -619,7 +604,6 @@ async def iterate_and_check_all_datatype_folders( folder_used = test_utils.get_all_broad_folders_used(value=False) for datatype in canonical_configs.get_broad_datatypes(): - await self.scroll_to_click_pause( pilot, f"#create_{datatype}_checkbox", diff --git a/tests/tests_tui/test_tui_datatypes.py b/tests/tests_tui/test_tui_datatypes.py index 508c2d06a..48bcbf27c 100644 --- a/tests/tests_tui/test_tui_datatypes.py +++ b/tests/tests_tui/test_tui_datatypes.py @@ -7,9 +7,7 @@ class TestDatatypesTUI(TuiBase): - """ - Test the datatype selection screen for the Create and Transfer tab. - """ + """Test the datatype selection screen for the Create and Transfer tab.""" @pytest.mark.asyncio async def test_select_displayed_datatypes_create( @@ -19,7 +17,6 @@ async def test_select_displayed_datatypes_create( app = TuiApp() async with app.run_test(size=self.tui_size()) as pilot: - # Set up the TUI on the 'create' tab, filling the # input with the subject and session folders to create. await self.check_and_click_onto_existing_project( @@ -134,7 +131,6 @@ async def test_select_displayed_datatypes_transfer( app = TuiApp() async with app.run_test(size=self.tui_size()) as pilot: - # Set up the TUI on the 'transfer' tab (custom) and # open the datatype selection screen await self.check_and_click_onto_existing_project( diff --git a/tests/tests_tui/test_tui_directorytree.py b/tests/tests_tui/test_tui_directorytree.py index bb93bb34d..11562e085 100644 --- a/tests/tests_tui/test_tui_directorytree.py +++ b/tests/tests_tui/test_tui_directorytree.py @@ -14,22 +14,19 @@ class TestTuiCreateDirectoryTree(TuiBase): - """ - Test the `Create` tab directory tree. + """Test the `Create` tab directory tree. `Transfer` """ @pytest.mark.asyncio async def test_fill_and_append_next_sub_and_ses(self, setup_project_paths): - """ - Test the CTRL+F and CTRL+A functions on the directorytree + """Test the CTRL+F and CTRL+A functions on the directorytree that fill and append subject / session name to the inputs. """ tmp_config_path, tmp_path, project_name = setup_project_paths.values() app = TuiApp() async with app.run_test(size=self.tui_size()) as pilot: - # Open the create tab and first fill the subject # and session inputs with -001. await self.check_and_click_onto_existing_project( @@ -116,8 +113,7 @@ async def test_fill_and_append_next_sub_and_ses(self, setup_project_paths): async def test_create_folders_directorytree_clipboard( self, setup_project_paths ): - """ - Check that pressing CTRL+Q on the directorytree copies the + """Check that pressing CTRL+Q on the directorytree copies the hovered folder to the clipboard (using pyperclip). """ tmp_config_path, tmp_path, project_name = setup_project_paths.values() @@ -159,7 +155,6 @@ async def test_failed_pyperclip_copy( app = TuiApp() async with app.run_test(size=self.tui_size()) as pilot: - # Set up a project and navigate to the directory tree await self.setup_existing_project_create_tab_filled_sub_and_ses( pilot, project_name, create_folders=True @@ -197,8 +192,7 @@ def mock_copy(_): async def test_create_folders_directorytree_open_filesystem( self, setup_project_paths, monkeypatch ): - """ - Test pressing CTRL+O on the filetree triggers the opening + """Test pressing CTRL+O on the filetree triggers the opening of a folder through the show-in-file-manager package (monkeypatched function). """ @@ -206,7 +200,6 @@ async def test_create_folders_directorytree_open_filesystem( app = TuiApp() async with app.run_test(size=self.tui_size()) as pilot: - # Set up the 'create tab' with loaded nodes await self.setup_existing_project_create_tab_filled_sub_and_ses( pilot, project_name, create_folders=True diff --git a/tests/tests_tui/test_tui_get_help.py b/tests/tests_tui/test_tui_get_help.py index 55d876f35..aa8986b8c 100644 --- a/tests/tests_tui/test_tui_get_help.py +++ b/tests/tests_tui/test_tui_get_help.py @@ -5,17 +5,14 @@ class TestTuiSettings(TuiBase): - """ - Test that the 'Get Help' page from the main menu. + """Test that the 'Get Help' page from the main menu. Open it, check the expected label is displayed, close it. """ @pytest.mark.asyncio async def test_get_help(self, empty_project_paths): - app = TuiApp() async with app.run_test(size=self.tui_size()) as pilot: - await self.scroll_to_click_pause( pilot, "#mainwindow_get_help_button" ) diff --git a/tests/tests_tui/test_tui_logging.py b/tests/tests_tui/test_tui_logging.py index cb4b8417f..6a046c3e8 100644 --- a/tests/tests_tui/test_tui_logging.py +++ b/tests/tests_tui/test_tui_logging.py @@ -7,11 +7,9 @@ class TestTuiLogging(TuiBase): - @pytest.mark.asyncio async def test_logging(self, setup_project_paths): - """ - Test logging by running some commands, checking they + """Test logging by running some commands, checking they are displayed on the logging tree, that the most recent log is correct and that the log screen opens when clicked. """ @@ -19,7 +17,6 @@ async def test_logging(self, setup_project_paths): app = TuiApp() async with app.run_test(size=self.tui_size()) as pilot: - # Update configs and create folders to make some logs project = DataShuttle(project_name) @@ -63,7 +60,9 @@ async def test_logging(self, setup_project_paths): ) assert ( "create-folders" in widg.get_node_at_line(2).data.path.stem - ), f"ERROR MESSAGE: {widg.get_node_at_line(0).data.path}-{widg.get_node_at_line(1).data.path}-{widg.get_node_at_line(2).data.path}" + ), ( + f"ERROR MESSAGE: {widg.get_node_at_line(0).data.path}-{widg.get_node_at_line(1).data.path}-{widg.get_node_at_line(2).data.path}" + ) # Check the latest logging path is correct assert ( diff --git a/tests/tests_tui/test_tui_settings.py b/tests/tests_tui/test_tui_settings.py index 77a65b090..8b9bcaa4d 100644 --- a/tests/tests_tui/test_tui_settings.py +++ b/tests/tests_tui/test_tui_settings.py @@ -5,20 +5,16 @@ class TestTuiSettings(TuiBase): - """ - Test the 'Settings' screen accessible from the Main Menu. - """ + """Test the 'Settings' screen accessible from the Main Menu.""" @pytest.mark.asyncio async def test_light_dark_mode(self): - """ - Check the light / dark mode switch which is stored + """Check the light / dark mode switch which is stored in the global tui settings. Global refers to set across all projects not related to a specific project. """ app = TuiApp() async with app.run_test(size=self.tui_size()) as pilot: - await self.scroll_to_click_pause( pilot, "#mainwindow_settings_button" ) @@ -45,8 +41,7 @@ async def test_light_dark_mode(self): @pytest.mark.asyncio async def test_show_transfer_tree_status(self, setup_project_paths): - """ - Check the 'show transfer tree' option that turns off transfer + """Check the 'show transfer tree' option that turns off transfer tree styling by default has the intended effects. It is difficult to test whether the tree is actually styled, so here all underlying configs + the transfer tree legend @@ -56,7 +51,6 @@ async def test_show_transfer_tree_status(self, setup_project_paths): app = TuiApp() async with app.run_test(size=self.tui_size()) as pilot: - # First check the show transfer tree styling is off # in the project manager tab and legend does not exist. await self.check_and_click_onto_existing_project( diff --git a/tests/tests_tui/test_tui_transfer.py b/tests/tests_tui/test_tui_transfer.py index 3e8c389f1..1fd3fe60a 100644 --- a/tests/tests_tui/test_tui_transfer.py +++ b/tests/tests_tui/test_tui_transfer.py @@ -7,8 +7,7 @@ class TestTuiTransfer(TuiBase): - """ - Test transferring through the TUI (entire project, top + """Test transferring through the TUI (entire project, top level only or custom). This class leverages the underlying test utils that check API transfers. """ @@ -24,7 +23,6 @@ async def test_transfer_entire_project( app = TuiApp() async with app.run_test(size=self.tui_size()) as pilot: - await self.check_and_click_onto_existing_project( pilot, project_name ) @@ -57,8 +55,7 @@ async def test_transfer_entire_project( await pilot.pause() async def check_persistent_settings(self, pilot): - """ - Run transfer with each overwrite setting and check it is propagated + """Run transfer with each overwrite setting and check it is propagated to datashuttle methods. """ await self.set_and_check_persistent_settings(pilot, "never", True) @@ -89,8 +86,7 @@ async def set_transfer_tab_dry_run_checkbox(self, pilot, dry_run_setting): async def set_and_check_persistent_settings( self, pilot, overwrite_setting, dry_run_setting ): - """ - Run transfer with an overwrite setting and check it is propagated + """Run transfer with an overwrite setting and check it is propagated to datashuttle methods by checking the logs. """ await self.set_overwrite_checkbox(pilot, overwrite_setting) @@ -118,7 +114,6 @@ async def test_transfer_top_level_folder( app = TuiApp() async with app.run_test(size=self.tui_size()) as pilot: - await self.check_and_click_onto_existing_project( pilot, project_name ) @@ -167,7 +162,6 @@ async def test_transfer_custom( app = TuiApp() async with app.run_test(size=self.tui_size()) as pilot: - await self.check_and_click_onto_existing_project( pilot, project_name ) @@ -228,7 +222,6 @@ async def test_transfer_custom( async def switch_top_level_folder_select( self, pilot, id, top_level_folder ): - if top_level_folder == "rawdata": assert pilot.app.screen.query_one(id).value == "rawdata" else: diff --git a/tests/tests_tui/test_tui_widgets_and_defaults.py b/tests/tests_tui/test_tui_widgets_and_defaults.py index e0b330797..897d066ce 100644 --- a/tests/tests_tui/test_tui_widgets_and_defaults.py +++ b/tests/tests_tui/test_tui_widgets_and_defaults.py @@ -14,8 +14,7 @@ class TestTuiWidgets(TuiBase): - """ - This class performs fundamental checks on the default display + """This class performs fundamental checks on the default display of widgets and that changing widgets properly change underlying configs. This does not perform any functional tests e.g. creation of configs of new files. @@ -27,12 +26,9 @@ class TestTuiWidgets(TuiBase): @pytest.mark.asyncio async def test_new_project_configs(self, empty_project_paths): - """ - Test all widgets display as expected on the New Project configs page. - """ + """Test all widgets display as expected on the New Project configs page.""" app = TuiApp() async with app.run_test(size=self.tui_size()) as pilot: - # Select a new project, check NewProjectScreen is displayed correctly. await self.scroll_to_click_pause( pilot, "#mainwindow_new_project_button" @@ -127,7 +123,6 @@ async def test_new_project_configs(self, empty_project_paths): == "" ) if platform.system() == "Windows": - assert ( configs_content.query_one( "#configs_central_path_input" @@ -240,8 +235,7 @@ async def check_new_project_ssh_widgets( @pytest.mark.asyncio async def test_existing_project_configs(self, setup_project_paths): - """ - Because the underlying screen is shared between new and existing + """Because the underlying screen is shared between new and existing project configs, in the existing project configs just check widgets are hidden as expected. """ @@ -249,7 +243,6 @@ async def test_existing_project_configs(self, setup_project_paths): app = TuiApp() async with app.run_test(size=self.tui_size()) as pilot: - # Navigate to the existing project and click onto the # configs tab. await self.check_and_click_onto_existing_project( @@ -278,15 +271,13 @@ async def test_existing_project_configs(self, setup_project_paths): @pytest.mark.asyncio async def test_create_folders_widgets_display(self, setup_project_paths): - """ - Test all widgets on the 'Create' tab of the project manager screen + """Test all widgets on the 'Create' tab of the project manager screen are displayed as expected. """ tmp_config_path, tmp_path, project_name = setup_project_paths.values() app = TuiApp() async with app.run_test(size=self.tui_size()) as pilot: - await self.check_and_click_onto_existing_project( pilot, project_name ) @@ -370,15 +361,13 @@ async def test_create_folders_widgets_display(self, setup_project_paths): @pytest.mark.asyncio async def test_create_folder_settings_widgets(self, setup_project_paths): - """ - Test the widgets in the 'Settings' menu of the project + """Test the widgets in the 'Settings' menu of the project manager's 'Create' tab. """ tmp_config_path, tmp_path, project_name = setup_project_paths.values() app = TuiApp() async with app.run_test(size=self.tui_size()) as pilot: - await self.setup_existing_project_create_tab_filled_sub_and_ses( pilot, project_name, create_folders=False ) @@ -487,8 +476,7 @@ async def test_create_folder_settings_widgets(self, setup_project_paths): async def test_name_templates_widgets_and_settings( self, setup_project_paths ): - """ - Check the 'Name Templates' section of the 'Create' tab 'Settings + """Check the 'Name Templates' section of the 'Create' tab 'Settings page. Here both subject and session configs share the same input, so ensure these are mapped correctly by the radiobutton setting, and that the underlying configs are set correctly. @@ -500,7 +488,6 @@ async def test_name_templates_widgets_and_settings( app = TuiApp() async with app.run_test(size=self.tui_size()) as pilot: - await self.setup_existing_project_create_tab_filled_sub_and_ses( pilot, project_name, create_folders=False ) @@ -650,15 +637,13 @@ async def test_name_templates_widgets_and_settings( @pytest.mark.asyncio async def test_bypass_validation_settings(self, setup_project_paths): - """ - Test all configs that underly the 'bypass validation' + """Test all configs that underly the 'bypass validation' setting are updated correctly by the widget. """ tmp_config_path, tmp_path, project_name = setup_project_paths.values() app = TuiApp() async with app.run_test(size=self.tui_size()) as pilot: - await self.setup_existing_project_create_tab_filled_sub_and_ses( pilot, project_name, create_folders=False ) @@ -717,15 +702,13 @@ async def test_bypass_validation_settings(self, setup_project_paths): @pytest.mark.asyncio async def test_all_top_level_folder_selects(self, setup_project_paths): - """ - Test all 'top level folder' selects (in Create and Transfer tabs) + """Test all 'top level folder' selects (in Create and Transfer tabs) update the underlying configs correctly. """ tmp_config_path, tmp_path, project_name = setup_project_paths.values() app = TuiApp() async with app.run_test(size=self.tui_size()) as pilot: - # Open project, check top level folder are correct await self.setup_existing_project_create_tab_filled_sub_and_ses( pilot, project_name, create_folders=False @@ -890,9 +873,7 @@ async def check_top_folder_select( expected_val, move_to_position: Union[bool, int] = False, ): - """ - If move to position is not False, must be int specifying position - """ + """If move to position is not False, must be int specifying position""" if move_to_position: await self.move_select_to_position(pilot, id, move_to_position) @@ -913,8 +894,7 @@ async def check_top_folder_select( @pytest.mark.asyncio async def test_all_checkboxes(self, setup_project_paths): - """ - Check all datatype checkboxes (Create and Transfer tab) + """Check all datatype checkboxes (Create and Transfer tab) correctly update the underlying configs. These are tested together to ensure there are no strange interaction between these as they both share stored in the project's 'tui' @@ -924,7 +904,6 @@ async def test_all_checkboxes(self, setup_project_paths): app = TuiApp() async with app.run_test(size=self.tui_size()) as pilot: - await self.check_and_click_onto_existing_project( pilot, project_name ) @@ -1041,7 +1020,6 @@ async def test_all_transfer_widgets(self, setup_project_paths): app = TuiApp() async with app.run_test(size=self.tui_size()) as pilot: - # Navigate to the existing project and click onto the # configs tab. await self.check_and_click_onto_existing_project( @@ -1228,7 +1206,6 @@ async def test_overwrite_existing_files(self, setup_project_paths): app = TuiApp() async with app.run_test(size=self.tui_size()) as pilot: - # Navigate to the existing project and click onto the # configs tab. await self.check_and_click_onto_existing_project( @@ -1280,8 +1257,7 @@ async def test_overwrite_existing_files(self, setup_project_paths): @pytest.mark.asyncio async def test_dry_run(self, setup_project_paths): - """ - Test the dry run setting. This is very similar in structure + """Test the dry run setting. This is very similar in structure to `test_overwrite_existing_files()`, merge if more persistent settings added. """ @@ -1289,7 +1265,6 @@ async def test_dry_run(self, setup_project_paths): app = TuiApp() async with app.run_test(size=self.tui_size()) as pilot: - # Navigate to the existing project and click onto the # configs tab. await self.check_and_click_onto_existing_project( diff --git a/tests/tests_tui/tui_base.py b/tests/tests_tui/tui_base.py index 28351074a..fc20cbbc2 100644 --- a/tests/tests_tui/tui_base.py +++ b/tests/tests_tui/tui_base.py @@ -8,13 +8,10 @@ class TuiBase: - """ - Contains fixtuers and helper functions for TUI tests. - """ + """Contains fixtuers and helper functions for TUI tests.""" def tui_size(self): - """ - If the TUI screen in the test environment is not + """If the TUI screen in the test environment is not large enough, often the error `textual.pilot.OutOfBounds: Target offset is outside of currently-visible screen region.` @@ -27,8 +24,7 @@ def tui_size(self): @pytest_asyncio.fixture(scope="function") async def empty_project_paths(self, tmp_path_factory, monkeypatch): - """ - Get the paths and project name for a non-existent (i.e. not + """Get the paths and project name for a non-existent (i.e. not yet setup) project. """ project_name = "my-test-project" @@ -48,9 +44,7 @@ async def empty_project_paths(self, tmp_path_factory, monkeypatch): @pytest_asyncio.fixture(scope="function") async def setup_project_paths(self, empty_project_paths): - """ - Get the paths and project name for a setup project. - """ + """Get the paths and project name for a setup project.""" test_utils.setup_project_fixture( empty_project_paths["tmp_path"], empty_project_paths["project_name"], @@ -59,8 +53,7 @@ async def setup_project_paths(self, empty_project_paths): return empty_project_paths def monkeypatch_get_datashuttle_path(self, tmp_config_path, _monkeypatch): - """ - For these tests, store the datashuttle configs (usually stored in + """For these tests, store the datashuttle configs (usually stored in Path.home()) in the `tmp_path` provided by pytest, as it simplifies testing here. @@ -79,8 +72,7 @@ def mock_get_datashuttle_path(): ) def monkeypatch_print(self, _monkeypatch): - """ - Calls to `print` in datashuttle crash the TUI in the + """Calls to `print` in datashuttle crash the TUI in the test environment. I am not sure why. Get around this in tests by monkeypatching the datashuttle print method. """ @@ -93,9 +85,7 @@ def return_none(arg1, arg2=None): ) async def fill_input(self, pilot, id, value): - """ - Fill and input of `id` with `value`. - """ + """Fill and input of `id` with `value`.""" await self.scroll_to_click_pause(pilot, id) pilot.app.screen.query_one(id).value = "" await pilot.press(*value) @@ -104,8 +94,7 @@ async def fill_input(self, pilot, id, value): async def setup_existing_project_create_tab_filled_sub_and_ses( self, pilot, project_name, create_folders=False ): - """ - Set up an existing project and switch to the 'Create' tab + """Set up an existing project and switch to the 'Create' tab on the project manager screen. """ await self.check_and_click_onto_existing_project(pilot, project_name) @@ -123,16 +112,14 @@ async def setup_existing_project_create_tab_filled_sub_and_ses( ) async def double_click(self, pilot, id, control=False): - """ - Double-click on a widget of `id`, if `control` is `True` the + """Double-click on a widget of `id`, if `control` is `True` the control modifier key will be used. """ for _ in range(2): await self.scroll_to_click_pause(pilot, id, control=control) async def reload_tree_nodes(self, pilot, id, num_nodes): - """ - For some reason, for TUI tree nodes to register in the + """For some reason, for TUI tree nodes to register in the test environment all need to have `reload_node` called on the node. """ @@ -143,41 +130,32 @@ async def reload_tree_nodes(self, pilot, id, num_nodes): await pilot.pause() async def hover_and_press_tree(self, pilot, id, hover_line, press_string): - """ - Hover over a directorytree at a node-line and press a specific string - """ + """Hover over a directorytree at a node-line and press a specific string""" await pilot.pause() pilot.app.screen.query_one(id).hover_line = hover_line await pilot.pause() await self.press_tree(pilot, id, press_string) async def press_tree(self, pilot, id, press_string): - """ - Click on a tree to give it focus and press buttons - """ + """Click on a tree to give it focus and press buttons""" await self.scroll_to_click_pause(pilot, id) await pilot.press(press_string) await pilot.pause() async def scroll_to_and_pause(self, pilot, id): - """ - Scroll to a widget and pause. - """ + """Scroll to a widget and pause.""" widget = pilot.app.screen.query_one(id) widget.scroll_visible(animate=False) await pilot.pause() async def scroll_to_click_pause(self, pilot, id, control=False): - """ - Scroll to a widget, click it and call pause. - """ + """Scroll to a widget, click it and call pause.""" await self.scroll_to_and_pause(pilot, id) await pilot.click(id, control=control) await pilot.pause() async def check_and_click_onto_existing_project(self, pilot, project_name): - """ - From the main menu, go onto the select project page and + """From the main menu, go onto the select project page and select the project created in the test environment. Perform general TUI checks during the navigation. """ @@ -208,9 +186,7 @@ async def switch_tab(self, pilot, tab): await self.scroll_to_click_pause(pilot, f"Tab#{content_tab}") async def turn_off_all_datatype_checkboxes(self, pilot, tab="create"): - """ - Make sure all checkboxes are off to start - """ + """Make sure all checkboxes are off to start""" assert tab in ["create", "transfer"] checkbox_names = canonical_configs.get_broad_datatypes() @@ -234,8 +210,7 @@ async def turn_off_all_datatype_checkboxes(self, pilot, tab="create"): async def exit_to_main_menu_and_reeneter_project_manager( self, pilot, project_name ): - """ - Exist from the project manager screen, then re-enter back + """Exist from the project manager screen, then re-enter back into the project. This refreshes the screen and is important in testing state is preserved across re-loading. """ @@ -244,15 +219,12 @@ async def exit_to_main_menu_and_reeneter_project_manager( await self.check_and_click_onto_existing_project(pilot, project_name) async def close_messagebox(self, pilot): - """ - Close the modal_dialogs.Messagebox - """ + """Close the modal_dialogs.Messagebox""" pilot.app.screen.on_button_pressed() await pilot.pause() async def move_select_to_position(self, pilot, id, position): - """ - Move a select widget to a specific position (e.g. "rawdata" + """Move a select widget to a specific position (e.g. "rawdata" or "derivatives" select). The position can be determined by trial and error. """ diff --git a/tests/tests_unit/test_links.py b/tests/tests_unit/test_links.py index 5da0e04c2..86adc8b9f 100644 --- a/tests/tests_unit/test_links.py +++ b/tests/tests_unit/test_links.py @@ -4,8 +4,7 @@ def test_links(): - """ - Test canonical links are working. Unfortunately Zulip links cannot + """Test canonical links are working. Unfortunately Zulip links cannot be validated. """ assert validators.url(links.get_docs_link()) diff --git a/tests/tests_unit/test_unit.py b/tests/tests_unit/test_unit.py index 23257bf6f..3c55b526a 100644 --- a/tests/tests_unit/test_unit.py +++ b/tests/tests_unit/test_unit.py @@ -7,9 +7,7 @@ class TestUnit: - """ - Currently contains misc. unit tests. - """ + """Currently contains misc. unit tests.""" @pytest.mark.parametrize( "underscore_position", ["left", "right", "both", "none"] @@ -18,8 +16,7 @@ class TestUnit: "key", [tags("date"), tags("time"), tags("datetime")] ) def test_datetime_string_replacement(self, key, underscore_position): - """ - Test the function that replaces @DATE, @TIME@ or @DATETIME@ + """Test the function that replaces @DATE, @TIME@ or @DATETIME@ keywords with the date / time / datetime. Also, it will pre/append underscores to the tags if they are not already there (e.g if user input "sub-001@DATE"). @@ -42,9 +39,9 @@ def test_datetime_string_replacement(self, key, underscore_position): name_list = [name] formatting.update_names_with_datetime(name_list) - assert ( - re.search(regex, name_list[0]) is not None - ), "datetime formatting is incorrect." + assert re.search(regex, name_list[0]) is not None, ( + "datetime formatting is incorrect." + ) @pytest.mark.parametrize("prefix", ["sub", "ses"]) def test_process_to_keyword_in_sub_input(self, prefix): @@ -109,8 +106,7 @@ def test_process_to_keyword_bad_input_raises_error( ) def test_get_value_from_bids_name_regexp(self): - """ - Test the regexp that finds the value from a BIDS-name + """Test the regexp that finds the value from a BIDS-name key-value pair. """ bids_name = "sub-0123125_ses-11312_datetime-5345323_id-3asd@523" @@ -128,8 +124,7 @@ def test_get_value_from_bids_name_regexp(self): assert id == "3asd@523" def test_num_leading_zeros(self): - """ - Check num_leading_zeros handles prefixed and non-prefixed + """Check num_leading_zeros handles prefixed and non-prefixed case from -1 to -(101x 0)1. """ for i in range(101): @@ -145,8 +140,7 @@ def test_num_leading_zeros(self): def test_get_max_sub_or_ses_num_and_value_length_empty( self, prefix, default_num_value_digits ): - """ - When the list of sub or ses names is empty, the returned max number + """When the list of sub or ses names is empty, the returned max number should be zero and the `default_num_value_digits` be set to the passed default """ @@ -162,8 +156,7 @@ def test_get_max_sub_or_ses_num_and_value_length_empty( @pytest.mark.parametrize("prefix", ["sub", "ses"]) def test_get_max_sub_or_ses_num_and_value_length_error(self, prefix): - """ - An error will be shown if the sub or ses value digits are inconsistent, + """An error will be shown if the sub or ses value digits are inconsistent, because it is not possible to return the number of values required. A warning should be shown in that the number of value digits are @@ -213,8 +206,7 @@ def test_get_max_sub_or_ses_num_and_value_length_error(self, prefix): def test_get_max_sub_or_ses_num_and_value_length( self, prefix, test_max_num, test_num_digits ): - """ - Test many combinations of subject names + """Test many combinations of subject names and number of digits for a project, e.g. `names = ["sub-001", ... "sub-101"]`. """ @@ -235,9 +227,7 @@ def test_get_max_sub_or_ses_num_and_value_length( @pytest.mark.parametrize("prefix", ["sub", "ses"]) def test_get_max_sub_or_ses_num_and_value_length_edge_case(self, prefix): - """ - Test the edge case where the subject number does not start at 1. - """ + """Test the edge case where the subject number does not start at 1.""" names = [f"{prefix}-09", f"{prefix}-10", f"{prefix}-11"] max_num, num_digits = getters.get_max_sub_or_ses_num_and_value_length( @@ -252,8 +242,7 @@ def test_get_max_sub_or_ses_num_and_value_length_edge_case(self, prefix): # ------------------------------------------------------------------------- def make_name(self, key, underscore_position, start, end): - """ - Make name with / without underscore to test every + """Make name with / without underscore to test every possibility. """ if underscore_position == "left": diff --git a/tests/tests_unit/test_validation_unit.py b/tests/tests_unit/test_validation_unit.py index 473a8a638..dfd404c3a 100644 --- a/tests/tests_unit/test_validation_unit.py +++ b/tests/tests_unit/test_validation_unit.py @@ -6,8 +6,7 @@ class TestValidationUnit: @pytest.mark.parametrize("prefix", ["sub", "ses"]) def test_more_than_one_instance(self, prefix): - """ - Check that any duplicate sub or ses values are caught + """Check that any duplicate sub or ses values are caught in `validate_list_of_names()`. """ error_message = validation.validate_list_of_names( @@ -36,8 +35,7 @@ def test_more_than_one_instance(self, prefix): ], ) def test_name_does_not_begin_with_prefix(self, prefix_and_names): - """ - Check validation that names passed to `validate_list_of_names()` + """Check validation that names passed to `validate_list_of_names()` start with the prefix prefix (sub or ses). """ prefix, names = prefix_and_names @@ -48,8 +46,7 @@ def test_name_does_not_begin_with_prefix(self, prefix_and_names): @pytest.mark.parametrize("prefix", ["sub", "ses"]) def test_special_characters_in_format_names(self, prefix): - """ - Check `validate_list_of_names()` catches + """Check `validate_list_of_names()` catches spaces in passed names (not all names are bad """ error_messages = validation.validate_list_of_names( @@ -88,11 +85,9 @@ def test_prefix_is_not_an_integer(self, prefix_and_names): def test_formatting_dashes_and_underscore_alternate_incorrectly( self, prefix ): - """ - Check `validate_list_of_names()` catches "-" and "_" that + """Check `validate_list_of_names()` catches "-" and "_" that are not in the correct order. """ - # Test a large range of bad names. Do not use # parametrize so we can use f"{prefix}". # There should always be two validation errors per list. @@ -144,8 +139,7 @@ def test_formatting_dashes_and_underscore_alternate_incorrectly( @pytest.mark.parametrize("prefix", ["sub", "ses"]) def test_inconsistent_value_lengths_in_list_of_names(self, prefix): - """ - Ensure a list of sub / ses names that contain inconsistent + """Ensure a list of sub / ses names that contain inconsistent leading zeros (e.g. ["sub-001", "sub-02"]) leads to an error. """ for names in [ @@ -163,8 +157,7 @@ def test_inconsistent_value_lengths_in_list_of_names(self, prefix): @pytest.mark.parametrize("prefix", ["sub", "ses"]) def test_duplicate_ids_in_list_of_names(self, prefix): - """ - Ensure a list of sub / ses names that contain duplicate sub / ses + """Ensure a list of sub / ses names that contain duplicate sub / ses ids (e.g. ["sub-001", "sub-001_@DATE@"]) leads to an error. """ names = [ @@ -183,12 +176,10 @@ def test_duplicate_ids_in_list_of_names(self, prefix): @pytest.mark.parametrize("prefix", ["sub", "ses"]) def test_new_name_duplicates_existing(self, prefix): - """ - Test the function `new_name_duplicates_existing()` + """Test the function `new_name_duplicates_existing()` that will throw an error if a sub / ses name matches an existing name (unless it matches exactly). """ - # Check an exactly matching case that should not raise and error new_name = f"{prefix}-002" existing_names = [f"{prefix}-001", f"{prefix}-002", f"{prefix}-003"] @@ -231,8 +222,7 @@ def test_new_name_duplicates_existing(self, prefix): ) def test_tags_autoreplace_in_regexp(self): - """ - Check the validation function `replace_tags_in_regexp()` + """Check the validation function `replace_tags_in_regexp()` correctly replaces tags in a regexp with their regexp equivalent. Test date, time and datetime with some random regexp that @@ -259,7 +249,6 @@ def test_tags_autoreplace_in_regexp(self): ) def test_handle_path(self): - output = validation.handle_path("message", None) assert output == "message" @@ -271,7 +260,6 @@ def test_handle_path(self): @pytest.mark.parametrize("prefix", ["sub", "ses"]) def test_datetime_iso_format(self, prefix): - # Test dates error_messages = validation.validate_list_of_names( [ From 462726873350f16bc07f027b4cfd5a4f093e79f1 Mon Sep 17 00:00:00 2001 From: MoffittAndrew Date: Fri, 21 Mar 2025 10:44:55 +0000 Subject: [PATCH 06/70] enabling ruff-format --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 299fba1d1..8fbbd4031 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -34,7 +34,7 @@ repos: rev: v0.9.9 hooks: - id: ruff - #- id: ruff-format + - id: ruff-format - repo: https://github.com/pre-commit/mirrors-mypy rev: v1.15.0 hooks: From 2947de1c6d258e53978e08d1ea7a7599525d49be Mon Sep 17 00:00:00 2001 From: MoffittAndrew Date: Fri, 21 Mar 2025 11:23:24 +0000 Subject: [PATCH 07/70] fixing pre-commit errors for test_configs and test_validation_unit --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8fbbd4031..299fba1d1 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -34,7 +34,7 @@ repos: rev: v0.9.9 hooks: - id: ruff - - id: ruff-format + #- id: ruff-format - repo: https://github.com/pre-commit/mirrors-mypy rev: v1.15.0 hooks: From 3d42f4a5a869c06b2bd0c70d3608d20847bcde6a Mon Sep 17 00:00:00 2001 From: MoffittAndrew Date: Fri, 21 Mar 2025 11:27:42 +0000 Subject: [PATCH 08/70] disabling pre-commit --- pyproject.toml | 1 + .../{_test_configs.py => test_configs.py} | 16 +++++++++------- tests/tests_unit/test_validation_unit.py | 8 ++++++-- 3 files changed, 16 insertions(+), 9 deletions(-) rename tests/tests_integration/{_test_configs.py => test_configs.py} (96%) diff --git a/pyproject.toml b/pyproject.toml index 77052bbf8..1e6aff893 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -100,6 +100,7 @@ fix = true ignore = [ "D203", # one blank line before class "D213", # multi-line-summary second line + "E501", # limit lines to 79 characters ] select = [ "E", # pycodestyle errors diff --git a/tests/tests_integration/_test_configs.py b/tests/tests_integration/test_configs.py similarity index 96% rename from tests/tests_integration/_test_configs.py rename to tests/tests_integration/test_configs.py index 3d617a072..32f59beec 100644 --- a/tests/tests_integration/_test_configs.py +++ b/tests/tests_integration/test_configs.py @@ -10,6 +10,8 @@ class TestConfigs(BaseTest): + """PLACEHOLDER.""" + # Test Errors # ------------------------------------------------------------- @@ -55,7 +57,7 @@ def test_warning_on_startup(self, no_cfg_project): ) @pytest.mark.parametrize("path_type", ["local_path", "central_path"]) def test_bad_path_syntax(self, project, bad_pattern, path_type, tmp_path): - """ "~", "." and "../" syntax is not supported because + """"~", "." and "../" syntax is not supported because it does not work with rclone. Theoretically it could be supported by checking for "." etc. and filling in manually, but it does not seem robust. @@ -90,7 +92,7 @@ def test_bad_path_syntax(self, project, bad_pattern, path_type, tmp_path): def test_no_ssh_options_set_on_make_config_file(self, no_cfg_project): """Check that program will assert if not all ssh options - are set on make_config_file + are set on make_config_file. """ with pytest.raises(ConfigError) as e: no_cfg_project.make_config_file( @@ -211,9 +213,9 @@ def test_existing_projects(self, monkeypatch, tmp_path): function is monkeypatched in order to point to a tmp_path. The tmp_path / "projects" is filled with a mix of project folders - with and without config, and tested against accordingly. The `local_path` - and `central_path` specified in the DataShuttle config are arbitrarily put in - `tmp_path`. + with and without config, and tested against accordingly. The + `local_path` and `central_path` specified in the DataShuttle config are + arbitrarily put in `tmp_path`. """ def patch_get_datashuttle_path(): @@ -249,9 +251,9 @@ def patch_get_datashuttle_path(): (tmp_path / "projects" / "project_3"), ] - # -------------------------------------------------------------------------------------------------------------------- + # ------------------------------------------------------------------------- # Utils - # -------------------------------------------------------------------------------------------------------------------- + # ------------------------------------------------------------------------- def check_config_reopen_and_check_config_again(self, project, *kwargs): """Check the config file and project.cfg against provided kwargs, diff --git a/tests/tests_unit/test_validation_unit.py b/tests/tests_unit/test_validation_unit.py index dfd404c3a..21c6437c0 100644 --- a/tests/tests_unit/test_validation_unit.py +++ b/tests/tests_unit/test_validation_unit.py @@ -4,6 +4,8 @@ class TestValidationUnit: + """PLACEHOLDER.""" + @pytest.mark.parametrize("prefix", ["sub", "ses"]) def test_more_than_one_instance(self, prefix): """Check that any duplicate sub or ses values are caught @@ -47,7 +49,7 @@ def test_name_does_not_begin_with_prefix(self, prefix_and_names): @pytest.mark.parametrize("prefix", ["sub", "ses"]) def test_special_characters_in_format_names(self, prefix): """Check `validate_list_of_names()` catches - spaces in passed names (not all names are bad + spaces in passed names (not all names are bad. """ error_messages = validation.validate_list_of_names( [ @@ -69,7 +71,7 @@ def test_special_characters_in_format_names(self, prefix): ], ) def test_prefix_is_not_an_integer(self, prefix_and_names): - """ """ + """PLACEHOLDER.""" prefix, names = prefix_and_names error_messages = validation.validate_list_of_names(names, prefix) @@ -249,6 +251,7 @@ def test_tags_autoreplace_in_regexp(self): ) def test_handle_path(self): + """PLACEHOLDER.""" output = validation.handle_path("message", None) assert output == "message" @@ -260,6 +263,7 @@ def test_handle_path(self): @pytest.mark.parametrize("prefix", ["sub", "ses"]) def test_datetime_iso_format(self, prefix): + """PLACEHOLDER.""" # Test dates error_messages = validation.validate_list_of_names( [ From ed41aaab9a7146d0987be5ea22732e0b4d5ec13b Mon Sep 17 00:00:00 2001 From: MoffittAndrew Date: Fri, 21 Mar 2025 12:04:50 +0000 Subject: [PATCH 09/70] Editing all test files to comply with linting rules --- pyproject.toml | 53 ++++++++++--------- tests/quick_make_project.py | 4 +- tests/ssh_test_utils.py | 8 +-- tests/test_utils.py | 24 ++++----- tests/tests_integration/base.py | 2 + .../tests_integration/test_create_folders.py | 8 ++- tests/tests_integration/test_datatypes.py | 2 +- .../test_filesystem_transfer.py | 12 +++-- tests/tests_integration/test_formatting.py | 3 ++ .../tests_integration/test_local_only_mode.py | 4 +- tests/tests_integration/test_logging.py | 12 +++-- tests/tests_integration/test_settings.py | 2 + .../test_ssh_file_transfer.py | 9 ++-- tests/tests_integration/test_ssh_setup.py | 10 ++-- .../tests_integration/test_transfer_checks.py | 3 ++ tests/tests_integration/test_validation.py | 24 ++++----- tests/tests_tui/test_local_only_project.py | 2 + tests/tests_tui/test_tui_configs.py | 3 ++ tests/tests_tui/test_tui_create_folders.py | 4 ++ tests/tests_tui/test_tui_datatypes.py | 2 + tests/tests_tui/test_tui_directorytree.py | 3 +- tests/tests_tui/test_tui_get_help.py | 1 + tests/tests_tui/test_tui_logging.py | 2 + tests/tests_tui/test_tui_transfer.py | 12 +++-- .../test_tui_widgets_and_defaults.py | 14 ++--- tests/tests_tui/tui_base.py | 11 ++-- tests/tests_unit/test_unit.py | 7 +-- 27 files changed, 148 insertions(+), 93 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 1e6aff893..b7f6e1278 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -97,21 +97,22 @@ fix = true [tool.ruff.lint] # See https://docs.astral.sh/ruff/rules/ -ignore = [ - "D203", # one blank line before class - "D213", # multi-line-summary second line - "E501", # limit lines to 79 characters -] -select = [ - "E", # pycodestyle errors - "F", # Pyflakes - "UP", # pyupgrade - "I", # isort - "B", # flake8 bugbear - "SIM", # flake8 simplify - "C90", # McCabe complexity - "D", # pydocstyle -] +#ignore = [ +# "D203", # one blank line before class +# "D213", # multi-line-summary second line +# "D401", # first line of docstrings should be in an imperative mood +# "E501", # limit lines to 79 characters +#] +#select = [ +# "E", # pycodestyle errors +# "F", # Pyflakes +# "UP", # pyupgrade +# "I", # isort +# "B", # flake8 bugbear +# "SIM", # flake8 simplify +# "C90", # McCabe complexity +# "D", # pydocstyle +#] per-file-ignores = { "tests/*" = [ "D100", # missing docstring in public module "D205", # missing blank line between summary and description @@ -129,15 +130,19 @@ per-file-ignores = { "tests/*" = [ # Old ruff ruleset + pydocstyle added # Inconsistent with movement repo, but saving this here for # now in case there are good reasons to keep these rules -#ignore = ["E203","E501","E731","C901","W291","W293","E402","E722"] -#select = [ -# "I", # isort -# "E", # pycodestyle errors -# "F", # Pyflakes -# "TC", # flake8-type-checking -# "TID252", # flake8-tidy-imports relative-imports -# "D", # pydocstyle -#] +ignore = ["E203","E501","E731","C901","W291","W293","E402","E722", + "D203", # one blank line before class + "D213", # multi-line-summary second line + "D401", # first line of docstrings should be in an imperative mood +] +select = [ + "I", # isort + "E", # pycodestyle errors + "F", # Pyflakes + "TC", # flake8-type-checking + "TID252", # flake8-tidy-imports relative-imports + "D", # pydocstyle +] [tool.ruff.format] docstring-code-format = true # Also format code in docstrings diff --git a/tests/quick_make_project.py b/tests/quick_make_project.py index 250e66507..cbb837a32 100644 --- a/tests/quick_make_project.py +++ b/tests/quick_make_project.py @@ -1,5 +1,5 @@ -base_path = r"C:/Users/Joe/work/git-repos/forks/yxtuix/joe" - from test_utils import quick_create_project +base_path = r"C:/Users/Joe/work/git-repos/forks/yxtuix/joe" + quick_create_project(base_path) diff --git a/tests/ssh_test_utils.py b/tests/ssh_test_utils.py index bf5bf2c74..a7af1a65c 100644 --- a/tests/ssh_test_utils.py +++ b/tests/ssh_test_utils.py @@ -8,7 +8,7 @@ def setup_project_for_ssh( project, central_path, central_host_id, central_host_username ): """Set up the project configs to use SSH connection - to central + to central. """ project.update_config_file( central_path=central_path, @@ -25,10 +25,10 @@ def setup_project_for_ssh( def setup_mock_input(input_): - """This is very similar to pytest monkeypatch but + """Very similar to pytest monkeypatch but using that was giving me very strange output, monkeypatch.setattr('builtins.input', lambda _: "n") - i.e. pdb went deep into some unrelated code stack + i.e. pdb went deep into some unrelated code stack. """ orig_builtin = copy.deepcopy(builtins.input) builtins.input = lambda _: input_ # type: ignore @@ -36,7 +36,7 @@ def setup_mock_input(input_): def restore_mock_input(orig_builtin): - """orig_builtin: the copied, original builtins.input""" + """orig_builtin: the copied, original builtins.input.""" builtins.input = orig_builtin diff --git a/tests/test_utils.py b/tests/test_utils.py index 46d0b23ea..fce9b15d5 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -28,8 +28,7 @@ def setup_project_default_configs( central_path=False, ): """Set up a fresh project to test on - - local_path / central_path: provide the config paths to set + local_path / central_path: provide the config paths to set. """ delete_project_if_it_exists(project_name) @@ -90,7 +89,7 @@ def glob_basenames(search_path, recursive=False, exclude=None): def teardown_project( cwd, project ): # 99% sure these are unnecessary with pytest tmp_path but keep until SSH testing. - """""" + """PLACEHOLDER.""" os.chdir(cwd) delete_all_folders_in_project_path(project, "central") delete_all_folders_in_project_path(project, "local") @@ -104,7 +103,7 @@ def delete_all_folders_in_local_path(project): def delete_all_folders_in_project_path(project, local_or_central): - """""" + """PLACEHOLDER.""" folder = f"{local_or_central}_path" if folder == "central_path" and project.cfg[folder] is None: @@ -119,7 +118,7 @@ def delete_all_folders_in_project_path(project, local_or_central): def delete_project_if_it_exists(project_name): - """""" + """PLACEHOLDER.""" config_path, _ = canonical_folders.get_project_datashuttle_path( project_name ) @@ -158,7 +157,7 @@ def make_test_path(base_path, local_or_central, test_project_name): def create_all_pathtable_files(pathtable): - """ """ + """PLACEHOLDER.""" for i in range(pathtable.shape[0]): filepath = pathtable["base_folder"][i] / pathtable["path"][i] filepath.parents[0].mkdir(parents=True, exist_ok=True) @@ -276,7 +275,7 @@ def check_folder_tree_is_correct( key, folder, ) in canonical_folders.get_datatype_folders().items(): - assert key in folder_used.keys(), ( + assert key in folder_used, ( "Key not found in folder_used. " "Update folder used and hard-coded tests: " "test_custom_folder_names(), test_explicitly_session_list()" @@ -398,7 +397,7 @@ def make_local_folders_with_files_in( def check_configs(project, kwargs, config_path=None): - """""" + """PLACEHOLDER.""" if config_path is None: config_path = project._config_path @@ -433,7 +432,7 @@ def check_project_configs( def check_config_file(config_path, *kwargs): - """""" + """PLACEHOLDER.""" with open(config_path) as config_file: config_yaml = yaml.full_load(config_file) @@ -449,8 +448,7 @@ def check_config_file(config_path, *kwargs): def get_top_level_folder_path( project, local_or_central="local", folder_name="rawdata" ): - """""" - + """PLACEHOLDER.""" assert folder_name in canonical_folders.get_top_level_folders(), ( "folder_name must be canonical e.g. rawdata" ) @@ -496,7 +494,7 @@ def handle_upload_or_download( def get_transfer_func( project, upload_or_download, transfer_method, top_level_folder=None ): - """""" + """PLACEHOLDER.""" if transfer_method == "top_level_folder": assert top_level_folder is not None, "must pass top-level-folder" assert top_level_folder in [None, "rawdata", "derivatives"] @@ -572,7 +570,7 @@ def swap_local_and_central_paths(project, swap_last_folder_only=False): def get_default_sub_sessions_to_test(): - """Canonical subs / sessions for these tests""" + """Canonical subs / sessions for these tests.""" subs = ["sub-001", "sub-002", "sub-003"] sessions = ["ses-001_datetime-20220516T135022", "ses-002", "ses-003"] return subs, sessions diff --git a/tests/tests_integration/base.py b/tests/tests_integration/base.py index c3ea76d20..75c01adf2 100644 --- a/tests/tests_integration/base.py +++ b/tests/tests_integration/base.py @@ -10,6 +10,8 @@ class BaseTest: + """PLACEHOLDER.""" + @pytest.fixture(scope="function") def no_cfg_project(test): """Fixture that creates an empty project. Ignore the warning diff --git a/tests/tests_integration/test_create_folders.py b/tests/tests_integration/test_create_folders.py index 0a69b0695..af9f7e6d2 100644 --- a/tests/tests_integration/test_create_folders.py +++ b/tests/tests_integration/test_create_folders.py @@ -13,6 +13,8 @@ class TestCreateFolders(BaseTest): + """PLACEHOLDER.""" + @pytest.mark.parametrize("project", ["local", "full"], indirect=True) def test_generate_folders_default_ses(self, project): """Make a subject folders with full tree. Don't specify @@ -202,7 +204,7 @@ def test_datatypes_subsection(self, project, files_to_test): @pytest.mark.parametrize("project", ["local", "full"], indirect=True) def test_date_flags_in_session(self, project): """Check that @DATE@ is converted into current date - in generated folder names + in generated folder names. """ date, time_ = self.get_formatted_date_and_time() @@ -224,7 +226,7 @@ def test_date_flags_in_session(self, project): @pytest.mark.parametrize("project", ["local", "full"], indirect=True) def test_datetime_flag_in_session(self, project): """Check that @DATETIME@ is converted to datetime - in generated folder names + in generated folder names. """ date, time_ = self.get_formatted_date_and_time() @@ -483,10 +485,12 @@ def test_get_next_sub_and_ses_name_template(self, project): # ---------------------------------------------------------------------------------- def get_formatted_date_and_time(self): + """PLACEHOLDER.""" date = str(datetime.datetime.now().date()) date = date.replace("-", "") time_ = datetime.datetime.now().time().strftime("%Hh%Mm") return date, time_ def broad_datatypes(self): + """PLACEHOLDER.""" return canonical_configs.get_broad_datatypes() diff --git a/tests/tests_integration/test_datatypes.py b/tests/tests_integration/test_datatypes.py index 10720e90b..ff08aa161 100644 --- a/tests/tests_integration/test_datatypes.py +++ b/tests/tests_integration/test_datatypes.py @@ -39,7 +39,7 @@ def test_create_narrow_datatypes(self, project): ) def get_narrow_only_datatypes_used(self, used=True): - """This is similar to test_utils.get_all_broad_folders_used + """Similar to test_utils.get_all_broad_folders_used but for narrow datatypes. """ return { diff --git a/tests/tests_integration/test_filesystem_transfer.py b/tests/tests_integration/test_filesystem_transfer.py index 98e31a629..d35b0e84e 100644 --- a/tests/tests_integration/test_filesystem_transfer.py +++ b/tests/tests_integration/test_filesystem_transfer.py @@ -13,6 +13,8 @@ class TestFileTransfer(BaseTest): + """PLACEHOLDER.""" + @pytest.mark.parametrize( "top_level_folder", canonical_folders.get_top_level_folders() ) @@ -57,6 +59,7 @@ def test_transfer_empty_folder_structure( ) def test_empty_folder_is_not_transferred(self, project): + """PLACEHOLDER.""" project.create_folders("rawdata", "sub-001") project.upload_rawdata() assert not ( @@ -127,7 +130,7 @@ def test_transfer_across_top_level_folders( @pytest.mark.parametrize("upload_or_download", ["upload", "download"]) def test_transfer_all_top_level_folders(self, project, upload_or_download): - """ """ + """PLACEHOLDER.""" subs, sessions = test_utils.get_default_sub_sessions_to_test() for top_level_folder in canonical_folders.get_top_level_folders(): @@ -529,7 +532,7 @@ def test_overwrite_same_size_later_to_earlier( top_level_folder, upload_or_download, ): - """This functions is extremely similar to + """Extremely similar to `test_overwrite_same_size_later_to_earlier()` but it is much easier to understand individually when they are split. @@ -609,6 +612,7 @@ def test_overwrite_different_size_different_times( assert test_utils.read_file(central_file_path) == ["file earlier"] def get_paths_to_a_local_and_central_file(self, project, top_level_folder): + """PLACEHOLDER.""" path_to_test_file = ( Path(top_level_folder) / "sub-001" @@ -625,7 +629,7 @@ def get_paths_to_a_local_and_central_file(self, project, top_level_folder): def setup_overwrite_file_tests( self, upload_or_download, top_level_folder, project ): - """""" + """PLACEHOLDER.""" local_file_path, central_file_path = ( self.get_paths_to_a_local_and_central_file( project, top_level_folder @@ -733,7 +737,7 @@ def test_specific_file_or_folder( assert transferred_files == to_test_against def setup_specific_file_or_folder_files(self, project, top_level_folder): - """ """ + """PLACEHOLDER.""" project.create_folders( top_level_folder, ["sub-001", "sub-002"], diff --git a/tests/tests_integration/test_formatting.py b/tests/tests_integration/test_formatting.py index 0b5db8338..005b7cbf7 100644 --- a/tests/tests_integration/test_formatting.py +++ b/tests/tests_integration/test_formatting.py @@ -6,6 +6,8 @@ class TestFormatting(BaseTest): + """PLACEHOLDER.""" + @pytest.mark.parametrize("prefix", ["sub", "ses"]) @pytest.mark.parametrize( "input", [1, {"test": "one"}, 1.0, ["1", "2", ["three"]]] @@ -63,6 +65,7 @@ def test_format_names_prefix(self): @pytest.mark.parametrize("top_level_folder", ["rawdata", "derivatives"]) @pytest.mark.parametrize("project", ["local", "full"], indirect=True) def test_warning_non_consecutive_numbers(self, project, top_level_folder): + """PLACEHOLDER.""" project.create_folders( top_level_folder, ["sub-01", "sub-02", "sub-04"], diff --git a/tests/tests_integration/test_local_only_mode.py b/tests/tests_integration/test_local_only_mode.py index 0db07a503..bcc42394b 100644 --- a/tests/tests_integration/test_local_only_mode.py +++ b/tests/tests_integration/test_local_only_mode.py @@ -13,9 +13,11 @@ class TestLocalOnlyProject(BaseTest): + """PLACEHOLDER.""" + def test_bad_setup(self, tmp_path): """Test setup without providing both central_path and connection - method (distinguishing a full vs local-only project) + method (distinguishing a full vs local-only project). """ local_path = tmp_path / "test_local" diff --git a/tests/tests_integration/test_logging.py b/tests/tests_integration/test_logging.py index fa193c5b7..ecf68b49e 100644 --- a/tests/tests_integration/test_logging.py +++ b/tests/tests_integration/test_logging.py @@ -18,6 +18,8 @@ class TestLogging: + """PLACEHOLDER.""" + @pytest.fixture(scope="function") def teardown_logger(self): """Ensure the logger is deleted at the end of each test.""" @@ -126,7 +128,7 @@ def project(self, tmp_path, clean_project_name, request): @pytest.mark.parametrize("project", ["local", "full"], indirect=True) def test_log_filename(self, project): """Check the log filename is formatted correctly, for - `update_config_file`, an arbitrary command + `update_config_file`, an arbitrary command. """ project.update_config_file(central_host_id="test_id") @@ -140,7 +142,7 @@ def test_log_filename(self, project): assert re.search(regex, log_filename) is not None def test_logs_make_config_file(self, clean_project_name, tmp_path): - """""" + """PLACEHOLDER.""" project = DataShuttle(clean_project_name) project.make_config_file( @@ -161,6 +163,7 @@ def test_logs_make_config_file(self, clean_project_name, tmp_path): assert "Update successful. New config file:" in log def test_logs_update_config_file(self, project): + """PLACEHOLDER.""" project.update_config_file(central_host_id="test_id") log = test_utils.read_log_file(project.cfg.logging_path) @@ -175,6 +178,7 @@ def test_logs_update_config_file(self, project): @pytest.mark.parametrize("project", ["local", "full"], indirect=True) def test_create_folders(self, project): + """PLACEHOLDER.""" subs = ["sub-111", f"sub-002{tags('to')}004"] ses = ["ses-123", "ses-101"] @@ -402,7 +406,7 @@ def test_clear_logging_path(self, clean_project_name, tmp_path): # ---------------------------------------------------------------------------------- def test_logs_check_update_config_error(self, project): - """""" + """PLACEHOLDER.""" with pytest.raises(ConfigError): project.update_config_file( connection_method="ssh", central_host_username=None @@ -421,7 +425,7 @@ def test_logs_check_update_config_error(self, project): @pytest.mark.parametrize("project", ["local", "full"], indirect=True) def test_logs_bad_create_folders_error(self, project): - """""" + """PLACEHOLDER.""" project.create_folders("rawdata", "sub-001", datatype="all") test_utils.delete_log_files(project.cfg.logging_path) diff --git a/tests/tests_integration/test_settings.py b/tests/tests_integration/test_settings.py index 5d5834c92..27f9eb7b6 100644 --- a/tests/tests_integration/test_settings.py +++ b/tests/tests_integration/test_settings.py @@ -11,6 +11,8 @@ class TestPersistentSettings(BaseTest): + """PLACEHOLDER.""" + @pytest.mark.parametrize("project", ["local", "full"], indirect=True) def test_persistent_settings_name_templates(self, project): """Test the 'name_templates' option that is stored in persistent diff --git a/tests/tests_integration/test_ssh_file_transfer.py b/tests/tests_integration/test_ssh_file_transfer.py index 4b4babce4..0f6f46bcd 100644 --- a/tests/tests_integration/test_ssh_file_transfer.py +++ b/tests/tests_integration/test_ssh_file_transfer.py @@ -1,5 +1,3 @@ -""" """ - import copy import glob import shutil @@ -15,6 +13,8 @@ class TestFileTransfer: + """PLACEHOLDER.""" + @pytest.fixture( scope="class", params=[ # Set running SSH or local filesystem (see docstring). @@ -115,6 +115,7 @@ def pathtable_and_project(self, request, tmpdir_factory): # ------------------------------------------------------------------------- def central_from_local(self, path_): + """PLACEHOLDER.""" return Path(str(copy.copy(path_)).replace("local", "central")) # ------------------------------------------------------------------------- @@ -168,7 +169,7 @@ def test_all_data_transfer_options( """Parse the arguments to filter the pathtable, getting the files expected to be transferred passed on the arguments Note files in sub/ses/datatype folders must be handled - separately to those in non-sub, non-ses, non-datatype folders + separately to those in non-sub, non-ses, non-datatype folders. see test_utils.swap_local_and_central_paths() for the logic on setting up and swapping local / central paths for @@ -245,7 +246,7 @@ def test_all_data_transfer_options( def query_table(self, pathtable, arguments): """Search the table for arguments, return empty - if arguments empty + if arguments empty. """ if any(arguments): folders = pathtable.query(" | ".join(arguments)) diff --git a/tests/tests_integration/test_ssh_setup.py b/tests/tests_integration/test_ssh_setup.py index f26b1d921..5a511abc2 100644 --- a/tests/tests_integration/test_ssh_setup.py +++ b/tests/tests_integration/test_ssh_setup.py @@ -1,6 +1,6 @@ """SSH configs are set in conftest.py . The password should be stored in a file called test_ssh_password.txt located -in the same folder as test_ssh.py +in the same folder as test_ssh.py. """ import pytest @@ -13,10 +13,12 @@ @pytest.mark.skipif(ssh_config.TEST_SSH is False, reason="TEST_SSH is false") class TestSSH: + """PLACEHOLDER.""" + @pytest.fixture(scope="function") def project(test, tmp_path): """Make a project as per usual, but now add - in test ssh configurations + in test ssh configurations. """ tmp_path = tmp_path / "test with space" @@ -45,7 +47,7 @@ def test_verify_ssh_central_host_do_not_accept( ): """Use the main function to test this. Test the sub-function when accepting, because this main function will also - call setup ssh key pairs which we don't want to do yet + call setup ssh key pairs which we don't want to do yet. This should only accept for "y" so try some random strings including "n" and check they all do not make the connection. @@ -84,7 +86,7 @@ def test_verify_ssh_central_host_accept(self, capsys, project): def test_generate_and_write_ssh_key(self, project): """Check ssh key for passwordless connection is written - to file + to file. """ path_to_save = project.cfg["local_path"] / "test" ssh.generate_and_write_ssh_key(path_to_save) diff --git a/tests/tests_integration/test_transfer_checks.py b/tests/tests_integration/test_transfer_checks.py index 7a6f631f3..e443cfae8 100644 --- a/tests/tests_integration/test_transfer_checks.py +++ b/tests/tests_integration/test_transfer_checks.py @@ -10,6 +10,8 @@ class TestTransferChecks(BaseTest): + """PLACEHOLDER.""" + @pytest.mark.parametrize( "top_level_folders", [["rawdata", "derivatives"], ["rawdata"], ["derivatives"]], @@ -90,6 +92,7 @@ def test_rclone_check(self, project, top_level_folders): assert path_ not in results_paths def get_folder_structure(self, top_level_folder): + """PLACEHOLDER.""" # fmt: off folder_structure = [ [f"{top_level_folder}/sub-001/ses-001/ephys/local_only_1.txt", "local_only"], diff --git a/tests/tests_integration/test_validation.py b/tests/tests_integration/test_validation.py index 71fd7d658..4827152df 100644 --- a/tests/tests_integration/test_validation.py +++ b/tests/tests_integration/test_validation.py @@ -14,6 +14,8 @@ class TestValidation(BaseTest): + """PLACEHOLDER.""" + @pytest.mark.parametrize( "sub_name", ["sub-001", "sub-999_@DATE@", "sub-001_random-tag_another-tag"], @@ -33,7 +35,7 @@ class TestValidation(BaseTest): def test_warn_on_inconsistent_sub_value_lengths( self, project, sub_name, bad_sub_name ): - """This test checks that inconsistent sub value lengths are properly + """Checks that inconsistent sub value lengths are properly detected across the project. This is performed with an assortment of possible filenames and leading zero conflicts. @@ -92,7 +94,7 @@ def test_warn_on_inconsistent_sub_value_lengths( def test_warn_on_inconsistent_ses_value_lengths( self, project, ses_name, bad_ses_name ): - """This function is exactly the same as + """Exactly the same as `test_warn_on_inconsistent_sub_value_lengths()` but operates at the session level. This is extreme code duplication, but factoring the main logic out got very messy and hard to follow. @@ -155,7 +157,7 @@ def test_warn_on_inconsistent_sub_and_ses_value_lengths(self, project): def check_inconsistent_sub_or_ses_value_length_warning( self, project, warn_idx=0, include_central=True ): - """""" + """PLACEHOLDER.""" with pytest.warns(UserWarning) as w: project.validate_project( "rawdata", display_mode="warn", include_central=include_central @@ -285,7 +287,7 @@ def test_duplicate_ses_across_subjects(self, project): @pytest.mark.parametrize("project", ["local", "full"], indirect=True) def test_invalid_sub_and_ses_name(self, project): - """This is a slightly weird case, the name is successfully + """Slightly weird case, the name is successfully prefixed as 'sub-sub_100` but when the value if `sub-` is extracted, it is also "sub" and so an error is raised. """ @@ -380,7 +382,7 @@ def test_validate_project(self, project): @pytest.mark.parametrize("prefix", ["sub", "ses"]) def test_validate_project_returned_list(self, project, prefix): - """ """ + """PLACEHOLDER.""" bad_names = [ f"{prefix}-001", f"{prefix}-001_@DATE@", @@ -409,7 +411,7 @@ def test_validate_project_returned_list(self, project, prefix): assert "VALUE_LENGTH" in concat_error def test_output_paths_are_valid(self, project): - """ """ + """PLACEHOLDER.""" sub_name = "sub-001x" ses_name = "ses-001x" project.create_folders( @@ -747,7 +749,7 @@ def test_tags_in_name_templates_pass_validation(self, project): assert "TEMPLATE: The name: ses-001_datex-20241212" in str(e.value) def test_name_templates_validate_project(self, project): - """TODO:""" + """TODO.""" name_templates = { "on": True, "sub": "sub-\d\d_id-\d.?", @@ -782,7 +784,7 @@ def test_name_templates_validate_project(self, project): # ---------------------------------------------------------------------------------- def test_quick_validation(self, mocker, project): - """ """ + """PLACEHOLDER.""" project.create_folders("rawdata", "sub-1") os.makedirs(project.cfg["local_path"] / "rawdata" / "sub-02") project.create_folders("derivatives", "sub-1") @@ -839,7 +841,7 @@ def test_quick_validation_top_level_folder(self, project): @pytest.mark.parametrize("top_level_folder", ["rawdata", "derivatives"]) def test_strict_mode_validation(self, project, top_level_folder): - """ """ + """PLACEHOLDER.""" project.create_folders( top_level_folder, ["sub-001", "sub-002"], @@ -908,9 +910,7 @@ def test_strict_mode_validation(self, project, top_level_folder): def test_check_high_level_project_structure( self, project, top_level_folder ): - """Check that local and central project names are properly formatted - and that - """ + """Check that local and central project names are properly formatted.""" with pytest.warns(UserWarning) as w: project.validate_project( top_level_folder, "warn", include_central=True diff --git a/tests/tests_tui/test_local_only_project.py b/tests/tests_tui/test_local_only_project.py index 8a06ccfa7..8802aee8c 100644 --- a/tests/tests_tui/test_local_only_project.py +++ b/tests/tests_tui/test_local_only_project.py @@ -5,6 +5,8 @@ class TestTuiLocalOnlyProject(TuiBase): + """PLACEHOLDER.""" + @pytest.mark.asyncio async def test_local_only_make_project( self, diff --git a/tests/tests_tui/test_tui_configs.py b/tests/tests_tui/test_tui_configs.py index a9f4d8fe0..1fa7802ad 100644 --- a/tests/tests_tui/test_tui_configs.py +++ b/tests/tests_tui/test_tui_configs.py @@ -14,6 +14,8 @@ class TestTuiConfigs(TuiBase): + """PLACEHOLDER.""" + # ------------------------------------------------------------------------- # Test New Project Configs # ------------------------------------------------------------------------- @@ -337,6 +339,7 @@ async def test_configs_select_path(self, monkeypatch): @pytest.mark.asyncio async def test_bad_configs_screen_input(self, empty_project_paths): + """PLACEHOLDER.""" app = TuiApp() async with app.run_test(size=self.tui_size()) as pilot: # Select a new project, check NewProjectScreen is displayed correctly. diff --git a/tests/tests_tui/test_tui_create_folders.py b/tests/tests_tui/test_tui_create_folders.py index 8fbc32a2a..736f4c63d 100644 --- a/tests/tests_tui/test_tui_create_folders.py +++ b/tests/tests_tui/test_tui_create_folders.py @@ -13,6 +13,8 @@ class TestTuiCreateFolders(TuiBase): + """PLACEHOLDER.""" + # ------------------------------------------------------------------------- # General test Create Folders # ------------------------------------------------------------------------- @@ -600,6 +602,7 @@ async def test_create_folders_settings_top_level_folder( async def iterate_and_check_all_datatype_folders( self, pilot, subs, sessions ): + """PLACEHOLDER.""" project = pilot.app.screen.interface.project folder_used = test_utils.get_all_broad_folders_used(value=False) @@ -617,6 +620,7 @@ async def iterate_and_check_all_datatype_folders( async def create_folders_and_check_output( self, pilot, project, subs, sessions, folder_used ): + """PLACEHOLDER.""" await self.scroll_to_click_pause( pilot, "#create_folders_create_folders_button", diff --git a/tests/tests_tui/test_tui_datatypes.py b/tests/tests_tui/test_tui_datatypes.py index 48bcbf27c..c3e34bd01 100644 --- a/tests/tests_tui/test_tui_datatypes.py +++ b/tests/tests_tui/test_tui_datatypes.py @@ -13,6 +13,7 @@ class TestDatatypesTUI(TuiBase): async def test_select_displayed_datatypes_create( self, setup_project_paths ): + """PLACEHOLDER.""" tmp_config_path, tmp_path, project_name = setup_project_paths.values() app = TuiApp() @@ -127,6 +128,7 @@ async def test_select_displayed_datatypes_create( async def test_select_displayed_datatypes_transfer( self, setup_project_paths, mocker ): + """PLACEHOLDER.""" tmp_config_path, tmp_path, project_name = setup_project_paths.values() app = TuiApp() diff --git a/tests/tests_tui/test_tui_directorytree.py b/tests/tests_tui/test_tui_directorytree.py index 11562e085..4d6e6d872 100644 --- a/tests/tests_tui/test_tui_directorytree.py +++ b/tests/tests_tui/test_tui_directorytree.py @@ -15,7 +15,7 @@ class TestTuiCreateDirectoryTree(TuiBase): """Test the `Create` tab directory tree. - `Transfer` + `Transfer`. """ @pytest.mark.asyncio @@ -151,6 +151,7 @@ async def test_create_folders_directorytree_clipboard( async def test_failed_pyperclip_copy( self, setup_project_paths, monkeypatch ): + """PLACEHOLDER.""" tmp_config_path, tmp_path, project_name = setup_project_paths.values() app = TuiApp() diff --git a/tests/tests_tui/test_tui_get_help.py b/tests/tests_tui/test_tui_get_help.py index aa8986b8c..192487557 100644 --- a/tests/tests_tui/test_tui_get_help.py +++ b/tests/tests_tui/test_tui_get_help.py @@ -11,6 +11,7 @@ class TestTuiSettings(TuiBase): @pytest.mark.asyncio async def test_get_help(self, empty_project_paths): + """PLACEHOLDER.""" app = TuiApp() async with app.run_test(size=self.tui_size()) as pilot: await self.scroll_to_click_pause( diff --git a/tests/tests_tui/test_tui_logging.py b/tests/tests_tui/test_tui_logging.py index 6a046c3e8..21e288a12 100644 --- a/tests/tests_tui/test_tui_logging.py +++ b/tests/tests_tui/test_tui_logging.py @@ -7,6 +7,8 @@ class TestTuiLogging(TuiBase): + """PLACEHOLDER.""" + @pytest.mark.asyncio async def test_logging(self, setup_project_paths): """Test logging by running some commands, checking they diff --git a/tests/tests_tui/test_tui_transfer.py b/tests/tests_tui/test_tui_transfer.py index 1fd3fe60a..c273884b9 100644 --- a/tests/tests_tui/test_tui_transfer.py +++ b/tests/tests_tui/test_tui_transfer.py @@ -17,6 +17,7 @@ class TestTuiTransfer(TuiBase): async def test_transfer_entire_project( self, setup_project_paths, upload_or_download ): + """PLACEHOLDER.""" tmp_config_path, tmp_path, project_name = setup_project_paths.values() subs, sessions = test_utils.get_default_sub_sessions_to_test() @@ -67,7 +68,7 @@ async def check_persistent_settings(self, pilot): ) async def set_overwrite_checkbox(self, pilot, overwrite_setting): - """""" + """PLACEHOLDER.""" all_positions = {"never": None, "always": 5, "if_source_newer": 6} position = all_positions[overwrite_setting] @@ -77,6 +78,7 @@ async def set_overwrite_checkbox(self, pilot, overwrite_setting): ) async def set_transfer_tab_dry_run_checkbox(self, pilot, dry_run_setting): + """PLACEHOLDER.""" if ( pilot.app.screen.query_one("#transfer_tab_dry_run_checkbox") is not dry_run_setting @@ -107,7 +109,7 @@ async def set_and_check_persistent_settings( async def test_transfer_top_level_folder( self, setup_project_paths, top_level_folder, upload_or_download ): - """""" + """PLACEHOLDER.""" tmp_config_path, tmp_path, project_name = setup_project_paths.values() subs, sessions = test_utils.get_default_sub_sessions_to_test() @@ -153,6 +155,7 @@ async def test_transfer_top_level_folder( async def test_transfer_custom( self, setup_project_paths, top_level_folder, upload_or_download ): + """PLACEHOLDER.""" tmp_config_path, tmp_path, project_name = setup_project_paths.values() subs, sessions = test_utils.get_default_sub_sessions_to_test() @@ -222,6 +225,7 @@ async def test_transfer_custom( async def switch_top_level_folder_select( self, pilot, id, top_level_folder ): + """PLACEHOLDER.""" if top_level_folder == "rawdata": assert pilot.app.screen.query_one(id).value == "rawdata" else: @@ -229,7 +233,7 @@ async def switch_top_level_folder_select( assert pilot.app.screen.query_one(id).value == "derivatives" async def run_transfer(self, pilot, upload_or_download): - """""" + """PLACEHOLDER.""" # Check assumed default is correct on the transfer switch assert pilot.app.screen.query_one("#transfer_switch").value is False @@ -246,7 +250,7 @@ def setup_project_for_data_transfer( top_level_folder_list, upload_or_download, ): - """""" + """PLACEHOLDER.""" for top_level_folder in top_level_folder_list: test_utils.make_and_check_local_project_folders( project, diff --git a/tests/tests_tui/test_tui_widgets_and_defaults.py b/tests/tests_tui/test_tui_widgets_and_defaults.py index 897d066ce..781e7473c 100644 --- a/tests/tests_tui/test_tui_widgets_and_defaults.py +++ b/tests/tests_tui/test_tui_widgets_and_defaults.py @@ -14,7 +14,7 @@ class TestTuiWidgets(TuiBase): - """This class performs fundamental checks on the default display + """Performs fundamental checks on the default display of widgets and that changing widgets properly change underlying configs. This does not perform any functional tests e.g. creation of configs of new files. @@ -208,7 +208,7 @@ async def test_new_project_configs(self, empty_project_paths): async def check_new_project_ssh_widgets( self, configs_content, ssh_on, save_pressed=False ): - """""" + """PLACEHOLDER.""" assert configs_content.query_one( "#configs_setup_ssh_connection_button" ).visible is ( @@ -873,7 +873,7 @@ async def check_top_folder_select( expected_val, move_to_position: Union[bool, int] = False, ): - """If move to position is not False, must be int specifying position""" + """If move to position is not False, must be int specifying position.""" if move_to_position: await self.move_select_to_position(pilot, id, move_to_position) @@ -992,7 +992,7 @@ async def test_all_checkboxes(self, setup_project_paths): await pilot.pause() def check_datatype_checkboxes(self, pilot, tab, expected_on): - """""" + """PLACEHOLDER.""" assert tab in ["create", "transfer"] if tab == "create": id = "#create_folders_datatype_checkboxes" @@ -1016,6 +1016,7 @@ def check_datatype_checkboxes(self, pilot, tab, expected_on): @pytest.mark.asyncio async def test_all_transfer_widgets(self, setup_project_paths): + """PLACEHOLDER.""" tmp_config_path, tmp_path, project_name = setup_project_paths.values() app = TuiApp() @@ -1201,7 +1202,7 @@ async def test_all_transfer_widgets(self, setup_project_paths): @pytest.mark.asyncio async def test_overwrite_existing_files(self, setup_project_paths): - """ """ + """PLACEHOLDER.""" tmp_config_path, tmp_path, project_name = setup_project_paths.values() app = TuiApp() @@ -1293,6 +1294,7 @@ async def test_dry_run(self, setup_project_paths): # combine. def check_dry_run(self, pilot, project_name, value): + """PLACEHOLDER.""" assert ( pilot.app.screen.query_one("#transfer_tab_dry_run_checkbox").value == value @@ -1307,7 +1309,7 @@ def check_dry_run(self, pilot, project_name, value): def check_overwrite_existing_files_configs( self, pilot, project_name, value ): - """""" + """PLACEHOLDER.""" assert ( pilot.app.screen.query_one("#transfer_tab_overwrite_select").value == value diff --git a/tests/tests_tui/tui_base.py b/tests/tests_tui/tui_base.py index fc20cbbc2..75b872099 100644 --- a/tests/tests_tui/tui_base.py +++ b/tests/tests_tui/tui_base.py @@ -130,14 +130,14 @@ async def reload_tree_nodes(self, pilot, id, num_nodes): await pilot.pause() async def hover_and_press_tree(self, pilot, id, hover_line, press_string): - """Hover over a directorytree at a node-line and press a specific string""" + """Hover over a directorytree at a node-line and press a specific string.""" await pilot.pause() pilot.app.screen.query_one(id).hover_line = hover_line await pilot.pause() await self.press_tree(pilot, id, press_string) async def press_tree(self, pilot, id, press_string): - """Click on a tree to give it focus and press buttons""" + """Click on a tree to give it focus and press buttons.""" await self.scroll_to_click_pause(pilot, id) await pilot.press(press_string) await pilot.pause() @@ -177,16 +177,18 @@ async def check_and_click_onto_existing_project(self, pilot, project_name): ) async def change_checkbox(self, pilot, id): + """PLACEHOLDER.""" pilot.app.screen.query_one(id).toggle() await pilot.pause() async def switch_tab(self, pilot, tab): + """PLACEHOLDER.""" assert tab in ["create", "transfer", "configs", "logging"] content_tab = ContentTab.add_prefix(f"tabscreen_{tab}_tab") await self.scroll_to_click_pause(pilot, f"Tab#{content_tab}") async def turn_off_all_datatype_checkboxes(self, pilot, tab="create"): - """Make sure all checkboxes are off to start""" + """Make sure all checkboxes are off to start.""" assert tab in ["create", "transfer"] checkbox_names = canonical_configs.get_broad_datatypes() @@ -219,7 +221,7 @@ async def exit_to_main_menu_and_reeneter_project_manager( await self.check_and_click_onto_existing_project(pilot, project_name) async def close_messagebox(self, pilot): - """Close the modal_dialogs.Messagebox""" + """Close the modal_dialogs.Messagebox.""" pilot.app.screen.on_button_pressed() await pilot.pause() @@ -233,6 +235,7 @@ async def move_select_to_position(self, pilot, id, position): await pilot.pause() async def click_and_await_transfer(self, pilot): + """PLACEHOLDER.""" await self.scroll_to_click_pause(pilot, "#transfer_transfer_button") await self.scroll_to_click_pause(pilot, "#confirm_ok_button") diff --git a/tests/tests_unit/test_unit.py b/tests/tests_unit/test_unit.py index 3c55b526a..775bc089a 100644 --- a/tests/tests_unit/test_unit.py +++ b/tests/tests_unit/test_unit.py @@ -16,7 +16,7 @@ class TestUnit: "key", [tags("date"), tags("time"), tags("datetime")] ) def test_datetime_string_replacement(self, key, underscore_position): - """Test the function that replaces @DATE, @TIME@ or @DATETIME@ + r"""Test the function that replaces @DATE, @TIME@ or @DATETIME@ keywords with the date / time / datetime. Also, it will pre/append underscores to the tags if they are not already there (e.g if user input "sub-001@DATE"). @@ -45,7 +45,7 @@ def test_datetime_string_replacement(self, key, underscore_position): @pytest.mark.parametrize("prefix", ["sub", "ses"]) def test_process_to_keyword_in_sub_input(self, prefix): - """ """ + """PLACEHOLDER.""" results = formatting.update_names_with_range_to_flag( [f"{prefix}-001", f"{prefix}-01{tags('to')}123"], prefix ) @@ -94,6 +94,7 @@ def test_process_to_keyword_in_sub_input(self, prefix): def test_process_to_keyword_bad_input_raises_error( self, prefix, bad_input ): + """PLACEHOLDER.""" bad_input = bad_input.replace("prefix", prefix) with pytest.raises(ValueError) as e: @@ -142,7 +143,7 @@ def test_get_max_sub_or_ses_num_and_value_length_empty( ): """When the list of sub or ses names is empty, the returned max number should be zero and the `default_num_value_digits` - be set to the passed default + be set to the passed default. """ ( max_value, From b33d756d5ac2548e321cb1990d72bcf269faf4e6 Mon Sep 17 00:00:00 2001 From: MoffittAndrew Date: Fri, 21 Mar 2025 12:15:12 +0000 Subject: [PATCH 10/70] ignoring rule D100 for now --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index b7f6e1278..68a7a2766 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -131,6 +131,7 @@ per-file-ignores = { "tests/*" = [ # Inconsistent with movement repo, but saving this here for # now in case there are good reasons to keep these rules ignore = ["E203","E501","E731","C901","W291","W293","E402","E722", + "D100", # missing docstring in public module (not enforced FOR NOW) "D203", # one blank line before class "D213", # multi-line-summary second line "D401", # first line of docstrings should be in an imperative mood From 98bb643664dbdd5c3810a0578c5cccbc299cfc0a Mon Sep 17 00:00:00 2001 From: MoffittAndrew Date: Fri, 21 Mar 2025 12:19:44 +0000 Subject: [PATCH 11/70] fixed datashuttle_functions.py --- datashuttle/datashuttle_functions.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/datashuttle/datashuttle_functions.py b/datashuttle/datashuttle_functions.py index 250fd8bd6..5ab642570 100644 --- a/datashuttle/datashuttle_functions.py +++ b/datashuttle/datashuttle_functions.py @@ -28,7 +28,9 @@ def quick_validate_project( display_mode: DisplayMode = "warn", name_templates: Optional[Dict] = None, ) -> List[str]: - """Perform validation on the project. This checks the subject + """Perform validation on the project. + + This checks the subject and session level folders to ensure there are not NeuroBlueprint formatting issues. @@ -90,7 +92,9 @@ def quick_validate_project( def _format_top_level_folder( top_level_folder: TopLevelFolder | None, ) -> List[TopLevelFolder]: - """Take a `top_level_folder` ("rawdata" or "derivatives" str) and + """Format the top level folder. + + Take a `top_level_folder` ("rawdata" or "derivatives" str) and convert to list, if `None`, convert it to a list of both possible top-level folders. """ From 561903a9911c224b90953e35fdd1949170379887 Mon Sep 17 00:00:00 2001 From: MoffittAndrew Date: Fri, 21 Mar 2025 12:32:54 +0000 Subject: [PATCH 12/70] fixing datashuttle_class --- datashuttle/datashuttle_class.py | 28 ++++++++++++++++++++++------ pyproject.toml | 1 + 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/datashuttle/datashuttle_class.py b/datashuttle/datashuttle_class.py index 6cc044625..5adec00b9 100644 --- a/datashuttle/datashuttle_class.py +++ b/datashuttle/datashuttle_class.py @@ -105,6 +105,7 @@ class DataShuttle: """ def __init__(self, project_name: str, print_startup_message: bool = True): + """PLACEHOLDER.""" self._error_on_base_project_name(project_name) self.project_name = project_name ( @@ -752,7 +753,7 @@ def _transfer_top_level_folder( init_log: bool = True, ): """Core function to upload / download files within a - particular top-level-folder. e.g. `upload_rawdata().` + particular top-level-folder. e.g. `upload_rawdata()`. """ if init_log: self._start_log( @@ -824,7 +825,7 @@ def _transfer_specific_file_or_folder( def setup_ssh_connection(self) -> None: """Setup a connection to the central server using SSH. Assumes the central_host_id and central_host_username - are set in configs (see make_config_file() and update_config_file()) + are set in configs (see make_config_file() and update_config_file()). First, the server key will be displayed, requiring verification of the server ID. This will store the @@ -971,7 +972,7 @@ def make_config_file( ds_logger.close_log_filehandler() def update_config_file(self, **kwargs) -> None: - """ """ + """PLACEHOLDER.""" if not self.cfg: utils.log_and_raise_error( "Must have a config loaded before updating configs.", @@ -1022,6 +1023,7 @@ def get_config_path(self) -> Path: @check_configs_set def get_configs(self) -> Configs: + """PLACEHOLDER.""" return self.cfg @check_configs_set @@ -1049,6 +1051,9 @@ def get_next_sub( Parameters ---------- + top_level_folder + "rawdata" or "derivatives" + return_with_prefix If `True`, return with the "sub-" prefix. @@ -1247,7 +1252,7 @@ def validate_project( def check_name_formatting(names: Union[str, list], prefix: Prefix) -> None: """Pass list of names to check how these will be auto-formatted, for example as when passed to create_folders() or upload_custom() - or download() + or download(). Useful for checking tags e.g. @TO@, @DATE@, @DATETIME@, @DATE@. This method will print the formatted list of names, @@ -1292,6 +1297,14 @@ def _transfer_entire_project( upload_or_download direction to transfer the data, either "upload" (from local to central) or "download" (from central to local). + + overwrite_existing_files + determines whether or not to overwrite existing files + + dry_run + perform a dry-run of transfer. This will output as if file + transfer was taking place, but no files will be moved. Useful + to check which files will be moved on data transfer. """ for top_level_folder in canonical_folders.get_top_level_folders(): @@ -1328,6 +1341,9 @@ def _start_log( store_in_temp_folder if `False`, existing logging path will be used (local project .datashuttle). + + verbose + print warnings and error messages. """ if local_vars is None: @@ -1372,7 +1388,7 @@ def _move_logs_from_temp_folder(self) -> None: ) def _clear_temp_log_path(self) -> None: - """""" + """PLACEHOLDER.""" log_files = glob.glob(str(self._temp_log_path / "*.log")) for file in log_files: os.remove(file) @@ -1461,7 +1477,7 @@ def _init_persistent_settings(self) -> None: self._save_persistent_settings(settings) def _save_persistent_settings(self, settings: Dict) -> None: - """Save the settings dict to file as .yaml""" + """Save the settings dict to file as ".yaml".""" with open(self._persistent_settings_path, "w") as settings_file: yaml.dump(settings, settings_file, sort_keys=False) diff --git a/pyproject.toml b/pyproject.toml index 68a7a2766..63d9cd3c8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -133,6 +133,7 @@ per-file-ignores = { "tests/*" = [ ignore = ["E203","E501","E731","C901","W291","W293","E402","E722", "D100", # missing docstring in public module (not enforced FOR NOW) "D203", # one blank line before class + "D205", # 1 blank line required between summary line and description "D213", # multi-line-summary second line "D401", # first line of docstrings should be in an imperative mood ] From 31844315e4e6500d615159bac5b26de87c162009 Mon Sep 17 00:00:00 2001 From: MoffittAndrew Date: Fri, 21 Mar 2025 15:09:41 +0000 Subject: [PATCH 13/70] fixing utils --- datashuttle/configs/config_class.py | 6 ++ datashuttle/utils/custom_exceptions.py | 4 ++ datashuttle/utils/data_transfer.py | 95 +++++++++++++------------- datashuttle/utils/decorators.py | 2 +- datashuttle/utils/ds_logger.py | 3 + datashuttle/utils/folder_class.py | 1 + datashuttle/utils/folders.py | 34 +++++++-- datashuttle/utils/formatting.py | 9 ++- datashuttle/utils/getters.py | 9 ++- datashuttle/utils/rclone.py | 12 ++-- datashuttle/utils/ssh.py | 11 ++- datashuttle/utils/utils.py | 11 +-- datashuttle/utils/validation.py | 15 +++- 13 files changed, 143 insertions(+), 69 deletions(-) diff --git a/datashuttle/configs/config_class.py b/datashuttle/configs/config_class.py index f1282d935..b62427b7b 100644 --- a/datashuttle/configs/config_class.py +++ b/datashuttle/configs/config_class.py @@ -144,6 +144,9 @@ def build_project_path( a list (or string for 1) of folder names to be joined into a path. If file included, must be last entry (with ext). + + top_level_folder + either "rawdata" or "derivatives" """ if isinstance(sub_folders, list): @@ -173,6 +176,9 @@ def get_base_folder( ---------- base base path, "local", "central" or "datashuttle" + + top_level_folder + either "rawdata" or "derivatives" """ if base == "local": diff --git a/datashuttle/utils/custom_exceptions.py b/datashuttle/utils/custom_exceptions.py index 1d3d8e86a..3fe7e3368 100644 --- a/datashuttle/utils/custom_exceptions.py +++ b/datashuttle/utils/custom_exceptions.py @@ -1,6 +1,10 @@ class ConfigError(Exception): + """PLACEHOLDER.""" + pass class NeuroBlueprintError(Exception): + """PLACEHOLDER.""" + pass diff --git a/datashuttle/utils/data_transfer.py b/datashuttle/utils/data_transfer.py index 10562f300..bfd092bc0 100644 --- a/datashuttle/utils/data_transfer.py +++ b/datashuttle/utils/data_transfer.py @@ -19,44 +19,6 @@ class TransferData: The properties on this class are to be read during generation of transfer lists and should never be changed during the lifetime of the class. - - Parameters - ---------- - cfg - datashuttle configs UserDict. - - upload_or_download - Direction to perform the transfer. - - top_level_folder - The top-level folder structure where data is organized. - - sub_names - List of subject names or single subject to transfer. This - can include transfer keywords (e.g. "all_non_sub"). - - ses_names - List of sessions or single session to transfer, for each - subject. May include session-level transfer keywords. - - datatype - List of datatypes to transfer, for the sessions / subjects - specified. Can include datatype-level tranfser keywords. - - overwrite_existing_files - If "never" files on target will never be overwritten by source. - If "always" files on target will be overwritten by source if - there is any difference in date or size. - If "if_source_newer" files on target will only be overwritten - by files on source with newer creation / modification datetime. - - dry_run - If `True`, transfer will not actually occur but will be logged - as if it did (to see what would happen for a transfer). - - log - if `True`, log and print the transfer output. - """ def __init__( @@ -71,6 +33,43 @@ def __init__( dry_run: bool, log: bool, ): + """Parameters + ---------- + cfg + datashuttle configs UserDict. + + upload_or_download + Direction to perform the transfer. + + top_level_folder + The top-level folder structure where data is organized. + + sub_names + List of subject names or single subject to transfer. This + can include transfer keywords (e.g. "all_non_sub"). + + ses_names + List of sessions or single session to transfer, for each + subject. May include session-level transfer keywords. + + datatype + List of datatypes to transfer, for the sessions / subjects + specified. Can include datatype-level tranfser keywords. + + overwrite_existing_files + If "never" files on target will never be overwritten by source. + If "always" files on target will be overwritten by source if + there is any difference in date or size. + If "if_source_newer" files on target will only be overwritten + by files on source with newer creation / modification datetime. + + dry_run + If `True`, transfer will not actually occur but will be logged + as if it did (to see what would happen for a transfer). + + log + if `True`, log and print the transfer output. + """ self.__cfg = cfg self.__upload_or_download = upload_or_download self.__top_level_folder = top_level_folder @@ -112,12 +111,17 @@ def __init__( def build_a_list_of_all_files_and_folders_to_transfer(self) -> List[str]: """Build a list of every file to transfer based on the user-passed - arguments. This cycles through every subject, session and datatype + arguments. + + This cycles through every subject, session and datatype and adds the outputs to three lists: - `sub_ses_dtype_include` - files within datatype folders - `extra_folder_names` - folders that do not fall within datatype folders - `extra_file_names` - files that do not fall within datatype folders + `sub_ses_dtype_include` + files within datatype folders + `extra_folder_names` + folders that do not fall within datatype folders + `extra_file_names` + files that do not fall within datatype folders Returns ------- @@ -346,6 +350,7 @@ def update_list_with_dtype_paths( # ------------------------------------------------------------------------- def to_list(self, names: Union[str, List[str]]) -> List[str]: + """PLACEHOLDER.""" if isinstance(names, str): names = [names] return names @@ -425,9 +430,7 @@ def get_processed_names( will be searched to determine what files exist to transfer, and the sub / ses names list generated. - Parameters - ---------- - see transfer_sub_ses_data() + see transfer_sub_ses_data() for list of parameters. """ prefix: Prefix @@ -469,7 +472,7 @@ def get_processed_names( def transfer_non_datatype(self, datatype_checked: List[str]) -> bool: """Convenience function, bool if all non-datatype folders - are to be transferred + are to be transferred. """ return any( [name in ["all_non_datatype", "all"] for name in datatype_checked] diff --git a/datashuttle/utils/decorators.py b/datashuttle/utils/decorators.py index 99dcde8ed..19f3794b2 100644 --- a/datashuttle/utils/decorators.py +++ b/datashuttle/utils/decorators.py @@ -6,7 +6,7 @@ def requires_ssh_configs(func): """Decorator to check file is loaded. Used on Mainwindow class - methods only as first arg is assumed to be self (containing cfgs) + methods only as first arg is assumed to be self (containing cfgs). """ @wraps(func) diff --git a/datashuttle/utils/ds_logger.py b/datashuttle/utils/ds_logger.py index 353fc0886..4269b391d 100644 --- a/datashuttle/utils/ds_logger.py +++ b/datashuttle/utils/ds_logger.py @@ -18,14 +18,17 @@ def get_logger_name(): + """PLACEHOLDER.""" return "datashuttle" def get_logger(): + """PLACEHOLDER.""" return logging.getLogger(get_logger_name()) def logging_is_active(): + """PLACEHOLDER.""" logger_exists = get_logger_name() in logging.root.manager.loggerDict if logger_exists and get_logger().handlers != []: return True diff --git a/datashuttle/utils/folder_class.py b/datashuttle/utils/folder_class.py index 8b85856de..33bc16b74 100644 --- a/datashuttle/utils/folder_class.py +++ b/datashuttle/utils/folder_class.py @@ -10,5 +10,6 @@ def __init__( name: str, level: str, ): + """PLACEHOLDER.""" self.name = name self.level = level diff --git a/datashuttle/utils/folders.py b/datashuttle/utils/folders.py index f7d86c4b7..3a3dd3372 100644 --- a/datashuttle/utils/folders.py +++ b/datashuttle/utils/folders.py @@ -47,6 +47,12 @@ def create_folder_trees( Parameters ---------- + cfg + datashuttle config UserDict + + top_level_folder + either "rawdata" or "derivatives" + sub_names, ses_names, datatype see create_folders() @@ -268,10 +274,8 @@ def items_from_datatype_input( directly from user input, or by searching what is available if "all" is passed. - Parameters - ---------- - see _transfer_datatype() for parameters. - + see _transfer_datatype() for full + parameters list. """ base_folder = cfg.get_base_folder(local_or_central, top_level_folder) @@ -385,6 +389,9 @@ def search_for_wildcards( Parameters ---------- + cfg + datashuttle configs + project initialised datashuttle project @@ -449,7 +456,7 @@ def search_sub_or_ses_level( return_full_path: bool = False, ) -> Tuple[List[str] | List[Path], List[str]]: """Search project folder at the subject or session level. - Only returns folders + Only returns folders. Parameters ---------- @@ -459,26 +466,33 @@ def search_sub_or_ses_level( arguments, but this is not nice and breaks the general rule that these functions should operate project-agnostic. + + base_folder + the path to the base folder. If sub is None, the search is + performed on this folder local_or_central search in local or central project sub either a subject name (string) or None. If None, the search - is performed at the top_level_folder level + is performed at the base_folder level ses either a session name (string) or None, This must not be a session name if sub is None. If provided (with sub) then the session folder is searched - str + search_str glob-format search string to search at the folder level. verbose If `True`, if a search folder cannot be found, a message will be printed with the un-found path. + + return_full_path + include the search_path in the returned paths """ if ses and not sub: @@ -518,6 +532,9 @@ def search_for_folders( Parameters ---------- + cfg + datashuttle configs + local_or_central "local" or "central" @@ -531,6 +548,9 @@ def search_for_folders( If `True`, when a search folder cannot be found, a message will be printed with the missing path. + return_full_path + include the search_path in the returned paths + """ if local_or_central == "central" and cfg["connection_method"] == "ssh": all_folder_names, all_filenames = ssh.search_ssh_central_for_folders( diff --git a/datashuttle/utils/formatting.py b/datashuttle/utils/formatting.py index 1cea80fa8..5afdae8c2 100644 --- a/datashuttle/utils/formatting.py +++ b/datashuttle/utils/formatting.py @@ -120,7 +120,7 @@ def update_names_with_range_to_flag( keyword must be in the form prefix-num1@num2. The maximum number of leading zeros are used to pad the output e.g. - sub-01@003 becomes ["sub-001", "sub-002", "sub-003"] + sub-01@003 becomes ["sub-001", "sub-002", "sub-003"]. Input can also be a mixed list e.g. names = ["sub-01", "sub-02@TO@04", "sub-05@TO@10"] @@ -278,14 +278,17 @@ def replace_date_time_tags_in_name( def format_date(date: str) -> str: + """PLACEHOLDER.""" return f"date-{date}" def format_time(time_: str) -> str: + """PLACEHOLDER.""" return f"time-{time_}" def format_datetime(date: str, time_: str) -> str: + """PLACEHOLDER.""" return f"datetime-{date}T{time_}" @@ -293,7 +296,7 @@ def add_underscore_before_after_if_not_there(string: str, key: str) -> str: """If names are passed with @DATE@, @TIME@, or @DATETIME@ but not surrounded by underscores, check and insert if required. e.g. sub-001@DATE@ becomes sub-001_@DATE@ - or sub-001@DATEid-101 becomes sub-001_@DATE_id-101 + or sub-001@DATEid-101 becomes sub-001_@DATE_id-101. """ key_len = len(key) key_start_idx = string.index(key) @@ -320,7 +323,7 @@ def add_missing_prefixes_to_names( all_names: Union[List[str], str], prefix: str ) -> List[str]: """Make sure all elements in the list of names are - prefixed with the prefix, typically "sub-" or "ses-" + prefixed with the prefix, typically "sub-" or "ses-". Use expanded list for readability """ diff --git a/datashuttle/utils/getters.py b/datashuttle/utils/getters.py index ac648f54f..44e58a499 100644 --- a/datashuttle/utils/getters.py +++ b/datashuttle/utils/getters.py @@ -77,6 +77,10 @@ def get_next_sub_or_ses( the desired value can be entered here. e.g. if 3 (the default), if no subjects are found the subject returned will be "sub-001". + name_template_regexp + the name template to try and get the num digits from. + If unspecified, the number of digits will be default_num_value_digits. + Returns ------- suggested_new_num @@ -133,7 +137,8 @@ def get_max_sub_or_ses_num_and_value_length( all_folders A list of BIDS-style formatted folder names. - see `get_next_sub_or_ses()` for other arguments. + prefix, default_num_value_digits, name_template_regexp + see `get_next_sub_or_ses()`. Returns ------- @@ -229,7 +234,7 @@ def get_num_value_digits_from_project( def get_num_value_digits_from_regexp( prefix: Prefix, name_template_regexp: str ) -> Union[Literal[False], int]: - """Given a name template regexp, find the number of values for the + r"""Given a name template regexp, find the number of values for the sub or ses key. These will be fixed with "\d" (digit) or ".?" (wildcard). If there is length-unspecific wildcard (.*) in the sub key, then skip. In practice, there should never really be a .* in the sub or ses diff --git a/datashuttle/utils/rclone.py b/datashuttle/utils/rclone.py index a0bd2772c..d651b8f96 100644 --- a/datashuttle/utils/rclone.py +++ b/datashuttle/utils/rclone.py @@ -112,6 +112,7 @@ def setup_rclone_config_for_ssh( def log_rclone_config_output(): + """PLACEHOLDER.""" output = call_rclone("config file", pipe_std=True) utils.log( f"Successfully created rclone config. {output.stdout.decode('utf-8')}" @@ -218,6 +219,9 @@ def get_local_and_central_file_differences( Parameters ---------- + cfg + datashuttle configs UserDict. + top_level_folders_to_check List of top-level folders to check. @@ -281,9 +285,9 @@ def assert_rclone_check_output_is_as_expected(result, symbol, convert_symbols): def perform_rclone_check( cfg: Configs, top_level_folder: TopLevelFolder ) -> str: - """Use Rclone's `check` command to build a list of files that + r"""Use Rclone's `check` command to build a list of files that are the same ("="), different ("*"), found in local only ("+") - or central only ("-"). The output is formatted as " \n". + or central only ("-"). The output is formatted as "\ \\n". """ local_filepath = cfg.get_base_folder( "local", top_level_folder @@ -306,7 +310,7 @@ def perform_rclone_check( def handle_rclone_arguments( rclone_options: Dict, include_list: List[str] ) -> str: - """Construct the extra arguments to pass to RClone,""" + """Construct the extra arguments to pass to RClone.""" extra_arguments_list = [] extra_arguments_list += ["-" + rclone_options["transfer_verbosity"]] @@ -336,7 +340,7 @@ def handle_rclone_arguments( def rclone_args(name: str) -> str: - """Central function to hold rclone commands""" + """Central function to hold rclone commands.""" valid_names = [ "dry_run", "copy", diff --git a/datashuttle/utils/ssh.py b/datashuttle/utils/ssh.py index b1cd9d2b0..9b80f7d4c 100644 --- a/datashuttle/utils/ssh.py +++ b/datashuttle/utils/ssh.py @@ -29,6 +29,7 @@ def connect_client_core( cfg: Configs, password: Optional[str] = None, ): + """PLACEHOLDER.""" client.get_host_keys().load(cfg.hostkeys_path.as_posix()) client.set_missing_host_key_policy(paramiko.RejectPolicy()) @@ -71,6 +72,7 @@ def add_public_key_to_central_authorized_keys( def generate_and_write_ssh_key(ssh_key_path: Path) -> None: + """PLACEHOLDER.""" key = paramiko.RSAKey.generate(4096) key.write_private_key_file(ssh_key_path.as_posix()) @@ -87,6 +89,7 @@ def get_remote_server_key(central_host_id: str): def save_hostkey_locally(key, central_host_id, hostkeys_path) -> None: + """PLACEHOLDER.""" client = paramiko.SSHClient() client.get_host_keys().add(central_host_id, key.get_name(), key) client.get_host_keys().save(hostkeys_path.as_posix()) @@ -262,6 +265,9 @@ def search_ssh_central_for_folders( If `True`, if a search folder cannot be found, a message will be printed with the un-found path. + return_full_path + include the search_path in the returned paths + """ client: paramiko.SSHClient with paramiko.SSHClient() as client: @@ -294,7 +300,7 @@ def get_list_of_folder_names_over_sftp( Parameters ---------- - stfp + sftp connected paramiko stfp object (see search_ssh_central_for_folders()) @@ -308,6 +314,9 @@ def get_list_of_folder_names_over_sftp( verbose If `True`, if a search folder cannot be found, a message will be printed with the un-found path. + + return_full_path + include the search_path in the returned paths """ all_folder_names = [] diff --git a/datashuttle/utils/utils.py b/datashuttle/utils/utils.py index 18b98f913..56dd43670 100644 --- a/datashuttle/utils/utils.py +++ b/datashuttle/utils/utils.py @@ -29,7 +29,7 @@ def log(message: str) -> None: def log_and_message(message: str, use_rich: bool = False) -> None: """Log the message and send it to user. - use_rich : is True, use rich's print() function + use_rich : is True, use rich's print() function. """ log(message) print_message_to_user(message, use_rich) @@ -45,7 +45,7 @@ def log_and_raise_error(message: str, exception: Any) -> None: def warn(message: str, log: bool) -> None: - """ """ + """PLACEHOLDER.""" if log and ds_logger.logging_is_active(): logger = ds_logger.get_logger() logger.warning(message) @@ -74,7 +74,7 @@ def print_message_to_user( def get_user_input(message: str) -> str: - """Centralised way to get user input""" + """Centralised way to get user input.""" input_ = input(message) return input_ @@ -85,6 +85,7 @@ def get_user_input(message: str) -> str: def path_starts_with_base_folder(base_folder: Path, path_: Path) -> bool: + """PLACEHOLDER.""" return path_.as_posix().startswith(base_folder.as_posix()) @@ -158,6 +159,7 @@ def get_values_from_bids_formatted_name( def sub_or_ses_value_to_int(value: str) -> int: + """PLACEHOLDER.""" try: int_value = int(value) except ValueError: @@ -182,6 +184,7 @@ def get_value_from_key_regexp(name: str, key: str) -> List[str]: def integers_are_consecutive(list_of_ints: List[int]) -> bool: + """PLACEHOLDER.""" diff_between_ints = diff(list_of_ints) return all([diff == 1 for diff in diff_between_ints]) @@ -194,7 +197,7 @@ def diff(x: List) -> List: def num_leading_zeros(string: str) -> int: - """int() strips leading zeros""" + """int() strips leading zeros.""" if string[:4] in ["sub-", "ses-"]: string = string[4:] diff --git a/datashuttle/utils/validation.py b/datashuttle/utils/validation.py index 99c798fb6..b4e89974d 100644 --- a/datashuttle/utils/validation.py +++ b/datashuttle/utils/validation.py @@ -34,6 +34,7 @@ def get_missing_prefix_error(name: str, prefix, path_: Path | None) -> str: + """PLACEHOLDER.""" return handle_path( f"MISSING_PREFIX: The prefix {prefix} was not found in the name: {name}", path_, @@ -41,6 +42,7 @@ def get_missing_prefix_error(name: str, prefix, path_: Path | None) -> str: def get_bad_value_error(name: str, prefix, path_: Path | None) -> str: + """PLACEHOLDER.""" return handle_path( f"BAD_VALUE: The value for prefix {prefix} in name {name} is not an integer.", path_, @@ -48,6 +50,7 @@ def get_bad_value_error(name: str, prefix, path_: Path | None) -> str: def get_duplicate_prefix_error(name: str, prefix, path_: Path | None) -> str: + """PLACEHOLDER.""" return handle_path( f"DUPLICATE_PREFIX: The name: {name} of contains more than one instance of the prefix {prefix}.", path_, @@ -55,12 +58,14 @@ def get_duplicate_prefix_error(name: str, prefix, path_: Path | None) -> str: def get_name_error(name: str, prefix: Prefix, path_: Path | None) -> str: + """PLACEHOLDER.""" return handle_path( f"BAD_NAME: The name: {name} of type: {prefix} is not valid.", path_ ) def get_special_char_error(name: str, path_: Path | None) -> str: + """PLACEHOLDER.""" return handle_path( f"SPECIAL_CHAR: The name: {name}, contains characters which are not alphanumeric, dash or underscore.", path_, @@ -68,6 +73,7 @@ def get_special_char_error(name: str, path_: Path | None) -> str: def get_name_format_error(name: str, path_: Path | None) -> str: + """PLACEHOLDER.""" return handle_path( f"NAME_FORMAT: The name {name} does not consist of key-value pairs separated by underscores.", path_, @@ -75,10 +81,12 @@ def get_name_format_error(name: str, path_: Path | None) -> str: def get_value_length_error(prefix: Prefix) -> str: + """PLACEHOLDER.""" return f"VALUE_LENGTH: Inconsistent value lengths for the prefix: {prefix} were found in the project." def get_datetime_error(key, name: str, strfmt: str, path_: Path | None) -> str: + """PLACEHOLDER.""" return handle_path( f"DATETIME: Name {name} contains an invalid {key}. It should be ISO format: {strfmt}.", path_, @@ -98,6 +106,7 @@ def get_template_error(name: str, regexp: str, path_: Path | None) -> str: def get_missing_top_level_folder_error( path_: Path | None, local_or_central: Literal["local", "central"] ) -> str: + """PLACEHOLDER.""" return handle_path( f"TOP_LEVEL_FOLDER: The {local_or_central} project must contain a 'rawdata' or 'derivatives' folder.", path_, @@ -107,6 +116,7 @@ def get_missing_top_level_folder_error( def get_duplicate_name_error( new_name: str, exist_name: str, exist_path: Path | None ) -> str: + """PLACEHOLDER.""" return handle_path( f"DUPLICATE_NAME: The prefix for {new_name} duplicates the name: {exist_name}.", exist_path, @@ -114,12 +124,14 @@ def get_duplicate_name_error( def get_datatype_error(datatype_name: str, path_: Path | None) -> str: + """PLACEHOLDER.""" return handle_path( f"DATATYPE: {datatype_name} is not a valid datatype name.", path_ ) def handle_path(message: str, path_: Path | None) -> str: + """PLACEHOLDER.""" if path_: message += f" Path: {path_.as_posix()}" return message @@ -301,7 +313,7 @@ def get_path_and_name(path_or_name: Path | str) -> Tuple[Optional[Path], str]: def replace_tags_in_regexp(regexp: str) -> str: - """Before validation, all tags in the names are converted to + r"""Before validation, all tags in the names are converted to their final values (e.g. @DATE@ -> _date-). We also want to allow template to be formatted like `sub-\d\d_@DATE@` as it is convenient for auto-completion in the TUI. @@ -349,6 +361,7 @@ def names_include_special_characters( def name_has_special_character(name: str) -> bool: + """PLACEHOLDER.""" return not re.match("^[A-Za-z0-9_-]*$", name) From 52f03ec6ded9e75d8e345803a6d581254a6acacb Mon Sep 17 00:00:00 2001 From: MoffittAndrew Date: Fri, 21 Mar 2025 15:19:42 +0000 Subject: [PATCH 14/70] fixed tui --- datashuttle/tui/app.py | 12 ++++++++++-- datashuttle/tui/configs.py | 11 +++++++---- datashuttle/tui/custom_widgets.py | 11 ++++++++++- datashuttle/tui/interface.py | 10 +++++++++- .../tui/screens/create_folder_settings.py | 10 +++++++++- datashuttle/tui/screens/datatypes.py | 8 +++++++- datashuttle/tui/screens/get_help.py | 9 ++++++++- datashuttle/tui/screens/modal_dialogs.py | 19 ++++++++++++++++--- datashuttle/tui/screens/new_project.py | 3 +++ datashuttle/tui/screens/project_manager.py | 3 +++ datashuttle/tui/screens/project_selector.py | 3 +++ datashuttle/tui/screens/settings.py | 7 ++++++- datashuttle/tui/screens/setup_ssh.py | 5 ++++- datashuttle/tui/tabs/create_folders.py | 10 +++++++--- datashuttle/tui/tabs/logging.py | 17 +++++++++++++++++ datashuttle/tui/tabs/transfer.py | 14 +++++++++++--- datashuttle/tui/tabs/transfer_status_tree.py | 2 ++ datashuttle/tui/utils/tui_validators.py | 2 ++ 18 files changed, 134 insertions(+), 22 deletions(-) diff --git a/datashuttle/tui/app.py b/datashuttle/tui/app.py index fb342c445..7fe2fc0a5 100644 --- a/datashuttle/tui/app.py +++ b/datashuttle/tui/app.py @@ -47,6 +47,7 @@ class TuiApp(App, inherit_bindings=False): # type: ignore ] def compose(self) -> ComposeResult: + """PLACEHOLDER.""" yield Container( Label("datashuttle", id="mainwindow_banner_label"), Button( @@ -60,9 +61,11 @@ def compose(self) -> ComposeResult: ) def on_mount(self) -> None: + """PLACEHOLDER.""" self.set_dark_mode(self.load_global_settings()["dark_mode"]) def set_dark_mode(self, dark_mode: bool) -> None: + """PLACEHOLDER.""" self.theme = "textual-dark" if dark_mode else "textual-light" def on_button_pressed(self, event: Button.Pressed) -> None: @@ -91,6 +94,7 @@ def on_button_pressed(self, event: Button.Pressed) -> None: self.push_screen(get_help.GetHelpScreen()) def load_project_page(self, interface: Interface) -> None: + """PLACEHOLDER.""" if interface: self.push_screen( project_manager.ProjectManagerScreen( @@ -99,6 +103,7 @@ def load_project_page(self, interface: Interface) -> None: ) def show_modal_error_dialog(self, message: str) -> None: + """PLACEHOLDER.""" self.push_screen(modal_dialogs.MessageBox(message, border_color="red")) def handle_open_filesystem_browser(self, path_: Path) -> None: @@ -131,14 +136,14 @@ def handle_open_filesystem_browser(self, path_: Path) -> None: self.show_modal_error_dialog(message) def prompt_rename_file_or_folder(self, path_): - """ """ + """PLACEHOLDER.""" self.push_screen( modal_dialogs.RenameFileOrFolderScreen(self, path_), lambda new_name: self.rename_file_or_folder(path_, new_name), ) def rename_file_or_folder(self, path_, new_name): - """ """ + """PLACEHOLDER.""" if new_name is False: return try: @@ -185,12 +190,14 @@ def get_global_settings_path(self) -> Path: return path_ / "global_tui_settings.yaml" def get_default_global_settings(self) -> Dict: + """PLACEHOLDER.""" return { "dark_mode": True, "show_transfer_tree_status": False, } def save_global_settings(self, global_settings: Dict) -> None: + """PLACEHOLDER.""" settings_path = self.get_global_settings_path() if not settings_path.parent.is_dir(): @@ -212,6 +219,7 @@ def copy_to_clipboard(self, value): def main(): + """PLACEHOLDER.""" TuiApp().run() diff --git a/datashuttle/tui/configs.py b/datashuttle/tui/configs.py index 27a701d24..0d50f8712 100644 --- a/datashuttle/tui/configs.py +++ b/datashuttle/tui/configs.py @@ -30,7 +30,7 @@ class ConfigsContent(Container): - """This screen holds widgets and logic for setting datashuttle configs. + """Holds widgets and logic for setting datashuttle configs. It is used in `NewProjectPage` to instantiate a new project and initialise configs, or in `TabbedContent` to update an existing project's configs. @@ -44,6 +44,8 @@ class ConfigsContent(Container): @dataclass class ConfigsSaved(Message): + """PLACEHOLDER.""" + pass def __init__( @@ -52,6 +54,7 @@ def __init__( interface: Optional[Interface], id: str, ) -> None: + """PLACEHOLDER.""" super(ConfigsContent, self).__init__(id=id) self.parent_class = parent_class @@ -62,7 +65,7 @@ def compose(self) -> ComposeResult: """`self.config_ssh_widgets` are SSH-setup related widgets that are only required when the user selects the SSH connection method. These are displayed / hidden based on the - `connection_method` + `connection_method`. `config_screen_widgets` are core config-related widgets that are always displayed. @@ -269,7 +272,7 @@ def set_central_path_input_tooltip(self, display_ssh: bool) -> None: def get_platform_dependent_example_paths( self, local_or_central: Literal["local", "central"], ssh: bool = False ) -> str: - """ """ + """PLACEHOLDER.""" assert local_or_central in ["local", "central"] # Handle the ssh central case separately @@ -385,7 +388,7 @@ def handle_input_fill_from_select_directory( ).value = path_.as_posix() def setup_ssh_connection(self) -> None: - """Set up the `SetupSshScreen` screen,""" + """Set up the `SetupSshScreen` screen.""" assert self.interface is not None, "type narrow flexible `interface`" if not self.widget_configs_match_saved_configs(): diff --git a/datashuttle/tui/custom_widgets.py b/datashuttle/tui/custom_widgets.py index 7f2b2e12b..77178842b 100644 --- a/datashuttle/tui/custom_widgets.py +++ b/datashuttle/tui/custom_widgets.py @@ -45,6 +45,8 @@ class ClickableInput(Input): @dataclass class Clicked(Message): + """PLACEHOLDER.""" + input: ClickableInput ctrl: bool @@ -56,6 +58,7 @@ def __init__( validate_on: Optional[List[str]] = None, validators: Optional[List[Validator]] = None, ) -> None: + """PLACEHOLDER.""" super(ClickableInput, self).__init__( placeholder=placeholder, id=id, @@ -69,9 +72,11 @@ def _on_click(self, event: events.Click) -> None: self.post_message(self.Clicked(self, event.ctrl)) def as_names_list(self) -> List[str]: + """PLACEHOLDER.""" return self.value.replace(" ", "").split(",") def on_key(self, event: events.Key) -> None: + """PLACEHOLDER.""" if event.key == "ctrl+q": self.mainwindow.copy_to_clipboard(self.value) @@ -92,12 +97,15 @@ class CustomDirectoryTree(DirectoryTree): @dataclass class DirectoryTreeSpecialKeyPress(Message): + """PLACEHOLDER.""" + key: str node_path: Optional[Path] def __init__( self, mainwindow: App, path: Path, id: Optional[str] = None ) -> None: + """PLACEHOLDER.""" super(CustomDirectoryTree, self).__init__(path=path, id=id) self.mainwindow = mainwindow @@ -139,7 +147,7 @@ def on_key(self, event: events.Key) -> None: def _render_line( self, y: int, x1: int, x2: int, base_style: Style ) -> Strip: - """This function is overridden from textual's `Tree` class to stop + """Overridden from textual's `Tree` class to stop CSS styling on hovering and clicking which was distracting / changed the default color used for transfer status, respectively. @@ -397,6 +405,7 @@ class TopLevelFolderSelect(Select): """ def __init__(self, interface: Interface, id: str) -> None: + """PLACEHOLDER.""" self.interface = interface top_level_folders = [ diff --git a/datashuttle/tui/interface.py b/datashuttle/tui/interface.py index 8ede9cb0e..0f97b06ec 100644 --- a/datashuttle/tui/interface.py +++ b/datashuttle/tui/interface.py @@ -30,6 +30,7 @@ class Interface: """ def __init__(self) -> None: + """PLACEHOLDER.""" self.project: DataShuttle self.name_templates: Dict = {} self.tui_settings: Dict = {} @@ -347,7 +348,7 @@ def save_tui_settings( value Value to set the `persistent_settings` tui field to - key_1 + key First key of the tui `persistent_settings` to update e.g. "top_level_folder_select" @@ -367,9 +368,11 @@ def save_tui_settings( # ---------------------------------------------------------------------------------- def get_central_host_id(self) -> str: + """PLACEHOLDER.""" return self.project.cfg["central_host_id"] def get_configs(self) -> Configs: + """PLACEHOLDER.""" return self.project.cfg def get_textual_compatible_project_configs(self) -> Configs: @@ -385,6 +388,7 @@ def get_textual_compatible_project_configs(self) -> Configs: def get_next_sub( self, top_level_folder: TopLevelFolder ) -> InterfaceOutput: + """PLACEHOLDER.""" try: next_sub = self.project.get_next_sub( top_level_folder, @@ -398,6 +402,7 @@ def get_next_sub( def get_next_ses( self, top_level_folder: TopLevelFolder, sub: str ) -> InterfaceOutput: + """PLACEHOLDER.""" try: next_ses = self.project.get_next_ses( top_level_folder, @@ -410,6 +415,7 @@ def get_next_ses( return False, str(e) def get_ssh_hostkey(self) -> InterfaceOutput: + """PLACEHOLDER.""" try: key = ssh.get_remote_server_key( self.project.cfg["central_host_id"] @@ -419,6 +425,7 @@ def get_ssh_hostkey(self) -> InterfaceOutput: return False, str(e) def save_hostkey_locally(self, key: paramiko.RSAKey) -> InterfaceOutput: + """PLACEHOLDER.""" try: ssh.save_hostkey_locally( key, @@ -433,6 +440,7 @@ def save_hostkey_locally(self, key: paramiko.RSAKey) -> InterfaceOutput: def setup_key_pair_and_rclone_config( self, password: str ) -> InterfaceOutput: + """PLACEHOLDER.""" try: ssh.add_public_key_to_central_authorized_keys( self.project.cfg, password, log=False diff --git a/datashuttle/tui/screens/create_folder_settings.py b/datashuttle/tui/screens/create_folder_settings.py index 26ae336ec..fa15349ba 100644 --- a/datashuttle/tui/screens/create_folder_settings.py +++ b/datashuttle/tui/screens/create_folder_settings.py @@ -27,7 +27,7 @@ class CreateFoldersSettingsScreen(ModalScreen): - """This screen handles setting datashuttle's `name_template`'s, as well + """Handles setting datashuttle's `name_template`'s, as well as the top-level-folder select and option to bypass all validation. Name Templates @@ -53,6 +53,7 @@ class CreateFoldersSettingsScreen(ModalScreen): TITLE = "Create Folders Settings" def __init__(self, mainwindow: App, interface: Interface) -> None: + """PLACEHOLDER.""" super(CreateFoldersSettingsScreen, self).__init__() self.mainwindow = mainwindow @@ -67,9 +68,11 @@ def __init__(self, mainwindow: App, interface: Interface) -> None: } def action_link_docs(self) -> None: + """PLACEHOLDER.""" webbrowser.open(links.get_docs_link()) def compose(self) -> ComposeResult: + """PLACEHOLDER.""" sub_on = True if self.input_mode == "sub" else False ses_on = not sub_on @@ -139,6 +142,7 @@ def compose(self) -> ComposeResult: ) def on_mount(self) -> None: + """PLACEHOLDER.""" for id in [ "#create_folders_settings_toplevel_select", "#create_folders_settings_bypass_validation_checkbox", @@ -151,11 +155,13 @@ def on_mount(self) -> None: self.switch_template_container_disabled() def init_input_values_holding_variable(self) -> None: + """PLACEHOLDER.""" name_templates = self.interface.get_name_templates() self.input_values["sub"] = name_templates["sub"] self.input_values["ses"] = name_templates["ses"] def switch_template_container_disabled(self) -> None: + """PLACEHOLDER.""" is_on = self.query_one( "#template_settings_validation_on_checkbox" ).value @@ -195,6 +201,7 @@ def on_button_pressed(self, event: Button.Pressed) -> None: self.interface.save_tui_settings(False, "bypass_validation") def make_name_templates_from_widgets(self) -> Dict: + """PLACEHOLDER.""" return { "on": self.query_one( "#template_settings_validation_on_checkbox" @@ -241,6 +248,7 @@ def on_radio_set_changed(self, event: RadioSet.Changed) -> None: self.fill_input_from_template() def on_input_changed(self, message: Input.Changed) -> None: + """PLACEHOLDER.""" if message.input.id == "template_settings_input": val = None if message.value == "" else message.value self.input_values[self.input_mode] = val diff --git a/datashuttle/tui/screens/datatypes.py b/datashuttle/tui/screens/datatypes.py index 78899342e..aea5c396f 100644 --- a/datashuttle/tui/screens/datatypes.py +++ b/datashuttle/tui/screens/datatypes.py @@ -89,6 +89,7 @@ def __init__( create_or_transfer: Literal["create", "transfer"], interface: Interface, ) -> None: + """PLACEHOLDER.""" super(DisplayedDatatypesScreen, self).__init__() self.interface = interface @@ -138,6 +139,7 @@ def compose(self) -> ComposeResult: ) def on_mount(self): + """PLACEHOLDER.""" pass # self.query_one("#display_datatypes_screen_container").action_scroll_up() @@ -212,6 +214,7 @@ def __init__( create_or_transfer: Literal["create", "transfer"] = "create", id: Optional[str] = None, ) -> None: + """PLACEHOLDER.""" super(DatatypeCheckboxes, self).__init__(id=id) self.interface = interface @@ -226,6 +229,7 @@ def __init__( ] def compose(self) -> ComposeResult: + """PLACEHOLDER.""" for datatype, setting in self.datatype_config.items(): if setting["displayed"]: yield Checkbox( @@ -251,7 +255,7 @@ def on_checkbox_changed(self) -> None: ) def on_mount(self) -> None: - """ """ + """PLACEHOLDER.""" for datatype in self.datatype_config.keys(): if self.datatype_config[datatype]["displayed"]: self.query_one( @@ -277,12 +281,14 @@ def selected_datatypes(self) -> List[str]: def get_checkbox_name( create_or_transfer: Literal["create", "transfer"], datatype ): + """PLACEHOLDER.""" return f"{create_or_transfer}_{datatype}_checkbox" def get_tui_settings_key_name( create_or_transfer: Literal["create", "transfer"], ) -> str: + """PLACEHOLDER.""" if create_or_transfer == "create": settings_key = "create_checkboxes_on" else: diff --git a/datashuttle/tui/screens/get_help.py b/datashuttle/tui/screens/get_help.py index 622f9474b..76a7ff1c6 100644 --- a/datashuttle/tui/screens/get_help.py +++ b/datashuttle/tui/screens/get_help.py @@ -18,9 +18,10 @@ class GetHelpScreen(ModalScreen): - """ """ + """PLACEHOLDER.""" def __init__(self) -> None: + """PLACEHOLDER.""" super(GetHelpScreen, self).__init__() self.text = """ @@ -35,18 +36,23 @@ def __init__(self) -> None: """ def action_link_docs(self) -> None: + """PLACEHOLDER.""" webbrowser.open(links.get_docs_link()) def action_link_github(self) -> None: + """PLACEHOLDER.""" webbrowser.open(links.get_github_link()) def action_link_github_issues(self) -> None: + """PLACEHOLDER.""" webbrowser.open(links.get_link_github_issues()) def action_link_zulip(self): + """PLACEHOLDER.""" webbrowser.open(links.get_link_zulip()) def compose(self) -> ComposeResult: + """PLACEHOLDER.""" yield Container( Static(self.text, id="get_help_label"), Button("Main Menu", id="all_main_menu_buttons"), @@ -54,5 +60,6 @@ def compose(self) -> ComposeResult: ) def on_button_pressed(self, event: Button.Pressed) -> None: + """PLACEHOLDER.""" if event.button.id == "all_main_menu_buttons": self.dismiss() diff --git a/datashuttle/tui/screens/modal_dialogs.py b/datashuttle/tui/screens/modal_dialogs.py index 9625c0318..6acb79b91 100644 --- a/datashuttle/tui/screens/modal_dialogs.py +++ b/datashuttle/tui/screens/modal_dialogs.py @@ -34,12 +34,14 @@ class MessageBox(ModalScreen): """ def __init__(self, message: str, border_color: str) -> None: + """PLACEHOLDER.""" super(MessageBox, self).__init__() self.message = message self.border_color = border_color def compose(self) -> ComposeResult: + """PLACEHOLDER.""" yield Container( Container( Static(self.message, id="messagebox_message_label"), @@ -50,6 +52,7 @@ def compose(self) -> ComposeResult: ) def on_mount(self) -> None: + """PLACEHOLDER.""" if self.border_color == "red": color = "rgb(140, 12, 0)" elif self.border_color == "green": @@ -65,6 +68,7 @@ def on_mount(self) -> None: ) def on_button_pressed(self) -> None: + """PLACEHOLDER.""" self.dismiss(True) @@ -81,12 +85,14 @@ def __init__( message: str, transfer_func: Callable[[], Worker[InterfaceOutput]], ) -> None: + """PLACEHOLDER.""" super().__init__() self.transfer_func = transfer_func self.message = message def compose(self) -> ComposeResult: + """PLACEHOLDER.""" yield Container( Label(self.message, id="confirm_message_label"), Horizontal( @@ -98,6 +104,7 @@ def compose(self) -> ComposeResult: ) def on_button_pressed(self, event: Button.Pressed) -> None: + """PLACEHOLDER.""" if event.button.id == "confirm_ok_button": self.query_one("#confirm_button_container").remove() @@ -114,7 +121,7 @@ def on_button_pressed(self, event: Button.Pressed) -> None: self.dismiss() async def handle_transfer_and_update_ui_when_complete(self) -> None: - """Runs the data transfer worker and updates the UI on completion""" + """Runs the data transfer worker and updates the UI on completion.""" data_transfer_worker = self.transfer_func() await data_transfer_worker.wait() success, output = data_transfer_worker.result @@ -152,6 +159,7 @@ class SelectDirectoryTreeScreen(ModalScreen): """ def __init__(self, mainwindow: App, path_: Optional[Path] = None) -> None: + """PLACEHOLDER.""" super(SelectDirectoryTreeScreen, self).__init__() self.mainwindow = mainwindow @@ -162,6 +170,7 @@ def __init__(self, mainwindow: App, path_: Optional[Path] = None) -> None: self.prev_click_time = 0 def compose(self) -> ComposeResult: + """PLACEHOLDER.""" label_message = ( "Select (double click) a folder with the same name as the project.\n" "If the project folder does not exist, select the parent folder and it will be created." @@ -180,26 +189,30 @@ def compose(self) -> ComposeResult: @require_double_click def on_directory_tree_directory_selected(self, node) -> None: + """PLACEHOLDER.""" if node.path.is_file(): return else: self.dismiss(node.path) def on_button_pressed(self, event: Button.Pressed) -> None: + """PLACEHOLDER.""" if event.button.id == "cancel_button": self.dismiss(False) class RenameFileOrFolderScreen(ModalScreen): - """ """ + """PLACEHOLDER.""" def __init__(self, mainwindow: App, path_: Path) -> None: + """PLACEHOLDER.""" super(RenameFileOrFolderScreen, self).__init__() self.mainwindow = mainwindow self.path_ = path_ def compose(self) -> ComposeResult: + """PLACEHOLDER.""" yield Container( Label("Input the new name:", id="rename_screen_label"), Input(value=self.path_.stem, id="rename_screen_input"), @@ -212,7 +225,7 @@ def compose(self) -> ComposeResult: ) def on_button_pressed(self, event: Button.Pressed) -> None: - """""" + """PLACEHOLDER.""" if event.button.id == "rename_screen_okay_button": self.dismiss(self.query_one("#rename_screen_input").value) diff --git a/datashuttle/tui/screens/new_project.py b/datashuttle/tui/screens/new_project.py index 769629ff3..d387eea39 100644 --- a/datashuttle/tui/screens/new_project.py +++ b/datashuttle/tui/screens/new_project.py @@ -37,11 +37,13 @@ class NewProjectScreen(Screen): TITLE = "Make New Project" def __init__(self, mainwindow: App) -> None: + """PLACEHOLDER.""" super(NewProjectScreen, self).__init__() self.mainwindow = mainwindow def compose(self) -> ComposeResult: + """PLACEHOLDER.""" yield Header() yield Button("Main Menu", id="all_main_menu_buttons") yield configs.ConfigsContent( @@ -49,5 +51,6 @@ def compose(self) -> ComposeResult: ) def on_button_pressed(self, event: Button.Pressed) -> None: + """PLACEHOLDER.""" if event.button.id == "all_main_menu_buttons": self.dismiss(None) diff --git a/datashuttle/tui/screens/project_manager.py b/datashuttle/tui/screens/project_manager.py index ebe0168e9..52664edee 100644 --- a/datashuttle/tui/screens/project_manager.py +++ b/datashuttle/tui/screens/project_manager.py @@ -41,6 +41,7 @@ class ProjectManagerScreen(Screen): """ def __init__(self, mainwindow: App, interface: Interface, id) -> None: + """PLACEHOLDER.""" super(ProjectManagerScreen, self).__init__(id=id) self.mainwindow = mainwindow @@ -51,6 +52,7 @@ def __init__(self, mainwindow: App, interface: Interface, id) -> None: self.tabbed_content_mount_signal = True def compose(self) -> ComposeResult: + """PLACEHOLDER.""" yield Header() yield Button("Main Menu", id="all_main_menu_buttons") with TabbedContent( @@ -118,6 +120,7 @@ def on_tabbed_content_tab_activated( ).update_most_recent_label() def update_active_tab_tree(self): + """PLACEHOLDER.""" active_tab_id = self.query_one("#tabscreen_tabbed_content").active self.query_one(f"#{active_tab_id}").reload_directorytree() diff --git a/datashuttle/tui/screens/project_selector.py b/datashuttle/tui/screens/project_selector.py index 3bd923570..d952e44fe 100644 --- a/datashuttle/tui/screens/project_selector.py +++ b/datashuttle/tui/screens/project_selector.py @@ -36,6 +36,7 @@ class ProjectSelectorScreen(Screen): TITLE = "Select Project" def __init__(self, mainwindow: App) -> None: + """PLACEHOLDER.""" super(ProjectSelectorScreen, self).__init__() self.project_names = [ @@ -44,6 +45,7 @@ def __init__(self, mainwindow: App) -> None: self.mainwindow = mainwindow def compose(self) -> ComposeResult: + """PLACEHOLDER.""" yield Header(id="project_select_header") yield Button("Main Menu", id="all_main_menu_buttons") yield Container( @@ -52,6 +54,7 @@ def compose(self) -> ComposeResult: ) def on_button_pressed(self, event: Button.Pressed) -> None: + """PLACEHOLDER.""" if event.button.id in self.project_names: project_name = event.button.id diff --git a/datashuttle/tui/screens/settings.py b/datashuttle/tui/screens/settings.py index 0634ea041..ace8a1d49 100644 --- a/datashuttle/tui/screens/settings.py +++ b/datashuttle/tui/screens/settings.py @@ -27,12 +27,14 @@ class SettingsScreen(ModalScreen): """ def __init__(self, mainwindow: App) -> None: + """PLACEHOLDER.""" super(SettingsScreen, self).__init__() self.mainwindow = mainwindow self.global_settings = self.mainwindow.load_global_settings() def compose(self) -> ComposeResult: + """PLACEHOLDER.""" dark_mode = self.global_settings["dark_mode"] yield Container( RadioSet( @@ -58,11 +60,12 @@ def compose(self) -> ComposeResult: ) def on_mount(self) -> None: - """""" + """PLACEHOLDER.""" id = "#show_transfer_tree_status_checkbox" self.query_one(id).tooltip = get_tooltip(id) def on_radio_set_changed(self, event: RadioSet.Changed) -> None: + """PLACEHOLDER.""" label = str(event.pressed.label) assert label in ["Light Mode", "Dark Mode"] @@ -73,9 +76,11 @@ def on_radio_set_changed(self, event: RadioSet.Changed) -> None: self.mainwindow.save_global_settings(self.global_settings) def on_checkbox_changed(self, event: Checkbox.Changed) -> None: + """PLACEHOLDER.""" self.global_settings["show_transfer_tree_status"] = event.value self.mainwindow.save_global_settings(self.global_settings) def on_button_pressed(self, event: Button.Pressed) -> None: + """PLACEHOLDER.""" if event.button.id == "all_main_menu_buttons": self.dismiss() diff --git a/datashuttle/tui/screens/setup_ssh.py b/datashuttle/tui/screens/setup_ssh.py index 7502caadb..bf9c84ca3 100644 --- a/datashuttle/tui/screens/setup_ssh.py +++ b/datashuttle/tui/screens/setup_ssh.py @@ -18,7 +18,7 @@ class SetupSshScreen(ModalScreen): - """This dialog windows handles the TUI equivalent of API's + """Dialog window that handles the TUI equivalent of API's setup_ssh_connection(). This asks to confirm the central hostkey, and takes password to setup SSH key pair. @@ -29,6 +29,7 @@ class SetupSshScreen(ModalScreen): """ def __init__(self, interface: Interface) -> None: + """PLACEHOLDER.""" super(SetupSshScreen, self).__init__() self.interface = interface @@ -38,6 +39,7 @@ def __init__(self, interface: Interface) -> None: self.key: paramiko.RSAKey def compose(self) -> ComposeResult: + """PLACEHOLDER.""" yield Container( Horizontal( Static( @@ -56,6 +58,7 @@ def compose(self) -> ComposeResult: ) def on_mount(self) -> None: + """PLACEHOLDER.""" self.query_one("#setup_ssh_password_input").visible = False def on_button_pressed(self, event: Button.pressed) -> None: diff --git a/datashuttle/tui/tabs/create_folders.py b/datashuttle/tui/tabs/create_folders.py index 7b98eb6db..73544767c 100644 --- a/datashuttle/tui/tabs/create_folders.py +++ b/datashuttle/tui/tabs/create_folders.py @@ -38,6 +38,7 @@ class CreateFoldersTab(TreeAndInputTab): """Create new project files formatted according to the NeuroBlueprint specification.""" def __init__(self, mainwindow: App, interface: Interface) -> None: + """PLACEHOLDER.""" super(CreateFoldersTab, self).__init__( "Create", id="tabscreen_create_tab" ) @@ -47,6 +48,7 @@ def __init__(self, mainwindow: App, interface: Interface) -> None: self.prev_click_time = 0.0 def compose(self) -> ComposeResult: + """PLACEHOLDER.""" yield CustomDirectoryTree( self.mainwindow, self.interface.get_configs()["local_path"], @@ -91,7 +93,7 @@ def compose(self) -> ComposeResult: ) def on_mount(self) -> None: - """""" + """PLACEHOLDER.""" if not self.interface: self.query_one("#configs_name_input").tooltip = get_tooltip( "#configs_name_input" @@ -130,6 +132,7 @@ def on_button_pressed(self, event: Button.Pressed) -> None: ) async def refresh_after_datatypes_changed(self, ignore): + """PLACEHOLDER.""" await self.recompose() self.on_mount() @@ -187,6 +190,7 @@ def fill_input_with_template(self, prefix: Prefix, input_id: str) -> None: input.value = fill_value def templates_on(self, prefix: Prefix) -> bool: + """PLACEHOLDER.""" return ( self.interface.get_name_templates()["on"] and self.interface.get_name_templates()[prefix] is not None @@ -251,7 +255,7 @@ def create_folders(self) -> None: self.mainwindow.show_modal_error_dialog(output) def reload_directorytree(self) -> None: - """This reloads the directorytree and also updates validation. + """Reloads the directorytree and also updates validation. Not now a good method name but done for consistency with other tab refresh methods. """ @@ -264,7 +268,7 @@ def reload_directorytree(self) -> None: def fill_input_with_next_sub_or_ses_template( self, prefix: Prefix, input_id: str ) -> None: - """This fills a sub / ses Input with a suggested name based on the + """Fills a sub / ses Input with a suggested name based on the next subject / session in the project (local). If `name_templates` are set, then the sub- or ses- first key diff --git a/datashuttle/tui/tabs/logging.py b/datashuttle/tui/tabs/logging.py index f35c7a987..8c4f5db92 100644 --- a/datashuttle/tui/tabs/logging.py +++ b/datashuttle/tui/tabs/logging.py @@ -13,29 +13,38 @@ class RichLogScreen(ModalScreen): + """PLACEHOLDER.""" + def __init__(self, log_file): + """PLACEHOLDER.""" super(RichLogScreen, self).__init__() with open(log_file) as file: self.log_contents = "".join(file.readlines()) def compose(self): + """PLACEHOLDER.""" yield Container( RichLog(highlight=True, markup=True, id="richlog_screen_rich_log"), Button("Close", id="richlog_screen_close_button"), ) def on_mount(self): + """PLACEHOLDER.""" text_log = self.query_one(RichLog) text_log.write(self.log_contents) def on_button_pressed(self, event): + """PLACEHOLDER.""" if event.button.id == "richlog_screen_close_button": self.dismiss() class LoggingTab(TabPane): + """PLACEHOLDER.""" + def __init__(self, title, mainwindow, project, id): + """PLACEHOLDER.""" super(LoggingTab, self).__init__(title=title, id=id) self.mainwindow = mainwindow @@ -48,6 +57,7 @@ def __init__(self, title, mainwindow, project, id): self.prev_click_time = 0 def update_latest_log_path(self): + """PLACEHOLDER.""" logs = list(self.project.get_logging_path().glob("*.log")) self.latest_log_path = ( max(logs, key=os.path.getctime) @@ -56,6 +66,7 @@ def update_latest_log_path(self): ) def compose(self): + """PLACEHOLDER.""" yield Container( Label( "Double click logging file to select:", @@ -83,6 +94,7 @@ def _on_mount(self, event: events.Mount) -> None: self.update_most_recent_label() def update_most_recent_label(self): + """PLACEHOLDER.""" self.update_latest_log_path() self.query_one("#logging_most_recent_label").update( f"or open most recent: {self.latest_log_path.stem}" @@ -90,11 +102,13 @@ def update_most_recent_label(self): self.refresh() def on_button_pressed(self, event): + """PLACEHOLDER.""" if event.button.id == "logging_tab_open_most_recent_button": self.push_rich_log_screen(self.latest_log_path) @require_double_click def on_directory_tree_file_selected(self, node): + """PLACEHOLDER.""" if not node.path.is_file(): self.mainwindow.show_modal_error_dialog( "Log file no longer exists. Refresh the directory tree" @@ -105,6 +119,7 @@ def on_directory_tree_file_selected(self, node): self.push_rich_log_screen(node.path) def push_rich_log_screen(self, log_path): + """PLACEHOLDER.""" self.mainwindow.push_screen( RichLogScreen( log_path, @@ -112,7 +127,9 @@ def push_rich_log_screen(self, log_path): ) def reload_directorytree(self): + """PLACEHOLDER.""" self.query_one("#logging_tab_custom_directory_tree").reload() def on_custom_directory_tree_directory_tree_special_key_press(self): + """PLACEHOLDER.""" self.reload_directorytree() diff --git a/datashuttle/tui/tabs/transfer.py b/datashuttle/tui/tabs/transfer.py index b50a14b08..9005fba06 100644 --- a/datashuttle/tui/tabs/transfer.py +++ b/datashuttle/tui/tabs/transfer.py @@ -43,7 +43,7 @@ class TransferTab(TreeAndInputTab): - """This tab handles the upload / download of files between local + """Handles the upload / download of files between local and central folders. It contains a TransferDirectoryTree that displays the transfer status of the files in the local folder, and calls underlying datashuttle transfer functions. @@ -83,6 +83,7 @@ def __init__( interface: Interface, id: Optional[str] = None, ) -> None: + """PLACEHOLDER.""" super(TransferTab, self).__init__(title, id=id) self.mainwindow = mainwindow self.interface = interface @@ -95,6 +96,7 @@ def __init__( # ---------------------------------------------------------------------------------- def compose(self) -> ComposeResult: + """PLACEHOLDER.""" self.transfer_all_widgets = [ Label( "All data from: \n\n - Rawdata \n - Derivatives \n\nwill be transferred.", @@ -208,6 +210,7 @@ def compose(self) -> ComposeResult: yield Label("â­• Legend", id="transfer_legend") def on_mount(self) -> None: + """PLACEHOLDER.""" for id in [ "#transfer_directorytree", "#transfer_switch_container", @@ -239,6 +242,7 @@ def on_mount(self) -> None: ) def on_select_changed(self, event: Select.Changed) -> None: + """PLACEHOLDER.""" if event.select.id == "transfer_tab_overwrite_select": assert event.select.value in ["Never", "Always", "If Source Newer"] format_select = event.select.value.lower().replace(" ", "_") @@ -248,6 +252,7 @@ def on_select_changed(self, event: Select.Changed) -> None: ) def on_checkbox_changed(self, event: Checkbox.Changed) -> None: + """PLACEHOLDER.""" if event.checkbox.id == "transfer_tab_dry_run_checkbox": self.interface.save_tui_settings( event.checkbox.value, @@ -317,6 +322,7 @@ def on_button_pressed(self, event: Button.Pressed) -> None: ) async def refresh_after_datatype_changed(self, ignore): + """PLACEHOLDER.""" await self.recompose() self.on_mount() self.query_one("#transfer_custom_radiobutton").value = True @@ -325,6 +331,7 @@ async def refresh_after_datatype_changed(self, ignore): def on_custom_directory_tree_directory_tree_special_key_press( self, event: CustomDirectoryTree.DirectoryTreeSpecialKeyPress ) -> None: + """PLACEHOLDER.""" if event.key == "ctrl+r": self.reload_directorytree() @@ -338,10 +345,11 @@ def on_custom_directory_tree_directory_tree_special_key_press( self.reload_directorytree() def reload_directorytree(self) -> None: + """PLACEHOLDER.""" self.query_one("#transfer_directorytree").update_transfer_tree() def update_directorytree_root(self, new_root_path: Path) -> None: - """This will automatically refresh the tree through the + """Automatically refreshes the tree through the reactive variable `path`. """ self.query_one("#transfer_directorytree").path = new_root_path @@ -351,7 +359,7 @@ def update_directorytree_root(self, new_root_path: Path) -> None: @work(exclusive=True, thread=True) def transfer_data(self) -> Worker[InterfaceOutput]: - """A threaded worker to transfer data + """A threaded worker to transfer data. This function transfers data based on the config provided by the radio buttons such as a) the data to be transferred (all / top-level-folders / custom) b) the diff --git a/datashuttle/tui/tabs/transfer_status_tree.py b/datashuttle/tui/tabs/transfer_status_tree.py index 1d4410fc7..373d1034d 100644 --- a/datashuttle/tui/tabs/transfer_status_tree.py +++ b/datashuttle/tui/tabs/transfer_status_tree.py @@ -38,6 +38,7 @@ class TransferStatusTree(CustomDirectoryTree): def __init__( self, mainwindow: App, interface: Interface, id: Optional[str] = None ): + """PLACEHOLDER.""" self.interface = interface self.local_path_str = self.interface.get_configs()[ "local_path" @@ -49,6 +50,7 @@ def __init__( ) def on_mount(self) -> None: + """PLACEHOLDER.""" self.update_transfer_tree(init=True) def update_transfer_tree(self, init: bool = False) -> None: diff --git a/datashuttle/tui/utils/tui_validators.py b/datashuttle/tui/utils/tui_validators.py index 1605c63ba..47e8dbd40 100644 --- a/datashuttle/tui/utils/tui_validators.py +++ b/datashuttle/tui/utils/tui_validators.py @@ -12,6 +12,8 @@ class NeuroBlueprintValidator(Validator): + """PLACEHOLDER.""" + def __init__(self, prefix: Prefix, parent: CreateFoldersTab) -> None: """Custom Validator() class that takes sub / ses prefix as input. Runs validation of From 95296d8dc1cefe7a36ebd38e1dbfcbd7452bf14d Mon Sep 17 00:00:00 2001 From: MoffittAndrew Date: Fri, 21 Mar 2025 15:26:07 +0000 Subject: [PATCH 15/70] fixed configs --- datashuttle/configs/canonical_configs.py | 5 ++- datashuttle/configs/canonical_folders.py | 9 ++--- datashuttle/configs/config_class.py | 45 +++++++++++++----------- datashuttle/configs/links.py | 4 +++ 4 files changed, 38 insertions(+), 25 deletions(-) diff --git a/datashuttle/configs/canonical_configs.py b/datashuttle/configs/canonical_configs.py index 8d03eed6e..5fab1aaa2 100644 --- a/datashuttle/configs/canonical_configs.py +++ b/datashuttle/configs/canonical_configs.py @@ -1,4 +1,4 @@ -"""This module contains all information for the required +"""Contains all information for the required format of the configs class. This is clearly defined as configs can be provided from file or input dynamically and so careful checks must be done. @@ -156,6 +156,7 @@ def raise_on_bad_local_only_project_configs(config_dict: Configs) -> None: def local_only_configs_are_none(config_dict: Configs) -> list[bool]: + """PLACEHOLDER.""" return [ config_dict[key] is None for key in ["central_path", "connection_method"] @@ -260,6 +261,7 @@ def get_tui_config_defaults() -> Dict: def get_name_templates_defaults() -> Dict: + """PLACEHOLDER.""" return {"name_templates": {"on": False, "sub": None, "ses": None}} @@ -288,6 +290,7 @@ def get_datatypes() -> List[str]: def get_broad_datatypes(): + """PLACEHOLDER.""" return ["ephys", "behav", "funcimg", "anat"] diff --git a/datashuttle/configs/canonical_folders.py b/datashuttle/configs/canonical_folders.py index 104019704..cc3db158d 100644 --- a/datashuttle/configs/canonical_folders.py +++ b/datashuttle/configs/canonical_folders.py @@ -11,7 +11,7 @@ def get_datatype_folders() -> dict: - """This function holds the canonical folders + """Holds the canonical folders managed by datashuttle. Notes @@ -41,7 +41,7 @@ def get_datatype_folders() -> dict: def get_non_sub_names() -> List[str]: """Get all arguments that are not allowed at the - subject level for data transfer, i.e. as sub_names + subject level for data transfer, i.e. as sub_names. """ return [ "all_ses", @@ -53,7 +53,7 @@ def get_non_sub_names() -> List[str]: def get_non_ses_names() -> List[str]: """Get all arguments that are not allowed at the - session level for data transfer, i.e. as ses_names + session level for data transfer, i.e. as ses_names. """ return [ "all_sub", @@ -65,12 +65,13 @@ def get_non_ses_names() -> List[str]: def canonical_reserved_keywords() -> List[str]: """Key keyword arguments that are passed to `sub_names` or - `ses_names` but that we + `ses_names`. """ return get_non_sub_names() + get_non_ses_names() def get_top_level_folders() -> List[TopLevelFolder]: + """PLACEHOLDER.""" return ["rawdata", "derivatives"] diff --git a/datashuttle/configs/config_class.py b/datashuttle/configs/config_class.py index b62427b7b..f01976fc1 100644 --- a/datashuttle/configs/config_class.py +++ b/datashuttle/configs/config_class.py @@ -30,26 +30,25 @@ class Configs(UserDict): The configs must match exactly the standard set in canonical_configs.py. If updating these configs, this should be done through changing canonical_configs.py - - The input dict is checked that it conforms to the - canonical standard by calling check_dict_values_raise_on_fail() - - project_name and all paths are set at runtime but not stored. - - Parameters - ---------- - file_path - full filepath to save the config .yaml file to. - - input_dict - a dict of config key-value pairs to input dict. - This must contain all canonical_config keys - """ def __init__( self, project_name: str, file_path: Path, input_dict: Union[dict, None] ) -> None: + """Parameters + ---------- + file_path + full filepath to save the config .yaml file to. + + input_dict + a dict of config key-value pairs to input dict. + This must contain all canonical_config keys + + The input dict is checked that it conforms to the + canonical standard by calling check_dict_values_raise_on_fail() + + project_name and all paths are set at runtime but not stored. + """ super(Configs, self).__init__(input_dict) self.project_name = project_name @@ -61,12 +60,13 @@ def __init__( self.project_metadata_path: Path def setup_after_load(self) -> None: + """PLACEHOLDER.""" load_configs.convert_str_and_pathlib_paths(self, "str_to_path") self.ensure_local_and_central_path_end_in_project_name() self.check_dict_values_raise_on_fail() def ensure_local_and_central_path_end_in_project_name(self): - """""" + """PLACEHOLDER.""" for path_type in ["local_path", "central_path"]: if path_type == "central_path" and self[path_type] is None: continue @@ -89,12 +89,15 @@ def check_dict_values_raise_on_fail(self) -> None: canonical_configs.check_dict_values_raise_on_fail(self) def keys(self) -> KeysView: + """D.keys() -> a set-like object providing a view on D's keys.""" return self.data.keys() def items(self) -> ItemsView: + """D.items() -> a set-like object providing a view on D's items.""" return self.data.items() def values(self) -> ValuesView: + """D.values() -> a set-like object providing a view on D's values.""" return self.data.values() # ------------------------------------------------------------------------- @@ -110,9 +113,11 @@ def dump_to_file(self) -> None: yaml.dump(cfg_to_save, config_file, sort_keys=False) def load_from_file(self) -> None: - """Load a config dict saved at .yaml file. Note this will + """Load a config dict saved at .yaml file. + + Note this will not automatically check the configs are valid, this - requires calling self.check_dict_values_raise_on_fail() + requires calling self.check_dict_values_raise_on_fail(). """ with open(self.file_path) as config_file: config_dict = yaml.full_load(config_file) @@ -203,7 +208,7 @@ def get_rclone_config_name( def make_rclone_transfer_options( self, overwrite_existing_files: OverwriteExistingFiles, dry_run: bool ) -> Dict: - """This function originally collected the relevant arguments + """Originally collected the relevant arguments from configs. Now, all are passed via function arguments However, now we fix the previously configurable arguments `show_transfer_progress` and `dry_run` here. @@ -226,7 +231,7 @@ def make_rclone_transfer_options( } def init_paths(self) -> None: - """""" + """PLACEHOLDER.""" self.project_metadata_path = self["local_path"] / ".datashuttle" datashuttle_path, _ = canonical_folders.get_project_datashuttle_path( diff --git a/datashuttle/configs/links.py b/datashuttle/configs/links.py index 0c354bc1a..e61ade29a 100644 --- a/datashuttle/configs/links.py +++ b/datashuttle/configs/links.py @@ -1,14 +1,18 @@ def get_docs_link(): + """PLACEHOLDER.""" return "https://datashuttle.neuroinformatics.dev/" def get_github_link(): + """PLACEHOLDER.""" return "https://github.com/neuroinformatics-unit/datashuttle" def get_link_github_issues(): + """PLACEHOLDER.""" return "https://github.com/neuroinformatics-unit/datashuttle/issues" def get_link_zulip(): + """PLACEHOLDER.""" return "https://neuroinformatics.zulipchat.com/#narrow/stream/405999-DataShuttle" From 2187d17ef4d16637e5f8ff8de27f0f32eb75761e Mon Sep 17 00:00:00 2001 From: MoffittAndrew Date: Fri, 21 Mar 2025 15:36:45 +0000 Subject: [PATCH 16/70] run pre-commit --- .pre-commit-config.yaml | 4 ++-- datashuttle/configs/config_class.py | 8 ++++---- datashuttle/datashuttle_class.py | 8 ++++---- datashuttle/datashuttle_functions.py | 4 ++-- datashuttle/tui/configs.py | 2 +- datashuttle/tui/custom_widgets.py | 4 ++-- datashuttle/tui/tabs/logging.py | 4 ++-- datashuttle/tui/utils/tui_validators.py | 2 +- datashuttle/utils/custom_exceptions.py | 4 ++-- datashuttle/utils/data_transfer.py | 2 +- datashuttle/utils/folders.py | 12 ++++++------ datashuttle/utils/ssh.py | 2 +- pyproject.toml | 2 +- .../{test_configs.py => _test_configs.py} | 0 tests/tests_integration/base.py | 2 +- tests/tests_integration/test_create_folders.py | 2 +- tests/tests_integration/test_filesystem_transfer.py | 2 +- tests/tests_integration/test_formatting.py | 2 +- tests/tests_integration/test_local_only_mode.py | 2 +- tests/tests_integration/test_logging.py | 2 +- tests/tests_integration/test_settings.py | 2 +- tests/tests_integration/test_ssh_file_transfer.py | 2 +- tests/tests_integration/test_transfer_checks.py | 2 +- tests/tests_integration/test_validation.py | 2 +- tests/tests_tui/test_local_only_project.py | 2 +- tests/tests_tui/test_tui_configs.py | 2 +- tests/tests_tui/test_tui_create_folders.py | 2 +- tests/tests_tui/test_tui_logging.py | 2 +- 28 files changed, 43 insertions(+), 43 deletions(-) rename tests/tests_integration/{test_configs.py => _test_configs.py} (100%) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 299fba1d1..1dbbe5cba 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -20,8 +20,8 @@ repos: - id: end-of-file-fixer - id: mixed-line-ending args: [--fix=lf] - - id: name-tests-test - args: ["--pytest-test-first"] + #- id: name-tests-test + # args: ["--pytest-test-first"] - id: requirements-txt-fixer - id: trailing-whitespace - repo: https://github.com/pre-commit/pygrep-hooks diff --git a/datashuttle/configs/config_class.py b/datashuttle/configs/config_class.py index f01976fc1..c7b71b8cc 100644 --- a/datashuttle/configs/config_class.py +++ b/datashuttle/configs/config_class.py @@ -43,7 +43,7 @@ def __init__( input_dict a dict of config key-value pairs to input dict. This must contain all canonical_config keys - + The input dict is checked that it conforms to the canonical standard by calling check_dict_values_raise_on_fail() @@ -114,7 +114,7 @@ def dump_to_file(self) -> None: def load_from_file(self) -> None: """Load a config dict saved at .yaml file. - + Note this will not automatically check the configs are valid, this requires calling self.check_dict_values_raise_on_fail(). @@ -149,7 +149,7 @@ def build_project_path( a list (or string for 1) of folder names to be joined into a path. If file included, must be last entry (with ext). - + top_level_folder either "rawdata" or "derivatives" @@ -181,7 +181,7 @@ def get_base_folder( ---------- base base path, "local", "central" or "datashuttle" - + top_level_folder either "rawdata" or "derivatives" diff --git a/datashuttle/datashuttle_class.py b/datashuttle/datashuttle_class.py index 5adec00b9..96b344536 100644 --- a/datashuttle/datashuttle_class.py +++ b/datashuttle/datashuttle_class.py @@ -1053,7 +1053,7 @@ def get_next_sub( ---------- top_level_folder "rawdata" or "derivatives" - + return_with_prefix If `True`, return with the "sub-" prefix. @@ -1297,10 +1297,10 @@ def _transfer_entire_project( upload_or_download direction to transfer the data, either "upload" (from local to central) or "download" (from central to local). - + overwrite_existing_files determines whether or not to overwrite existing files - + dry_run perform a dry-run of transfer. This will output as if file transfer was taking place, but no files will be moved. Useful @@ -1341,7 +1341,7 @@ def _start_log( store_in_temp_folder if `False`, existing logging path will be used (local project .datashuttle). - + verbose print warnings and error messages. diff --git a/datashuttle/datashuttle_functions.py b/datashuttle/datashuttle_functions.py index 5ab642570..b474859b2 100644 --- a/datashuttle/datashuttle_functions.py +++ b/datashuttle/datashuttle_functions.py @@ -29,7 +29,7 @@ def quick_validate_project( name_templates: Optional[Dict] = None, ) -> List[str]: """Perform validation on the project. - + This checks the subject and session level folders to ensure there are not NeuroBlueprint formatting issues. @@ -93,7 +93,7 @@ def _format_top_level_folder( top_level_folder: TopLevelFolder | None, ) -> List[TopLevelFolder]: """Format the top level folder. - + Take a `top_level_folder` ("rawdata" or "derivatives" str) and convert to list, if `None`, convert it to a list of both possible top-level folders. diff --git a/datashuttle/tui/configs.py b/datashuttle/tui/configs.py index 0d50f8712..e7f7b044a 100644 --- a/datashuttle/tui/configs.py +++ b/datashuttle/tui/configs.py @@ -45,7 +45,7 @@ class ConfigsContent(Container): @dataclass class ConfigsSaved(Message): """PLACEHOLDER.""" - + pass def __init__( diff --git a/datashuttle/tui/custom_widgets.py b/datashuttle/tui/custom_widgets.py index 77178842b..9a303b0da 100644 --- a/datashuttle/tui/custom_widgets.py +++ b/datashuttle/tui/custom_widgets.py @@ -46,7 +46,7 @@ class ClickableInput(Input): @dataclass class Clicked(Message): """PLACEHOLDER.""" - + input: ClickableInput ctrl: bool @@ -98,7 +98,7 @@ class CustomDirectoryTree(DirectoryTree): @dataclass class DirectoryTreeSpecialKeyPress(Message): """PLACEHOLDER.""" - + key: str node_path: Optional[Path] diff --git a/datashuttle/tui/tabs/logging.py b/datashuttle/tui/tabs/logging.py index 8c4f5db92..21b2ddc4e 100644 --- a/datashuttle/tui/tabs/logging.py +++ b/datashuttle/tui/tabs/logging.py @@ -14,7 +14,7 @@ class RichLogScreen(ModalScreen): """PLACEHOLDER.""" - + def __init__(self, log_file): """PLACEHOLDER.""" super(RichLogScreen, self).__init__() @@ -42,7 +42,7 @@ def on_button_pressed(self, event): class LoggingTab(TabPane): """PLACEHOLDER.""" - + def __init__(self, title, mainwindow, project, id): """PLACEHOLDER.""" super(LoggingTab, self).__init__(title=title, id=id) diff --git a/datashuttle/tui/utils/tui_validators.py b/datashuttle/tui/utils/tui_validators.py index 47e8dbd40..4124e4f44 100644 --- a/datashuttle/tui/utils/tui_validators.py +++ b/datashuttle/tui/utils/tui_validators.py @@ -13,7 +13,7 @@ class NeuroBlueprintValidator(Validator): """PLACEHOLDER.""" - + def __init__(self, prefix: Prefix, parent: CreateFoldersTab) -> None: """Custom Validator() class that takes sub / ses prefix as input. Runs validation of diff --git a/datashuttle/utils/custom_exceptions.py b/datashuttle/utils/custom_exceptions.py index 3fe7e3368..47480d57b 100644 --- a/datashuttle/utils/custom_exceptions.py +++ b/datashuttle/utils/custom_exceptions.py @@ -1,10 +1,10 @@ class ConfigError(Exception): """PLACEHOLDER.""" - + pass class NeuroBlueprintError(Exception): """PLACEHOLDER.""" - + pass diff --git a/datashuttle/utils/data_transfer.py b/datashuttle/utils/data_transfer.py index bfd092bc0..4351834c1 100644 --- a/datashuttle/utils/data_transfer.py +++ b/datashuttle/utils/data_transfer.py @@ -112,7 +112,7 @@ def __init__( def build_a_list_of_all_files_and_folders_to_transfer(self) -> List[str]: """Build a list of every file to transfer based on the user-passed arguments. - + This cycles through every subject, session and datatype and adds the outputs to three lists: diff --git a/datashuttle/utils/folders.py b/datashuttle/utils/folders.py index 3a3dd3372..fb0e3e0f1 100644 --- a/datashuttle/utils/folders.py +++ b/datashuttle/utils/folders.py @@ -49,10 +49,10 @@ def create_folder_trees( ---------- cfg datashuttle config UserDict - + top_level_folder either "rawdata" or "derivatives" - + sub_names, ses_names, datatype see create_folders() @@ -391,7 +391,7 @@ def search_for_wildcards( ---------- cfg datashuttle configs - + project initialised datashuttle project @@ -466,7 +466,7 @@ def search_sub_or_ses_level( arguments, but this is not nice and breaks the general rule that these functions should operate project-agnostic. - + base_folder the path to the base folder. If sub is None, the search is performed on this folder @@ -490,7 +490,7 @@ def search_sub_or_ses_level( verbose If `True`, if a search folder cannot be found, a message will be printed with the un-found path. - + return_full_path include the search_path in the returned paths @@ -534,7 +534,7 @@ def search_for_folders( ---------- cfg datashuttle configs - + local_or_central "local" or "central" diff --git a/datashuttle/utils/ssh.py b/datashuttle/utils/ssh.py index 9b80f7d4c..568773438 100644 --- a/datashuttle/utils/ssh.py +++ b/datashuttle/utils/ssh.py @@ -314,7 +314,7 @@ def get_list_of_folder_names_over_sftp( verbose If `True`, if a search folder cannot be found, a message will be printed with the un-found path. - + return_full_path include the search_path in the returned paths diff --git a/pyproject.toml b/pyproject.toml index 63d9cd3c8..101f9e0e1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -141,7 +141,7 @@ select = [ "I", # isort "E", # pycodestyle errors "F", # Pyflakes - "TC", # flake8-type-checking + "TCH", # flake8-type-checking "TID252", # flake8-tidy-imports relative-imports "D", # pydocstyle ] diff --git a/tests/tests_integration/test_configs.py b/tests/tests_integration/_test_configs.py similarity index 100% rename from tests/tests_integration/test_configs.py rename to tests/tests_integration/_test_configs.py diff --git a/tests/tests_integration/base.py b/tests/tests_integration/base.py index 75c01adf2..b28f8cec4 100644 --- a/tests/tests_integration/base.py +++ b/tests/tests_integration/base.py @@ -11,7 +11,7 @@ class BaseTest: """PLACEHOLDER.""" - + @pytest.fixture(scope="function") def no_cfg_project(test): """Fixture that creates an empty project. Ignore the warning diff --git a/tests/tests_integration/test_create_folders.py b/tests/tests_integration/test_create_folders.py index af9f7e6d2..e82815c9f 100644 --- a/tests/tests_integration/test_create_folders.py +++ b/tests/tests_integration/test_create_folders.py @@ -14,7 +14,7 @@ class TestCreateFolders(BaseTest): """PLACEHOLDER.""" - + @pytest.mark.parametrize("project", ["local", "full"], indirect=True) def test_generate_folders_default_ses(self, project): """Make a subject folders with full tree. Don't specify diff --git a/tests/tests_integration/test_filesystem_transfer.py b/tests/tests_integration/test_filesystem_transfer.py index d35b0e84e..d06ddc4d6 100644 --- a/tests/tests_integration/test_filesystem_transfer.py +++ b/tests/tests_integration/test_filesystem_transfer.py @@ -14,7 +14,7 @@ class TestFileTransfer(BaseTest): """PLACEHOLDER.""" - + @pytest.mark.parametrize( "top_level_folder", canonical_folders.get_top_level_folders() ) diff --git a/tests/tests_integration/test_formatting.py b/tests/tests_integration/test_formatting.py index 005b7cbf7..cf5dffafe 100644 --- a/tests/tests_integration/test_formatting.py +++ b/tests/tests_integration/test_formatting.py @@ -7,7 +7,7 @@ class TestFormatting(BaseTest): """PLACEHOLDER.""" - + @pytest.mark.parametrize("prefix", ["sub", "ses"]) @pytest.mark.parametrize( "input", [1, {"test": "one"}, 1.0, ["1", "2", ["three"]]] diff --git a/tests/tests_integration/test_local_only_mode.py b/tests/tests_integration/test_local_only_mode.py index bcc42394b..0edc28b6f 100644 --- a/tests/tests_integration/test_local_only_mode.py +++ b/tests/tests_integration/test_local_only_mode.py @@ -14,7 +14,7 @@ class TestLocalOnlyProject(BaseTest): """PLACEHOLDER.""" - + def test_bad_setup(self, tmp_path): """Test setup without providing both central_path and connection method (distinguishing a full vs local-only project). diff --git a/tests/tests_integration/test_logging.py b/tests/tests_integration/test_logging.py index ecf68b49e..e5196c283 100644 --- a/tests/tests_integration/test_logging.py +++ b/tests/tests_integration/test_logging.py @@ -19,7 +19,7 @@ class TestLogging: """PLACEHOLDER.""" - + @pytest.fixture(scope="function") def teardown_logger(self): """Ensure the logger is deleted at the end of each test.""" diff --git a/tests/tests_integration/test_settings.py b/tests/tests_integration/test_settings.py index 27f9eb7b6..570f26dfb 100644 --- a/tests/tests_integration/test_settings.py +++ b/tests/tests_integration/test_settings.py @@ -12,7 +12,7 @@ class TestPersistentSettings(BaseTest): """PLACEHOLDER.""" - + @pytest.mark.parametrize("project", ["local", "full"], indirect=True) def test_persistent_settings_name_templates(self, project): """Test the 'name_templates' option that is stored in persistent diff --git a/tests/tests_integration/test_ssh_file_transfer.py b/tests/tests_integration/test_ssh_file_transfer.py index 0f6f46bcd..aa8a8d20c 100644 --- a/tests/tests_integration/test_ssh_file_transfer.py +++ b/tests/tests_integration/test_ssh_file_transfer.py @@ -14,7 +14,7 @@ class TestFileTransfer: """PLACEHOLDER.""" - + @pytest.fixture( scope="class", params=[ # Set running SSH or local filesystem (see docstring). diff --git a/tests/tests_integration/test_transfer_checks.py b/tests/tests_integration/test_transfer_checks.py index e443cfae8..145ddc0f6 100644 --- a/tests/tests_integration/test_transfer_checks.py +++ b/tests/tests_integration/test_transfer_checks.py @@ -11,7 +11,7 @@ class TestTransferChecks(BaseTest): """PLACEHOLDER.""" - + @pytest.mark.parametrize( "top_level_folders", [["rawdata", "derivatives"], ["rawdata"], ["derivatives"]], diff --git a/tests/tests_integration/test_validation.py b/tests/tests_integration/test_validation.py index 4827152df..d46fa321d 100644 --- a/tests/tests_integration/test_validation.py +++ b/tests/tests_integration/test_validation.py @@ -15,7 +15,7 @@ class TestValidation(BaseTest): """PLACEHOLDER.""" - + @pytest.mark.parametrize( "sub_name", ["sub-001", "sub-999_@DATE@", "sub-001_random-tag_another-tag"], diff --git a/tests/tests_tui/test_local_only_project.py b/tests/tests_tui/test_local_only_project.py index 8802aee8c..0ac50e587 100644 --- a/tests/tests_tui/test_local_only_project.py +++ b/tests/tests_tui/test_local_only_project.py @@ -6,7 +6,7 @@ class TestTuiLocalOnlyProject(TuiBase): """PLACEHOLDER.""" - + @pytest.mark.asyncio async def test_local_only_make_project( self, diff --git a/tests/tests_tui/test_tui_configs.py b/tests/tests_tui/test_tui_configs.py index 1fa7802ad..8e27a6701 100644 --- a/tests/tests_tui/test_tui_configs.py +++ b/tests/tests_tui/test_tui_configs.py @@ -15,7 +15,7 @@ class TestTuiConfigs(TuiBase): """PLACEHOLDER.""" - + # ------------------------------------------------------------------------- # Test New Project Configs # ------------------------------------------------------------------------- diff --git a/tests/tests_tui/test_tui_create_folders.py b/tests/tests_tui/test_tui_create_folders.py index 736f4c63d..d05615439 100644 --- a/tests/tests_tui/test_tui_create_folders.py +++ b/tests/tests_tui/test_tui_create_folders.py @@ -14,7 +14,7 @@ class TestTuiCreateFolders(TuiBase): """PLACEHOLDER.""" - + # ------------------------------------------------------------------------- # General test Create Folders # ------------------------------------------------------------------------- diff --git a/tests/tests_tui/test_tui_logging.py b/tests/tests_tui/test_tui_logging.py index 21e288a12..adce58e7c 100644 --- a/tests/tests_tui/test_tui_logging.py +++ b/tests/tests_tui/test_tui_logging.py @@ -8,7 +8,7 @@ class TestTuiLogging(TuiBase): """PLACEHOLDER.""" - + @pytest.mark.asyncio async def test_logging(self, setup_project_paths): """Test logging by running some commands, checking they From e0c4130a5568ccc6c70f86698318b5c493433882 Mon Sep 17 00:00:00 2001 From: MoffittAndrew Date: Fri, 21 Mar 2025 15:40:32 +0000 Subject: [PATCH 17/70] renaming TCH to TC --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 101f9e0e1..63d9cd3c8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -141,7 +141,7 @@ select = [ "I", # isort "E", # pycodestyle errors "F", # Pyflakes - "TCH", # flake8-type-checking + "TC", # flake8-type-checking "TID252", # flake8-tidy-imports relative-imports "D", # pydocstyle ] From 26ddc29a5ad863abf226e1b46f99e76ca11af43e Mon Sep 17 00:00:00 2001 From: MoffittAndrew Date: Fri, 21 Mar 2025 15:42:50 +0000 Subject: [PATCH 18/70] moving Iterable import into type checking block to comply with TC003 --- datashuttle/tui/custom_widgets.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/datashuttle/tui/custom_widgets.py b/datashuttle/tui/custom_widgets.py index 9a303b0da..01f73e9e0 100644 --- a/datashuttle/tui/custom_widgets.py +++ b/datashuttle/tui/custom_widgets.py @@ -1,6 +1,5 @@ from __future__ import annotations -from collections.abc import Iterable from typing import ( TYPE_CHECKING, List, @@ -10,6 +9,8 @@ ) if TYPE_CHECKING: + from collections.abc import Iterable + from textual import events from textual.validation import Validator From c82608cca957fe1e387619191ae18913b612b995 Mon Sep 17 00:00:00 2001 From: MoffittAndrew Date: Fri, 21 Mar 2025 15:47:09 +0000 Subject: [PATCH 19/70] re-arranging rules --- datashuttle/tui/custom_widgets.py | 2 +- pyproject.toml | 46 +++++++++---------------------- 2 files changed, 14 insertions(+), 34 deletions(-) diff --git a/datashuttle/tui/custom_widgets.py b/datashuttle/tui/custom_widgets.py index 01f73e9e0..5118b1f5b 100644 --- a/datashuttle/tui/custom_widgets.py +++ b/datashuttle/tui/custom_widgets.py @@ -10,7 +10,7 @@ if TYPE_CHECKING: from collections.abc import Iterable - + from textual import events from textual.validation import Validator diff --git a/pyproject.toml b/pyproject.toml index 63d9cd3c8..3483bdda2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -97,39 +97,6 @@ fix = true [tool.ruff.lint] # See https://docs.astral.sh/ruff/rules/ -#ignore = [ -# "D203", # one blank line before class -# "D213", # multi-line-summary second line -# "D401", # first line of docstrings should be in an imperative mood -# "E501", # limit lines to 79 characters -#] -#select = [ -# "E", # pycodestyle errors -# "F", # Pyflakes -# "UP", # pyupgrade -# "I", # isort -# "B", # flake8 bugbear -# "SIM", # flake8 simplify -# "C90", # McCabe complexity -# "D", # pydocstyle -#] -per-file-ignores = { "tests/*" = [ - "D100", # missing docstring in public module - "D205", # missing blank line between summary and description - "D103", # missing docstring in public function -], "examples/*" = [ - "D400", # first line should end with a period. - "D415", # first line should end with a period, question mark... - "D205", # missing blank line between summary and description -], "__init__.py" = [ - # This was part of the old config - # Is this needed? __init__.py is already part of tool.ruff.exclude - "F401", # auto remove unused imports -] } - -# Old ruff ruleset + pydocstyle added -# Inconsistent with movement repo, but saving this here for -# now in case there are good reasons to keep these rules ignore = ["E203","E501","E731","C901","W291","W293","E402","E722", "D100", # missing docstring in public module (not enforced FOR NOW) "D203", # one blank line before class @@ -145,6 +112,19 @@ select = [ "TID252", # flake8-tidy-imports relative-imports "D", # pydocstyle ] +per-file-ignores = { "tests/*" = [ + "D100", # missing docstring in public module + "D205", # missing blank line between summary and description + "D103", # missing docstring in public function +], "examples/*" = [ + "D400", # first line should end with a period. + "D415", # first line should end with a period, question mark... + "D205", # missing blank line between summary and description +], "__init__.py" = [ + # This was part of the old config + # Is this needed? __init__.py is already part of tool.ruff.exclude + "F401", # auto remove unused imports +] } [tool.ruff.format] docstring-code-format = true # Also format code in docstrings From c62c887f9f079dbf1c9509c89140c47383398ac4 Mon Sep 17 00:00:00 2001 From: MoffittAndrew Date: Fri, 21 Mar 2025 15:51:27 +0000 Subject: [PATCH 20/70] Adding informative comment about ruff config --- pyproject.toml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 3483bdda2..ee024ea62 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -95,6 +95,13 @@ line-length = 79 exclude = ["__init__.py","build",".eggs"] fix = true + + +# Ruff config is not exactly the same as in the movement repo, as +# currently we are only adding linting enforcement to docstrings. +# Other ruff rules that are also present in movement repo can be +# added here in a separate PR after these changes have been merged. + [tool.ruff.lint] # See https://docs.astral.sh/ruff/rules/ ignore = ["E203","E501","E731","C901","W291","W293","E402","E722", From 4f37efcdb7dff9d47d664c510764178881523a64 Mon Sep 17 00:00:00 2001 From: MoffittAndrew Date: Sun, 23 Mar 2025 14:25:55 +0000 Subject: [PATCH 21/70] Filling in PLACEHOLDER dosctrings in configs and utils --- datashuttle/configs/canonical_configs.py | 8 +++-- datashuttle/configs/canonical_folders.py | 10 ++++-- datashuttle/configs/config_class.py | 10 +++--- datashuttle/configs/links.py | 8 ++--- datashuttle/datashuttle_class.py | 41 ++++++++++++------------ datashuttle/utils/custom_exceptions.py | 4 +-- datashuttle/utils/data_transfer.py | 2 +- datashuttle/utils/ds_logger.py | 6 ++-- datashuttle/utils/folder_class.py | 11 ++++++- datashuttle/utils/formatting.py | 6 ++-- datashuttle/utils/rclone.py | 2 +- datashuttle/utils/utils.py | 4 +-- datashuttle/utils/validation.py | 2 +- 13 files changed, 65 insertions(+), 49 deletions(-) diff --git a/datashuttle/configs/canonical_configs.py b/datashuttle/configs/canonical_configs.py index 5fab1aaa2..61bb2d5e8 100644 --- a/datashuttle/configs/canonical_configs.py +++ b/datashuttle/configs/canonical_configs.py @@ -156,7 +156,9 @@ def raise_on_bad_local_only_project_configs(config_dict: Configs) -> None: def local_only_configs_are_none(config_dict: Configs) -> list[bool]: - """PLACEHOLDER.""" + """Check if the central_path and connection_method config options + are set to `None`. + """ return [ config_dict[key] is None for key in ["central_path", "connection_method"] @@ -261,7 +263,7 @@ def get_tui_config_defaults() -> Dict: def get_name_templates_defaults() -> Dict: - """PLACEHOLDER.""" + """Get the default values for name_templates.""" return {"name_templates": {"on": False, "sub": None, "ses": None}} @@ -290,7 +292,7 @@ def get_datatypes() -> List[str]: def get_broad_datatypes(): - """PLACEHOLDER.""" + """Return a list of broad datatypes.""" return ["ephys", "behav", "funcimg", "anat"] diff --git a/datashuttle/configs/canonical_folders.py b/datashuttle/configs/canonical_folders.py index cc3db158d..c2c90065c 100644 --- a/datashuttle/configs/canonical_folders.py +++ b/datashuttle/configs/canonical_folders.py @@ -25,12 +25,16 @@ def get_datatype_folders() -> dict: The value is a Folder() class instance with the required fields - name : The display name for the datatype, that will + Parameters + ---------- + name + The display name for the datatype, that will be used for making and transferring files in practice. This should always match the canonical name, but left as an option for rare cases in which advanced users want to change it. - level : "sub" or "ses", level to make the folder at. + level + "sub" or "ses", level to make the folder at. """ return { @@ -71,7 +75,7 @@ def canonical_reserved_keywords() -> List[str]: def get_top_level_folders() -> List[TopLevelFolder]: - """PLACEHOLDER.""" + """Return a list of the different top level folder names.""" return ["rawdata", "derivatives"] diff --git a/datashuttle/configs/config_class.py b/datashuttle/configs/config_class.py index c7b71b8cc..072297154 100644 --- a/datashuttle/configs/config_class.py +++ b/datashuttle/configs/config_class.py @@ -60,13 +60,15 @@ def __init__( self.project_metadata_path: Path def setup_after_load(self) -> None: - """PLACEHOLDER.""" + """Setup the config after loading it.""" load_configs.convert_str_and_pathlib_paths(self, "str_to_path") self.ensure_local_and_central_path_end_in_project_name() self.check_dict_values_raise_on_fail() def ensure_local_and_central_path_end_in_project_name(self): - """PLACEHOLDER.""" + """Ensure that the local and central path end in the name of + the project. + """ for path_type in ["local_path", "central_path"]: if path_type == "central_path" and self[path_type] is None: continue @@ -157,7 +159,7 @@ def build_project_path( if isinstance(sub_folders, list): sub_folders_str = "/".join(sub_folders) else: - sub_folders_str = cast(str, sub_folders) + sub_folders_str = cast("str", sub_folders) sub_folders_path = Path(sub_folders_str) @@ -231,7 +233,7 @@ def make_rclone_transfer_options( } def init_paths(self) -> None: - """PLACEHOLDER.""" + """Initiate the datashuttle paths.""" self.project_metadata_path = self["local_path"] / ".datashuttle" datashuttle_path, _ = canonical_folders.get_project_datashuttle_path( diff --git a/datashuttle/configs/links.py b/datashuttle/configs/links.py index e61ade29a..2daa03013 100644 --- a/datashuttle/configs/links.py +++ b/datashuttle/configs/links.py @@ -1,18 +1,18 @@ def get_docs_link(): - """PLACEHOLDER.""" + """Return the link to the datashuttle page.""" return "https://datashuttle.neuroinformatics.dev/" def get_github_link(): - """PLACEHOLDER.""" + """Return the link to the datashuttle repository.""" return "https://github.com/neuroinformatics-unit/datashuttle" def get_link_github_issues(): - """PLACEHOLDER.""" + """Return the link to the datashuttle repository issues page.""" return "https://github.com/neuroinformatics-unit/datashuttle/issues" def get_link_zulip(): - """PLACEHOLDER.""" + """Return the link to the datashuttle Zulip chatroom.""" return "https://neuroinformatics.zulipchat.com/#narrow/stream/405999-DataShuttle" diff --git a/datashuttle/datashuttle_class.py b/datashuttle/datashuttle_class.py index 96b344536..a07462249 100644 --- a/datashuttle/datashuttle_class.py +++ b/datashuttle/datashuttle_class.py @@ -85,27 +85,26 @@ class DataShuttle: with SSH, use setup setup_ssh_connection(). This will allow you to check the server key, add host key to profile if accepted, and setup ssh key pair. - - Parameters - ---------- - project_name - The project name to use the datashuttle - Folders containing all project files - and folders are specified in make_config_file(). - Datashuttle-related files are stored in - a .datashuttle folder in the user home - folder. Use get_datashuttle_path() to - see the path to this folder. - - print_startup_message - If `True`, a start-up message displaying the - current state of the program (e.g. persistent - settings such as the 'top-level folder') is shown. - """ def __init__(self, project_name: str, print_startup_message: bool = True): - """PLACEHOLDER.""" + """Parameters + ---------- + project_name + The project name to use the datashuttle + Folders containing all project files + and folders are specified in make_config_file(). + Datashuttle-related files are stored in + a .datashuttle folder in the user home + folder. Use get_datashuttle_path() to + see the path to this folder. + + print_startup_message + If `True`, a start-up message displaying the + current state of the program (e.g. persistent + settings such as the 'top-level folder') is shown. + + """ self._error_on_base_project_name(project_name) self.project_name = project_name ( @@ -972,7 +971,7 @@ def make_config_file( ds_logger.close_log_filehandler() def update_config_file(self, **kwargs) -> None: - """PLACEHOLDER.""" + """Update the configuration file.""" if not self.cfg: utils.log_and_raise_error( "Must have a config loaded before updating configs.", @@ -1023,7 +1022,7 @@ def get_config_path(self) -> Path: @check_configs_set def get_configs(self) -> Configs: - """PLACEHOLDER.""" + """Get the datashuttle configs.""" return self.cfg @check_configs_set @@ -1388,7 +1387,7 @@ def _move_logs_from_temp_folder(self) -> None: ) def _clear_temp_log_path(self) -> None: - """PLACEHOLDER.""" + """Delete temporary log files.""" log_files = glob.glob(str(self._temp_log_path / "*.log")) for file in log_files: os.remove(file) diff --git a/datashuttle/utils/custom_exceptions.py b/datashuttle/utils/custom_exceptions.py index 47480d57b..05be4b6d0 100644 --- a/datashuttle/utils/custom_exceptions.py +++ b/datashuttle/utils/custom_exceptions.py @@ -1,10 +1,10 @@ class ConfigError(Exception): - """PLACEHOLDER.""" + """Raise an error relating to a configuration problem.""" pass class NeuroBlueprintError(Exception): - """PLACEHOLDER.""" + """Raise an error when something doesn't conform to the NeuroBlueprint pattern.""" pass diff --git a/datashuttle/utils/data_transfer.py b/datashuttle/utils/data_transfer.py index 4351834c1..e43fd4540 100644 --- a/datashuttle/utils/data_transfer.py +++ b/datashuttle/utils/data_transfer.py @@ -350,7 +350,7 @@ def update_list_with_dtype_paths( # ------------------------------------------------------------------------- def to_list(self, names: Union[str, List[str]]) -> List[str]: - """PLACEHOLDER.""" + """Convert a name or list of names to a list.""" if isinstance(names, str): names = [names] return names diff --git a/datashuttle/utils/ds_logger.py b/datashuttle/utils/ds_logger.py index 4269b391d..140ab7c54 100644 --- a/datashuttle/utils/ds_logger.py +++ b/datashuttle/utils/ds_logger.py @@ -18,17 +18,17 @@ def get_logger_name(): - """PLACEHOLDER.""" + """Return the name of the logger.""" return "datashuttle" def get_logger(): - """PLACEHOLDER.""" + """Return the instance of the logger object.""" return logging.getLogger(get_logger_name()) def logging_is_active(): - """PLACEHOLDER.""" + """Check if the logger is active.""" logger_exists = get_logger_name() in logging.root.manager.loggerDict if logger_exists and get_logger().handlers != []: return True diff --git a/datashuttle/utils/folder_class.py b/datashuttle/utils/folder_class.py index 33bc16b74..87c036bab 100644 --- a/datashuttle/utils/folder_class.py +++ b/datashuttle/utils/folder_class.py @@ -10,6 +10,15 @@ def __init__( name: str, level: str, ): - """PLACEHOLDER.""" + """Parameters + ------------- + + name + the name of the folder. + + level + level to make the folder at. + + """ self.name = name self.level = level diff --git a/datashuttle/utils/formatting.py b/datashuttle/utils/formatting.py index 5afdae8c2..5177ff35e 100644 --- a/datashuttle/utils/formatting.py +++ b/datashuttle/utils/formatting.py @@ -278,17 +278,17 @@ def replace_date_time_tags_in_name( def format_date(date: str) -> str: - """PLACEHOLDER.""" + """Format the `date` as `date-`.""" return f"date-{date}" def format_time(time_: str) -> str: - """PLACEHOLDER.""" + """Format the `time_` as `time-`.""" return f"time-{time_}" def format_datetime(date: str, time_: str) -> str: - """PLACEHOLDER.""" + """Format the `date` and `time_` as `datetime-T`.""" return f"datetime-{date}T{time_}" diff --git a/datashuttle/utils/rclone.py b/datashuttle/utils/rclone.py index d651b8f96..e3fb8bcdb 100644 --- a/datashuttle/utils/rclone.py +++ b/datashuttle/utils/rclone.py @@ -112,7 +112,7 @@ def setup_rclone_config_for_ssh( def log_rclone_config_output(): - """PLACEHOLDER.""" + """Log the output from creating Rclone config.""" output = call_rclone("config file", pipe_std=True) utils.log( f"Successfully created rclone config. {output.stdout.decode('utf-8')}" diff --git a/datashuttle/utils/utils.py b/datashuttle/utils/utils.py index 56dd43670..8e51d9185 100644 --- a/datashuttle/utils/utils.py +++ b/datashuttle/utils/utils.py @@ -159,7 +159,7 @@ def get_values_from_bids_formatted_name( def sub_or_ses_value_to_int(value: str) -> int: - """PLACEHOLDER.""" + """Convert a subject or session value to an integer.""" try: int_value = int(value) except ValueError: @@ -184,7 +184,7 @@ def get_value_from_key_regexp(name: str, key: str) -> List[str]: def integers_are_consecutive(list_of_ints: List[int]) -> bool: - """PLACEHOLDER.""" + """Check if a list of integers is consecutive.""" diff_between_ints = diff(list_of_ints) return all([diff == 1 for diff in diff_between_ints]) diff --git a/datashuttle/utils/validation.py b/datashuttle/utils/validation.py index b4e89974d..4f6ca09f6 100644 --- a/datashuttle/utils/validation.py +++ b/datashuttle/utils/validation.py @@ -361,7 +361,7 @@ def names_include_special_characters( def name_has_special_character(name: str) -> bool: - """PLACEHOLDER.""" + """Check if the name contains special characters.""" return not re.match("^[A-Za-z0-9_-]*$", name) From 419cea635a4e10ffb8f85c792cf9ba8b817451e1 Mon Sep 17 00:00:00 2001 From: MoffittAndrew Date: Sun, 23 Mar 2025 14:38:12 +0000 Subject: [PATCH 22/70] syncing branch with main and adding PLACEHOLDER docstring --- tests/tests_tui/test_tui_directorytree.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/tests_tui/test_tui_directorytree.py b/tests/tests_tui/test_tui_directorytree.py index 8c07e4716..6618dc3b9 100644 --- a/tests/tests_tui/test_tui_directorytree.py +++ b/tests/tests_tui/test_tui_directorytree.py @@ -244,6 +244,7 @@ def set_signal_to_path(path_): async def test_create_folders_directorytree_rename( self, setup_project_paths ): + """PLACEHOLDER.""" tmp_config_path, tmp_path, project_name = setup_project_paths.values() rawdata_path = tmp_path / "local" / project_name / "rawdata" From b1b5d9d9bb83899e5ff7854f60616dd25396a168 Mon Sep 17 00:00:00 2001 From: MoffittAndrew Date: Sun, 23 Mar 2025 14:44:35 +0000 Subject: [PATCH 23/70] enabling ruff-format --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1dbbe5cba..fe9386ed8 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -34,7 +34,7 @@ repos: rev: v0.9.9 hooks: - id: ruff - #- id: ruff-format + - id: ruff-format - repo: https://github.com/pre-commit/mirrors-mypy rev: v1.15.0 hooks: From 4e2e70a55587ede3bb6c49c23d53209ee5d15f42 Mon Sep 17 00:00:00 2001 From: MoffittAndrew Date: Sun, 23 Mar 2025 14:47:01 +0000 Subject: [PATCH 24/70] pre-commit autofix --- tests/tests_integration/_test_configs.py | 2 +- tests/tests_tui/test_tui_directorytree.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/tests_integration/_test_configs.py b/tests/tests_integration/_test_configs.py index 32f59beec..9d983acf5 100644 --- a/tests/tests_integration/_test_configs.py +++ b/tests/tests_integration/_test_configs.py @@ -57,7 +57,7 @@ def test_warning_on_startup(self, no_cfg_project): ) @pytest.mark.parametrize("path_type", ["local_path", "central_path"]) def test_bad_path_syntax(self, project, bad_pattern, path_type, tmp_path): - """"~", "." and "../" syntax is not supported because + """ "~", "." and "../" syntax is not supported because it does not work with rclone. Theoretically it could be supported by checking for "." etc. and filling in manually, but it does not seem robust. diff --git a/tests/tests_tui/test_tui_directorytree.py b/tests/tests_tui/test_tui_directorytree.py index 6618dc3b9..73f340d24 100644 --- a/tests/tests_tui/test_tui_directorytree.py +++ b/tests/tests_tui/test_tui_directorytree.py @@ -251,7 +251,6 @@ async def test_create_folders_directorytree_rename( app = TuiApp() async with app.run_test(size=self.tui_size()) as pilot: - # Set up the 'create tab' with loaded nodes await self.setup_existing_project_create_tab_filled_sub_and_ses( pilot, project_name, create_folders=True From 7881a786df4745968a25b447b98af8532f76621e Mon Sep 17 00:00:00 2001 From: MoffittAndrew Date: Sun, 23 Mar 2025 14:55:25 +0000 Subject: [PATCH 25/70] pre-commit autofix --- tests/tests_integration/_test_configs.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/tests_integration/_test_configs.py b/tests/tests_integration/_test_configs.py index 9d983acf5..d96284f28 100644 --- a/tests/tests_integration/_test_configs.py +++ b/tests/tests_integration/_test_configs.py @@ -57,9 +57,9 @@ def test_warning_on_startup(self, no_cfg_project): ) @pytest.mark.parametrize("path_type", ["local_path", "central_path"]) def test_bad_path_syntax(self, project, bad_pattern, path_type, tmp_path): - """ "~", "." and "../" syntax is not supported because + """`~`, `.` and `../` syntax is not supported because it does not work with rclone. Theoretically it - could be supported by checking for "." etc. and + could be supported by checking for `.` etc. and filling in manually, but it does not seem robust. Here check an error is raised when path contains From ba5d7c5c547554e20ac48441057f00b9d1d64574 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 18 Jun 2025 14:43:14 +0000 Subject: [PATCH 26/70] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- datashuttle/tui/interface.py | 5 ++--- datashuttle/tui/screens/modal_dialogs.py | 1 + datashuttle/tui/screens/setup_ssh.py | 1 - datashuttle/tui/screens/validate_at_path.py | 3 +-- datashuttle/tui/shared/validate_content.py | 13 +++---------- datashuttle/utils/rclone.py | 3 +-- tests/tests_tui/test_tui_validate.py | 14 +++----------- 7 files changed, 11 insertions(+), 29 deletions(-) diff --git a/datashuttle/tui/interface.py b/datashuttle/tui/interface.py index a085727a5..850eb9bca 100644 --- a/datashuttle/tui/interface.py +++ b/datashuttle/tui/interface.py @@ -181,13 +181,11 @@ def validate_project( include_central: bool, strict_mode: bool, ) -> tuple[bool, list[str] | str]: - """ - Wrap the validate project function. This returns a list of validation + """Wrap the validate project function. This returns a list of validation errors (empty if there are none). Parameters ---------- - top_level_folder The "rawdata" or "derivatives" folder to validate. If `None`, both will be validated. @@ -195,6 +193,7 @@ def validate_project( If `True`, the central project is also validated. strict_mode If `True`, validation will be run in strict mode. + """ try: results = self.project.validate_project( diff --git a/datashuttle/tui/screens/modal_dialogs.py b/datashuttle/tui/screens/modal_dialogs.py index 65201f16f..de7ce687f 100644 --- a/datashuttle/tui/screens/modal_dialogs.py +++ b/datashuttle/tui/screens/modal_dialogs.py @@ -157,6 +157,7 @@ class SelectDirectoryTreeScreen(ModalScreen): if `None` set to the system user home. """ + def __init__( self, mainwindow: TuiApp, path_: Optional[Path] = None ) -> None: diff --git a/datashuttle/tui/screens/setup_ssh.py b/datashuttle/tui/screens/setup_ssh.py index 4e6c21d4b..54c428ec1 100644 --- a/datashuttle/tui/screens/setup_ssh.py +++ b/datashuttle/tui/screens/setup_ssh.py @@ -44,7 +44,6 @@ def compose(self) -> ComposeResult: Horizontal( Static( "Ready to setup SSH. Press OK to proceed.", - id="messagebox_message_label", ), id="messagebox_message_container", diff --git a/datashuttle/tui/screens/validate_at_path.py b/datashuttle/tui/screens/validate_at_path.py index b4157add1..965658d80 100644 --- a/datashuttle/tui/screens/validate_at_path.py +++ b/datashuttle/tui/screens/validate_at_path.py @@ -14,8 +14,7 @@ class ValidateScreen(Screen): - """ - Screen to hold the validation window for + """Screen to hold the validation window for validating an existing project at a given path. All widgets are stored in `ValidateContent`, which is shared between here and the validation tab on the project manager. diff --git a/datashuttle/tui/shared/validate_content.py b/datashuttle/tui/shared/validate_content.py index a8cce2747..bb373b7fb 100644 --- a/datashuttle/tui/shared/validate_content.py +++ b/datashuttle/tui/shared/validate_content.py @@ -27,7 +27,6 @@ class ValidateContent(Container): - def __init__( self, parent_class: Union[ @@ -42,7 +41,6 @@ def __init__( self.interface = interface def compose(self) -> ComposeResult: - if platform.system() == "Windows": example_path = r"C:\path\to\project\project_name" else: @@ -117,9 +115,7 @@ def set_select_path(self, path_): self.query_one("#validate_path_input").value = path_.as_posix() def on_button_pressed(self, event: Button.Pressed) -> None: - if event.button.id == "validate_select_button": - self.parent_class.mainwindow.push_screen( modal_dialogs.SelectDirectoryTreeScreen( self.parent_class.mainwindow @@ -128,7 +124,6 @@ def on_button_pressed(self, event: Button.Pressed) -> None: ) elif event.button.id == "validate_validate_button": - select_value = self.query_one( "#validate_top_level_folder_select" ).value @@ -140,7 +135,6 @@ def on_button_pressed(self, event: Button.Pressed) -> None: # assert False, f"strict mode: {strict_mode}" if self.interface: - if self.interface.project.is_local_project(): include_central = False else: @@ -159,11 +153,10 @@ def on_button_pressed(self, event: Button.Pressed) -> None: ) else: self.write_results_to_richlog(output) - self.query_one("#validate_logs_label").value = ( - f"Logs output to: {self.interface.project.get_logging_path()}" - ) + self.query_one( + "#validate_logs_label" + ).value = f"Logs output to: {self.interface.project.get_logging_path()}" else: - path_ = self.query_one("#validate_path_input").value if path_ == "": diff --git a/datashuttle/utils/rclone.py b/datashuttle/utils/rclone.py index be70b9adf..7913cf863 100644 --- a/datashuttle/utils/rclone.py +++ b/datashuttle/utils/rclone.py @@ -36,8 +36,7 @@ def call_rclone(command: str, pipe_std: bool = False) -> CompletedProcess: def call_rclone_through_script(command: str) -> CompletedProcess: - """ - Call rclone through a script, to avoid limits on command-line calls + """Call rclone through a script, to avoid limits on command-line calls (in particular on Windows). Used for transfers due to generation of large call strings. """ diff --git a/tests/tests_tui/test_tui_validate.py b/tests/tests_tui/test_tui_validate.py index 0972c9756..5771e3959 100644 --- a/tests/tests_tui/test_tui_validate.py +++ b/tests/tests_tui/test_tui_validate.py @@ -7,19 +7,15 @@ class TestTuiValidate(TuiBase): - @pytest.mark.asyncio async def test_validate_on_project_manager_output( self, setup_project_paths ): - """ - Check that the validate RichLog is updated as expected. - """ + """Check that the validate RichLog is updated as expected.""" tmp_config_path, tmp_path, project_name = setup_project_paths.values() app = TuiApp() async with app.run_test(size=self.tui_size()) as pilot: - # Go to the validate tab on project manager, make # some badly formatted files. await self.check_and_click_onto_existing_project( @@ -70,15 +66,13 @@ async def test_validate_on_project_manager_output( async def test_validate_on_project_manager_kwargs( self, setup_project_paths, mocker ): - """ - Check options are properly passed through to validate_project + """Check options are properly passed through to validate_project from the project manager validate tab (using mocker). """ tmp_config_path, tmp_path, project_name = setup_project_paths.values() app = TuiApp() async with app.run_test(size=self.tui_size()) as pilot: - # Set up a project and open the validate tab await self.check_and_click_onto_existing_project( pilot, project_name @@ -163,8 +157,7 @@ async def test_validate_on_project_manager_kwargs( @pytest.mark.asyncio async def test_validate_at_path_kwargs(self, setup_project_paths, mocker): - """ - Test kwargs are properly passed through from the TUI to `quick_validate_project` + """Test kwargs are properly passed through from the TUI to `quick_validate_project` with mocker. Note that the 'Select' button / directorytree is not tested here, as the screen is tested elsewhere and it's non-critical feature here. """ @@ -172,7 +165,6 @@ async def test_validate_at_path_kwargs(self, setup_project_paths, mocker): app = TuiApp() async with app.run_test(size=self.tui_size()) as pilot: - # Open the validation window and input path to project project_path = (tmp_path / "local" / project_name).as_posix() From cd6672c55a2ffe23e8f115501ecba82ddc085310 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 21 Jun 2025 22:44:29 +0000 Subject: [PATCH 27/70] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- datashuttle/configs/canonical_configs.py | 7 +++---- datashuttle/tui/app.py | 3 +-- .../tui/screens/create_folder_settings.py | 6 +++--- datashuttle/tui/screens/modal_dialogs.py | 15 ++++++--------- datashuttle/tui/tabs/create_folders.py | 11 ++++------- datashuttle/tui/utils/tui_decorators.py | 4 +--- tests/tests_integration/test_logging.py | 1 - tests/tests_integration/test_validation.py | 1 - .../test_backwards_compatibility.py | 17 +++++------------ tests/tests_tui/test_tui_create_folders.py | 6 ++---- tests/tests_tui/test_tui_selectdirectorytree.py | 5 +---- .../tests_tui/test_tui_widgets_and_defaults.py | 4 +--- 12 files changed, 27 insertions(+), 53 deletions(-) diff --git a/datashuttle/configs/canonical_configs.py b/datashuttle/configs/canonical_configs.py index a8edc3954..d0887153e 100644 --- a/datashuttle/configs/canonical_configs.py +++ b/datashuttle/configs/canonical_configs.py @@ -379,7 +379,9 @@ def in_place_update_settings_for_narrow_datatype(settings: dict): dtype in user_settings["tui"]["transfer_checkboxes_on"] for dtype in all_narrow_datatypes ] - ), "Somehow there are datatypes missing in `transfer_checkboxes_on` but not `create_checkboxes_on`" + ), ( + "Somehow there are datatypes missing in `transfer_checkboxes_on` but not `create_checkboxes_on`" + ) if has_narrow_datatypes and is_not_missing_any_narrow_datatypes: return @@ -411,15 +413,12 @@ def in_place_update_settings_for_narrow_datatype(settings: dict): # Copy any datatype information that exists. Broad datatypes will all be there # but some narrow datatypes might be missing. for checkbox_type in ["create_checkboxes_on", "transfer_checkboxes_on"]: - datatypes_that_user_has = list( user_settings["tui"][checkbox_type].keys() ) for dtype in get_datatypes(): - if dtype in datatypes_that_user_has: - if has_narrow_datatypes: new_checkbox_configs[checkbox_type][dtype] = user_settings[ "tui" diff --git a/datashuttle/tui/app.py b/datashuttle/tui/app.py index d57ddfa91..c4c2dcf25 100644 --- a/datashuttle/tui/app.py +++ b/datashuttle/tui/app.py @@ -119,8 +119,7 @@ def show_modal_error_dialog(self, message: str) -> None: self.push_screen(modal_dialogs.MessageBox(message, border_color="red")) def show_modal_error_dialog_from_main_thread(self, message: str) -> None: - """ - Used to call `show_modal_error_dialog from main thread when executing + """Used to call `show_modal_error_dialog from main thread when executing in another thread. Throws error when called from main thread. """ self.call_from_thread(self.show_modal_error_dialog, message) diff --git a/datashuttle/tui/screens/create_folder_settings.py b/datashuttle/tui/screens/create_folder_settings.py index ce72e2042..ca9cec099 100644 --- a/datashuttle/tui/screens/create_folder_settings.py +++ b/datashuttle/tui/screens/create_folder_settings.py @@ -247,9 +247,9 @@ def on_checkbox_changed(self, event: Checkbox.Changed) -> None: "#template_settings_validation_on_checkbox" ).value - self.query_one("#template_inner_container").disabled = ( - disable_container - ) + self.query_one( + "#template_inner_container" + ).disabled = disable_container elif event.checkbox.id == "suggest_next_sub_ses_central_checkbox": self.interface.save_tui_settings( is_on, "suggest_next_sub_ses_central" diff --git a/datashuttle/tui/screens/modal_dialogs.py b/datashuttle/tui/screens/modal_dialogs.py index a8ea05e04..69512f35d 100644 --- a/datashuttle/tui/screens/modal_dialogs.py +++ b/datashuttle/tui/screens/modal_dialogs.py @@ -155,8 +155,7 @@ async def handle_transfer_and_update_ui_when_complete(self) -> None: class SearchingCentralForNextSubSesPopup(ModalScreen): - """ - A popup to show message and a loading indicator when awaiting search next sub/ses across + """A popup to show message and a loading indicator when awaiting search next sub/ses across the folders present in both local and central machines. This search happens in a separate thread so as to allow TUI to display the loading indicate without freezing. @@ -231,8 +230,7 @@ def compose(self) -> ComposeResult: @staticmethod def get_drives(): - """ - Get drives available on the machine to switch between. + """Get drives available on the machine to switch between. For Windows, use `psutil` to get the list of drives. Otherwise, assume root is "/" and take all folders from that level. """ @@ -253,8 +251,7 @@ def get_drives(): ] def get_selected_drive(self): - """ - Get the default drive which the select starts on. For windows, + """Get the default drive which the select starts on. For windows, use the .drive attribute but for macOS and Linux this is blank. On these Os use the first folder (e.g. /Users) as the default drive. """ @@ -267,9 +264,9 @@ def get_selected_drive(self): def on_select_changed(self, event: Select.Changed) -> None: """Updates the directory tree when the drive is changed.""" self.path_ = Path(event.value) - self.query_one("#select_directory_tree_directory_tree").path = ( - self.path_ - ) + self.query_one( + "#select_directory_tree_directory_tree" + ).path = self.path_ @require_double_click def on_directory_tree_directory_selected( diff --git a/datashuttle/tui/tabs/create_folders.py b/datashuttle/tui/tabs/create_folders.py index 1410939e5..1fa9f3c90 100644 --- a/datashuttle/tui/tabs/create_folders.py +++ b/datashuttle/tui/tabs/create_folders.py @@ -284,8 +284,7 @@ def reload_directorytree(self) -> None: def suggest_next_sub_ses( self, prefix: Prefix, input_id: str, include_central: bool ): - """ - This handles suggesting next sub/ses for the project. Shows + """This handles suggesting next sub/ses for the project. Shows a pop up screen in cases when searching for next sub/ses takes time such as searching central in SSH connection method. @@ -317,8 +316,7 @@ def suggest_next_sub_ses( async def fill_suggestion_and_dismiss_popup( self, prefix, input_id, include_central ): - """ - This handles running the `fill_input_with_next_sub_or_ses_template` + """This handles running the `fill_input_with_next_sub_or_ses_template` worker and waiting for it to complete. If an error occurs in `fill_input_with_next_sub_or_ses_template`, it dismisses the popup itself. @@ -337,7 +335,7 @@ async def fill_suggestion_and_dismiss_popup( def fill_input_with_next_sub_or_ses_template( self, prefix: Prefix, input_id: str, include_central: bool ) -> Worker: - """ Fills a sub / ses Input with a suggested name based on the + """Fills a sub / ses Input with a suggested name based on the next subject / session in the project (local). @@ -418,8 +416,7 @@ def fill_input_with_next_sub_or_ses_template( def dismiss_popup_and_show_modal_error_dialog_from_thread( self, message: str ) -> None: - """ - This is a utility function that the `fill_input_with_next_sub_or_ses_template` + """This is a utility function that the `fill_input_with_next_sub_or_ses_template` worker calls to display error dialog an if an error occurs while suggesting the next sub/ses. Handles the TUI widget manipulation from the main thread when called from within a worker thread. diff --git a/datashuttle/tui/utils/tui_decorators.py b/datashuttle/tui/utils/tui_decorators.py index 46b236c21..2de10d811 100644 --- a/datashuttle/tui/utils/tui_decorators.py +++ b/datashuttle/tui/utils/tui_decorators.py @@ -13,14 +13,12 @@ class ClickInfo: - """ - A class to hold click-info to checking + """A class to hold click-info to checking double clicks are within the time threshold and match the widget id. """ def __init__(self): - self.prev_click_time = 0.0 self.prev_click_widget_id = "" diff --git a/tests/tests_integration/test_logging.py b/tests/tests_integration/test_logging.py index 30dc2838f..fe1b11468 100644 --- a/tests/tests_integration/test_logging.py +++ b/tests/tests_integration/test_logging.py @@ -141,7 +141,6 @@ def test_log_filename(self, project): assert re.search(regex, log_filename) is not None def test_logs_make_config_file(self, clean_project_name, tmp_path): - project = test_utils.make_project(clean_project_name) project.make_config_file( diff --git a/tests/tests_integration/test_validation.py b/tests/tests_integration/test_validation.py index e7dd00d59..b35510680 100644 --- a/tests/tests_integration/test_validation.py +++ b/tests/tests_integration/test_validation.py @@ -755,7 +755,6 @@ def test_tags_in_name_templates_pass_validation(self, project): assert "TEMPLATE: The name: ses-001_datex-20241212" in str(e.value) def test_name_templates_validate_project(self, project): - # set up name templates name_templates = { "on": True, diff --git a/tests/tests_regression/test_backwards_compatibility.py b/tests/tests_regression/test_backwards_compatibility.py index 79cefe5a1..7d73ff1d0 100644 --- a/tests/tests_regression/test_backwards_compatibility.py +++ b/tests/tests_regression/test_backwards_compatibility.py @@ -9,11 +9,9 @@ class TestBackwardsCompatibility: - @pytest.fixture(scope="function") def project(self): - """ - Delete the project configs if they exist, + """Delete the project configs if they exist, and tear down after the test has run. """ test_utils.delete_project_if_it_exists(TEST_PROJECT_NAME) @@ -25,8 +23,7 @@ def project(self): test_utils.delete_project_if_it_exists(TEST_PROJECT_NAME) def test_v0_6_0(self, project, tmp_path): - """ - v0.6.0 is the first version with narrow datatypes, and the checkboxes was refactored to + """v0.6.0 is the first version with narrow datatypes, and the checkboxes was refactored to be a {"on": bool, "displayed": bool} dict rather than a bool indicating whether the checkbox is on. However, this version is missing narrow datatypes added later (e.g. "motion"). In the test file, all 'displayed' are turned off except f2pe. @@ -53,8 +50,7 @@ def test_v0_6_0(self, project, tmp_path): assert transfer_checkboxes[key]["displayed"] is (key == "f2pe") def test_v0_5_3(self, project, tmp_path): - """ - This version did not have narrow datatypes, and the persistent checkbox setting was only a + """This version did not have narrow datatypes, and the persistent checkbox setting was only a bool. Therefore, the "displayed" uses the canonical defaults (because they don't exist in the file yet). """ reloaded_ver_configs, reloaded_ver_persistent_settings = ( @@ -82,8 +78,7 @@ def test_v0_5_3(self, project, tmp_path): def load_and_check_old_version_yamls( self, project, tmp_path, datashuttle_version ): - """ - Load an old config file in the current datashuttle version, + """Load an old config file in the current datashuttle version, and check that the new-version ('canonical') configs and persistent settings match the structure of the files loaded from the old datashuttle version. @@ -130,8 +125,7 @@ def load_and_check_old_version_yamls( return reloaded_ver_configs, reloaded_ver_persistent_settings def recursive_test_dictionary(self, dict_canonical, dict_to_test): - """ - A dictionary to check all keys in a nested dictionary + """A dictionary to check all keys in a nested dictionary match and all value types are the same. """ keys_canonical = list(dict_canonical.keys()) @@ -144,7 +138,6 @@ def recursive_test_dictionary(self, dict_canonical, dict_to_test): ) for key in dict_canonical.keys(): - if isinstance(dict_canonical[key], dict): self.recursive_test_dictionary( dict_canonical[key], dict_to_test[key] diff --git a/tests/tests_tui/test_tui_create_folders.py b/tests/tests_tui/test_tui_create_folders.py index 65263bb97..e86aa0885 100644 --- a/tests/tests_tui/test_tui_create_folders.py +++ b/tests/tests_tui/test_tui_create_folders.py @@ -529,8 +529,7 @@ async def test_get_next_sub_and_ses_no_template(self, setup_project_paths): async def test_get_next_sub_and_ses_central_no_template( self, setup_project_paths, mocker ): - """ - Test getting the next subject / session with the include_central option. Check the + """Test getting the next subject / session with the include_central option. Check the checkbox widget that turns the setting on. Trigger a get next subject / session and mock the underlying datashuttle function to ensure include_central is properly called. """ @@ -590,8 +589,7 @@ async def test_get_next_sub_and_ses_central_no_template( @pytest.mark.asyncio async def test_get_next_sub_and_ses_error_popup(self, setup_project_paths): - """ - Test the modal error dialog display on encountering an error + """Test the modal error dialog display on encountering an error while suggesting next sub/ses. Since getting the suggestion happens in a thread, the `dismiss_popup_and_show_modal_error_dialog_from_thread` function which is used to display the modal error dialog from main thread diff --git a/tests/tests_tui/test_tui_selectdirectorytree.py b/tests/tests_tui/test_tui_selectdirectorytree.py index 3d1a9d362..1885ddc34 100644 --- a/tests/tests_tui/test_tui_selectdirectorytree.py +++ b/tests/tests_tui/test_tui_selectdirectorytree.py @@ -10,11 +10,9 @@ class TestSelectTree(TuiBase): @pytest.mark.asyncio async def test_select_directory_tree(self, monkeypatch): - """ - Test that changing the drive in SelectDirectoryTreeScreen + """Test that changing the drive in SelectDirectoryTreeScreen updates the DirectoryTree path as expected. """ - # Set the Select drives to be these test cases monkeypatch.setattr( SelectDirectoryTreeScreen, @@ -30,7 +28,6 @@ async def test_select_directory_tree(self, monkeypatch): app = TuiApp() async with app.run_test() as pilot: - # Open the select directory tree screen await self.scroll_to_click_pause( pilot, "#mainwindow_new_project_button" diff --git a/tests/tests_tui/test_tui_widgets_and_defaults.py b/tests/tests_tui/test_tui_widgets_and_defaults.py index 0713ba326..12c81031e 100644 --- a/tests/tests_tui/test_tui_widgets_and_defaults.py +++ b/tests/tests_tui/test_tui_widgets_and_defaults.py @@ -910,8 +910,7 @@ async def check_top_folder_select( async def test_search_central_for_suggestion_settings( self, setup_project_paths ): - """ - Check the settings for the checkbox that selects include_central when + """Check the settings for the checkbox that selects include_central when getting the next subject or session in the 'Create' tab and ensure that the underlying settings are changed. """ @@ -919,7 +918,6 @@ async def test_search_central_for_suggestion_settings( app = TuiApp() async with app.run_test(size=self.tui_size()) as pilot: - await self.setup_existing_project_create_tab_filled_sub_and_ses( pilot, project_name, create_folders=False ) From 53c9a24959b514e3cc8baae9e4ec4f6b63c878de Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Sat, 21 Jun 2025 23:59:47 +0100 Subject: [PATCH 28/70] Fix linting. --- datashuttle/configs/canonical_configs.py | 10 ++++------ datashuttle/datashuttle_class.py | 4 +--- datashuttle/tui/app.py | 3 +-- .../tui/screens/create_folder_settings.py | 6 +++--- datashuttle/tui/screens/modal_dialogs.py | 18 ++++++++---------- datashuttle/tui/screens/validate_at_path.py | 2 ++ datashuttle/tui/shared/validate_content.py | 8 +++++++- datashuttle/tui/tabs/create_folders.py | 15 +++++++-------- datashuttle/tui/utils/tui_decorators.py | 4 +--- datashuttle/utils/validation.py | 1 - pyproject.toml | 5 ++--- tests/test_utils.py | 2 -- tests/tests_integration/test_logging.py | 1 - tests/tests_integration/test_validation.py | 1 - .../test_backwards_compatibility.py | 17 +++++------------ tests/tests_tui/test_tui_create_folders.py | 6 ++---- .../tests_tui/test_tui_selectdirectorytree.py | 5 +---- .../tests_tui/test_tui_widgets_and_defaults.py | 4 +--- tests/tests_unit/test_unit.py | 9 ++++----- 19 files changed, 49 insertions(+), 72 deletions(-) diff --git a/datashuttle/configs/canonical_configs.py b/datashuttle/configs/canonical_configs.py index a8edc3954..f8700a610 100644 --- a/datashuttle/configs/canonical_configs.py +++ b/datashuttle/configs/canonical_configs.py @@ -341,9 +341,8 @@ def quick_get_narrow_datatypes(): return flat_narrow_datatypes -def in_place_update_settings_for_narrow_datatype(settings: dict): +def in_place_update_narrow_datatypes_if_required(user_settings: dict): """In versions < v0.6.0, only 'broad' datatypes were implemented - and available in the TUI. Since, 'narrow' datatypes are introduced and datatype tui can be set to be both on / off but also displayed / not displayed. @@ -379,7 +378,9 @@ def in_place_update_settings_for_narrow_datatype(settings: dict): dtype in user_settings["tui"]["transfer_checkboxes_on"] for dtype in all_narrow_datatypes ] - ), "Somehow there are datatypes missing in `transfer_checkboxes_on` but not `create_checkboxes_on`" + ), ( + "Somehow there are datatypes missing in `transfer_checkboxes_on` but not `create_checkboxes_on`" + ) if has_narrow_datatypes and is_not_missing_any_narrow_datatypes: return @@ -411,15 +412,12 @@ def in_place_update_settings_for_narrow_datatype(settings: dict): # Copy any datatype information that exists. Broad datatypes will all be there # but some narrow datatypes might be missing. for checkbox_type in ["create_checkboxes_on", "transfer_checkboxes_on"]: - datatypes_that_user_has = list( user_settings["tui"][checkbox_type].keys() ) for dtype in get_datatypes(): - if dtype in datatypes_that_user_has: - if has_narrow_datatypes: new_checkbox_configs[checkbox_type][dtype] = user_settings[ "tui" diff --git a/datashuttle/datashuttle_class.py b/datashuttle/datashuttle_class.py index 910b2478c..764cedc7e 100644 --- a/datashuttle/datashuttle_class.py +++ b/datashuttle/datashuttle_class.py @@ -1045,9 +1045,7 @@ def get_next_sub( return_with_prefix: bool = True, include_central: bool = False, ) -> str: - """Convenience function for `get_next_sub_or_ses` - - to find the next subject number. + """Convenience function for `get_next_sub_or_ses` to find the next subject number. Parameters ---------- diff --git a/datashuttle/tui/app.py b/datashuttle/tui/app.py index d57ddfa91..c4c2dcf25 100644 --- a/datashuttle/tui/app.py +++ b/datashuttle/tui/app.py @@ -119,8 +119,7 @@ def show_modal_error_dialog(self, message: str) -> None: self.push_screen(modal_dialogs.MessageBox(message, border_color="red")) def show_modal_error_dialog_from_main_thread(self, message: str) -> None: - """ - Used to call `show_modal_error_dialog from main thread when executing + """Used to call `show_modal_error_dialog from main thread when executing in another thread. Throws error when called from main thread. """ self.call_from_thread(self.show_modal_error_dialog, message) diff --git a/datashuttle/tui/screens/create_folder_settings.py b/datashuttle/tui/screens/create_folder_settings.py index ce72e2042..ca9cec099 100644 --- a/datashuttle/tui/screens/create_folder_settings.py +++ b/datashuttle/tui/screens/create_folder_settings.py @@ -247,9 +247,9 @@ def on_checkbox_changed(self, event: Checkbox.Changed) -> None: "#template_settings_validation_on_checkbox" ).value - self.query_one("#template_inner_container").disabled = ( - disable_container - ) + self.query_one( + "#template_inner_container" + ).disabled = disable_container elif event.checkbox.id == "suggest_next_sub_ses_central_checkbox": self.interface.save_tui_settings( is_on, "suggest_next_sub_ses_central" diff --git a/datashuttle/tui/screens/modal_dialogs.py b/datashuttle/tui/screens/modal_dialogs.py index a8ea05e04..176526719 100644 --- a/datashuttle/tui/screens/modal_dialogs.py +++ b/datashuttle/tui/screens/modal_dialogs.py @@ -155,8 +155,7 @@ async def handle_transfer_and_update_ui_when_complete(self) -> None: class SearchingCentralForNextSubSesPopup(ModalScreen): - """ - A popup to show message and a loading indicator when awaiting search next sub/ses across + """A popup to show message and a loading indicator when awaiting search next sub/ses across the folders present in both local and central machines. This search happens in a separate thread so as to allow TUI to display the loading indicate without freezing. @@ -168,6 +167,7 @@ def __init__(self, sub_or_ses: Prefix) -> None: self.message = f"Searching central for next {sub_or_ses}" def compose(self) -> ComposeResult: + """PLACEHOLDER.""" yield Container( Label(self.message, id="searching_message_label"), LoadingIndicator(id="searching_animated_indicator"), @@ -231,8 +231,7 @@ def compose(self) -> ComposeResult: @staticmethod def get_drives(): - """ - Get drives available on the machine to switch between. + """Get drives available on the machine to switch between. For Windows, use `psutil` to get the list of drives. Otherwise, assume root is "/" and take all folders from that level. """ @@ -253,8 +252,7 @@ def get_drives(): ] def get_selected_drive(self): - """ - Get the default drive which the select starts on. For windows, + """Get the default drive which the select starts on. For windows, use the .drive attribute but for macOS and Linux this is blank. On these Os use the first folder (e.g. /Users) as the default drive. """ @@ -267,15 +265,15 @@ def get_selected_drive(self): def on_select_changed(self, event: Select.Changed) -> None: """Updates the directory tree when the drive is changed.""" self.path_ = Path(event.value) - self.query_one("#select_directory_tree_directory_tree").path = ( - self.path_ - ) + self.query_one( + "#select_directory_tree_directory_tree" + ).path = self.path_ @require_double_click def on_directory_tree_directory_selected( self, event: DirectoryTree.DirectorySelected ) -> None: - """PLACEHOLDER""" + """PLACEHOLDER.""" if event.path.is_file(): return else: diff --git a/datashuttle/tui/screens/validate_at_path.py b/datashuttle/tui/screens/validate_at_path.py index 965658d80..18b9fcdea 100644 --- a/datashuttle/tui/screens/validate_at_path.py +++ b/datashuttle/tui/screens/validate_at_path.py @@ -28,6 +28,7 @@ def __init__(self, mainwindow: TuiApp) -> None: self.mainwindow = mainwindow def compose(self) -> ComposeResult: + """PLACEHOLDER.""" yield Header() yield Button("Main Menu", id="all_main_menu_buttons") yield validate_content.ValidateContent( @@ -35,5 +36,6 @@ def compose(self) -> ComposeResult: ) def on_button_pressed(self, event: Button.Pressed) -> None: + """PLACEHOLDER.""" if event.button.id == "all_main_menu_buttons": self.dismiss(None) diff --git a/datashuttle/tui/shared/validate_content.py b/datashuttle/tui/shared/validate_content.py index bb373b7fb..87ae57b71 100644 --- a/datashuttle/tui/shared/validate_content.py +++ b/datashuttle/tui/shared/validate_content.py @@ -27,6 +27,8 @@ class ValidateContent(Container): + """PLACEHOLDER.""" + def __init__( self, parent_class: Union[ @@ -41,6 +43,7 @@ def __init__( self.interface = interface def compose(self) -> ComposeResult: + """PLACEHOLDER.""" if platform.system() == "Windows": example_path = r"C:\path\to\project\project_name" else: @@ -87,7 +90,7 @@ def compose(self) -> ComposeResult: yield Container(*widgets, id="validate_top_container") def on_mount(self) -> None: - """ """ + """PLACEHOLDER.""" for id in [ "validate_path_input", "validate_top_level_folder_select", @@ -111,10 +114,12 @@ def on_mount(self) -> None: self.query_one("#validate_include_central_checkbox").remove() def set_select_path(self, path_): + """PLACEHOLDER.""" if path_: self.query_one("#validate_path_input").value = path_.as_posix() def on_button_pressed(self, event: Button.Pressed) -> None: + """PLACEHOLDER.""" if event.button.id == "validate_select_button": self.parent_class.mainwindow.push_screen( modal_dialogs.SelectDirectoryTreeScreen( @@ -179,6 +184,7 @@ def on_button_pressed(self, event: Button.Pressed) -> None: self.write_results_to_richlog(output) def write_results_to_richlog(self, results): + """PLACEHOLDER.""" text_log = self.query_one("#validate_richlog") text_log.clear() if any(results): diff --git a/datashuttle/tui/tabs/create_folders.py b/datashuttle/tui/tabs/create_folders.py index 1410939e5..d36eb00d5 100644 --- a/datashuttle/tui/tabs/create_folders.py +++ b/datashuttle/tui/tabs/create_folders.py @@ -284,8 +284,7 @@ def reload_directorytree(self) -> None: def suggest_next_sub_ses( self, prefix: Prefix, input_id: str, include_central: bool ): - """ - This handles suggesting next sub/ses for the project. Shows + """Suggests the next sub/ses name for the project. Shows a pop up screen in cases when searching for next sub/ses takes time such as searching central in SSH connection method. @@ -317,8 +316,7 @@ def suggest_next_sub_ses( async def fill_suggestion_and_dismiss_popup( self, prefix, input_id, include_central ): - """ - This handles running the `fill_input_with_next_sub_or_ses_template` + """Runs the `fill_input_with_next_sub_or_ses_template` worker and waiting for it to complete. If an error occurs in `fill_input_with_next_sub_or_ses_template`, it dismisses the popup itself. @@ -337,8 +335,7 @@ async def fill_suggestion_and_dismiss_popup( def fill_input_with_next_sub_or_ses_template( self, prefix: Prefix, input_id: str, include_central: bool ) -> Worker: - """ Fills a sub / ses Input with a suggested name based on the - + """Fills a sub / ses Input with a suggested name based on the next subject / session in the project (local). If `name_templates` are set, then the sub- or ses- first key @@ -357,6 +354,9 @@ def fill_input_with_next_sub_or_ses_template( input_id The textual input name to update. + include_central + If `True`, the central project is also validated. + """ top_level_folder = self.interface.tui_settings[ "top_level_folder_select" @@ -418,8 +418,7 @@ def fill_input_with_next_sub_or_ses_template( def dismiss_popup_and_show_modal_error_dialog_from_thread( self, message: str ) -> None: - """ - This is a utility function that the `fill_input_with_next_sub_or_ses_template` + """Utility function that the `fill_input_with_next_sub_or_ses_template` worker calls to display error dialog an if an error occurs while suggesting the next sub/ses. Handles the TUI widget manipulation from the main thread when called from within a worker thread. diff --git a/datashuttle/tui/utils/tui_decorators.py b/datashuttle/tui/utils/tui_decorators.py index 46b236c21..2de10d811 100644 --- a/datashuttle/tui/utils/tui_decorators.py +++ b/datashuttle/tui/utils/tui_decorators.py @@ -13,14 +13,12 @@ class ClickInfo: - """ - A class to hold click-info to checking + """A class to hold click-info to checking double clicks are within the time threshold and match the widget id. """ def __init__(self): - self.prev_click_time = 0.0 self.prev_click_widget_id = "" diff --git a/datashuttle/utils/validation.py b/datashuttle/utils/validation.py index 077edaba0..76c91a070 100644 --- a/datashuttle/utils/validation.py +++ b/datashuttle/utils/validation.py @@ -314,7 +314,6 @@ def get_path_and_name(path_or_name: Path | str) -> Tuple[Optional[Path], str]: def replace_tags_in_regexp(regexp: str) -> str: r"""Before validation, all tags in the names are converted to - their final values (e.g. @DATE@ -> _date-). We also want to allow template to be formatted like `sub-\d\d_@DATE@` as it is convenient for auto-completion in the TUI. diff --git a/pyproject.toml b/pyproject.toml index 92216d4b5..825d6b1f4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -111,6 +111,7 @@ ignore = ["E203","E501","E731","C901","W291","W293","E402","E722", "D205", # 1 blank line required between summary line and description "D213", # multi-line-summary second line "D401", # first line of docstrings should be in an imperative mood + "D107", # Missing docstring in `__init__` ] select = [ "I", # isort @@ -121,9 +122,7 @@ select = [ "D", # pydocstyle ] per-file-ignores = { "tests/*" = [ - "D100", # missing docstring in public module - "D205", # missing blank line between summary and description - "D103", # missing docstring in public function + "D" # ignore docstring formatting in tests for now ], "examples/*" = [ "D400", # first line should end with a period. "D415", # first line should end with a period, question mark... diff --git a/tests/test_utils.py b/tests/test_utils.py index 934ba5dd5..ade43bf76 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -83,8 +83,6 @@ def glob_basenames(search_path, recursive=False, exclude=None): def teardown_project( project, ): # 99% sure these are unnecessary with pytest tmp_path but keep until SSH testing. - os.chdir(cwd) - delete_all_folders_in_project_path(project, "central") delete_all_folders_in_project_path(project, "local") delete_project_if_it_exists(project.project_name) diff --git a/tests/tests_integration/test_logging.py b/tests/tests_integration/test_logging.py index 30dc2838f..fe1b11468 100644 --- a/tests/tests_integration/test_logging.py +++ b/tests/tests_integration/test_logging.py @@ -141,7 +141,6 @@ def test_log_filename(self, project): assert re.search(regex, log_filename) is not None def test_logs_make_config_file(self, clean_project_name, tmp_path): - project = test_utils.make_project(clean_project_name) project.make_config_file( diff --git a/tests/tests_integration/test_validation.py b/tests/tests_integration/test_validation.py index e7dd00d59..b35510680 100644 --- a/tests/tests_integration/test_validation.py +++ b/tests/tests_integration/test_validation.py @@ -755,7 +755,6 @@ def test_tags_in_name_templates_pass_validation(self, project): assert "TEMPLATE: The name: ses-001_datex-20241212" in str(e.value) def test_name_templates_validate_project(self, project): - # set up name templates name_templates = { "on": True, diff --git a/tests/tests_regression/test_backwards_compatibility.py b/tests/tests_regression/test_backwards_compatibility.py index 79cefe5a1..7d73ff1d0 100644 --- a/tests/tests_regression/test_backwards_compatibility.py +++ b/tests/tests_regression/test_backwards_compatibility.py @@ -9,11 +9,9 @@ class TestBackwardsCompatibility: - @pytest.fixture(scope="function") def project(self): - """ - Delete the project configs if they exist, + """Delete the project configs if they exist, and tear down after the test has run. """ test_utils.delete_project_if_it_exists(TEST_PROJECT_NAME) @@ -25,8 +23,7 @@ def project(self): test_utils.delete_project_if_it_exists(TEST_PROJECT_NAME) def test_v0_6_0(self, project, tmp_path): - """ - v0.6.0 is the first version with narrow datatypes, and the checkboxes was refactored to + """v0.6.0 is the first version with narrow datatypes, and the checkboxes was refactored to be a {"on": bool, "displayed": bool} dict rather than a bool indicating whether the checkbox is on. However, this version is missing narrow datatypes added later (e.g. "motion"). In the test file, all 'displayed' are turned off except f2pe. @@ -53,8 +50,7 @@ def test_v0_6_0(self, project, tmp_path): assert transfer_checkboxes[key]["displayed"] is (key == "f2pe") def test_v0_5_3(self, project, tmp_path): - """ - This version did not have narrow datatypes, and the persistent checkbox setting was only a + """This version did not have narrow datatypes, and the persistent checkbox setting was only a bool. Therefore, the "displayed" uses the canonical defaults (because they don't exist in the file yet). """ reloaded_ver_configs, reloaded_ver_persistent_settings = ( @@ -82,8 +78,7 @@ def test_v0_5_3(self, project, tmp_path): def load_and_check_old_version_yamls( self, project, tmp_path, datashuttle_version ): - """ - Load an old config file in the current datashuttle version, + """Load an old config file in the current datashuttle version, and check that the new-version ('canonical') configs and persistent settings match the structure of the files loaded from the old datashuttle version. @@ -130,8 +125,7 @@ def load_and_check_old_version_yamls( return reloaded_ver_configs, reloaded_ver_persistent_settings def recursive_test_dictionary(self, dict_canonical, dict_to_test): - """ - A dictionary to check all keys in a nested dictionary + """A dictionary to check all keys in a nested dictionary match and all value types are the same. """ keys_canonical = list(dict_canonical.keys()) @@ -144,7 +138,6 @@ def recursive_test_dictionary(self, dict_canonical, dict_to_test): ) for key in dict_canonical.keys(): - if isinstance(dict_canonical[key], dict): self.recursive_test_dictionary( dict_canonical[key], dict_to_test[key] diff --git a/tests/tests_tui/test_tui_create_folders.py b/tests/tests_tui/test_tui_create_folders.py index 65263bb97..e86aa0885 100644 --- a/tests/tests_tui/test_tui_create_folders.py +++ b/tests/tests_tui/test_tui_create_folders.py @@ -529,8 +529,7 @@ async def test_get_next_sub_and_ses_no_template(self, setup_project_paths): async def test_get_next_sub_and_ses_central_no_template( self, setup_project_paths, mocker ): - """ - Test getting the next subject / session with the include_central option. Check the + """Test getting the next subject / session with the include_central option. Check the checkbox widget that turns the setting on. Trigger a get next subject / session and mock the underlying datashuttle function to ensure include_central is properly called. """ @@ -590,8 +589,7 @@ async def test_get_next_sub_and_ses_central_no_template( @pytest.mark.asyncio async def test_get_next_sub_and_ses_error_popup(self, setup_project_paths): - """ - Test the modal error dialog display on encountering an error + """Test the modal error dialog display on encountering an error while suggesting next sub/ses. Since getting the suggestion happens in a thread, the `dismiss_popup_and_show_modal_error_dialog_from_thread` function which is used to display the modal error dialog from main thread diff --git a/tests/tests_tui/test_tui_selectdirectorytree.py b/tests/tests_tui/test_tui_selectdirectorytree.py index 3d1a9d362..1885ddc34 100644 --- a/tests/tests_tui/test_tui_selectdirectorytree.py +++ b/tests/tests_tui/test_tui_selectdirectorytree.py @@ -10,11 +10,9 @@ class TestSelectTree(TuiBase): @pytest.mark.asyncio async def test_select_directory_tree(self, monkeypatch): - """ - Test that changing the drive in SelectDirectoryTreeScreen + """Test that changing the drive in SelectDirectoryTreeScreen updates the DirectoryTree path as expected. """ - # Set the Select drives to be these test cases monkeypatch.setattr( SelectDirectoryTreeScreen, @@ -30,7 +28,6 @@ async def test_select_directory_tree(self, monkeypatch): app = TuiApp() async with app.run_test() as pilot: - # Open the select directory tree screen await self.scroll_to_click_pause( pilot, "#mainwindow_new_project_button" diff --git a/tests/tests_tui/test_tui_widgets_and_defaults.py b/tests/tests_tui/test_tui_widgets_and_defaults.py index 0713ba326..12c81031e 100644 --- a/tests/tests_tui/test_tui_widgets_and_defaults.py +++ b/tests/tests_tui/test_tui_widgets_and_defaults.py @@ -910,8 +910,7 @@ async def check_top_folder_select( async def test_search_central_for_suggestion_settings( self, setup_project_paths ): - """ - Check the settings for the checkbox that selects include_central when + """Check the settings for the checkbox that selects include_central when getting the next subject or session in the 'Create' tab and ensure that the underlying settings are changed. """ @@ -919,7 +918,6 @@ async def test_search_central_for_suggestion_settings( app = TuiApp() async with app.run_test(size=self.tui_size()) as pilot: - await self.setup_existing_project_create_tab_filled_sub_and_ses( pilot, project_name, create_folders=False ) diff --git a/tests/tests_unit/test_unit.py b/tests/tests_unit/test_unit.py index 2a6499dd3..c63b045a8 100644 --- a/tests/tests_unit/test_unit.py +++ b/tests/tests_unit/test_unit.py @@ -17,12 +17,11 @@ class TestUnit: ) def test_datetime_string_replacement(self, key, underscore_position): r"""Test the function that replaces @DATE, @TIME@ or @DATETIME@ + keywords with the date / time / datetime. - keywords with the date / time / datetime. Also, it will - pre/append underscores to the tags if they are not - already there (e.g if user input "sub-001@DATE"). - Note cannot use regex \d{8} format because we are in an - f-string. + Also, it will pre/append underscores to the tags if they are not + already there (e.g if user input "sub-001@DATE"). Note cannot use + regex \d{8} format because we are in an f-string. """ start = "sub-001" end = "other-tag" From 041b8900dbc0d591d0bd360c00a4bd722943374c Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Sun, 22 Jun 2025 00:57:00 +0100 Subject: [PATCH 29/70] Fixing validation as a test. --- datashuttle/utils/validation.py | 150 ++++++++---------- pyproject.toml | 3 - .../test_filesystem_transfer.py | 2 +- 3 files changed, 65 insertions(+), 90 deletions(-) diff --git a/datashuttle/utils/validation.py b/datashuttle/utils/validation.py index 76c91a070..88fc02a77 100644 --- a/datashuttle/utils/validation.py +++ b/datashuttle/utils/validation.py @@ -94,9 +94,7 @@ def get_datetime_error(key, name: str, strfmt: str, path_: Path | None) -> str: def get_template_error(name: str, regexp: str, path_: Path | None) -> str: - """The missing full-stop at the end is intentional, to avoid - confusion when reading the regexp. - """ + """The missing full-stop at the end is intentional, to avoid confusion when reading the regexp.""" return handle_path( f"TEMPLATE: The name: {name} does not match the template: {regexp}", path_, @@ -154,17 +152,17 @@ def validate_list_of_names( Parameters ---------- path_or_name_list - A list of pathlib.Path to NeuroBlueprint-formatted folders to validate + A list of pathlib.Path to NeuroBlueprint-formatted folders to validate prefix - Whether these are subject (sub) or session (ses) level names + Whether these are subject (sub) or session (ses) level names name_templates - A `name_template` dictionary to validate against. See `set_name_templates()`. + A `name_template` dictionary to validate against. See `set_name_templates()`. check_value_lengths - If `True`, check that the prefix- value lengths - are consistent across the passed list. + If `True`, check that the prefix- value lengths + are consistent across the passed list. """ if len(path_or_name_list) == 0: @@ -216,10 +214,7 @@ def validate_list_of_names( def prefix_is_duplicate_or_has_bad_values( name: str, prefix: Prefix, path_: Path | None ) -> List[str]: - """Check that the prefix (sub- or ses-) is found only - once in the name and that its value can be converted - to integer. - """ + """Check that the prefix (sub- or ses-) is found only once in the name and that its value can be converted to integer.""" value = re.findall(f"{prefix}(.*?)(?=_|$)", name) if len(value) == 0: @@ -240,15 +235,9 @@ def new_name_duplicates_existing( existing_path_or_name_list: List[Path] | List[str], prefix: Prefix, ) -> List[str]: - """Check that a subject or session value does not duplicate - an existing value. The only case this is allowed is - when the names match exactly. - - For example, if "sub-001" exists, we can pass - "sub-001" as a valid subject name (for example, when making sessions). - However, if "sub-001_another-tag" exists, we should throw an - error, because this shares the same subject id but refers to - a different subject. + """Check that a subject or session value does not duplicate an existing value. + + The only case this is allowed is when the names match exactly. For example, if "sub-001" exists, we can pass "sub-001" as a valid subject name (for example, when making sessions). However, if "sub-001_another-tag" exists, we should throw an error, because this shares the same subject id but refers to a different subject. """ # Make a list of matches between `new_name` and any in `existing_names` new_name_id = utils.get_values_from_bids_formatted_name( @@ -281,6 +270,7 @@ def names_dont_match_templates( ) -> List[str]: """Test a list of subject or session names against the respective `name_templates`, a regexp template. + """ if name_templates is None: return [] @@ -303,6 +293,7 @@ def names_dont_match_templates( def get_path_and_name(path_or_name: Path | str) -> Tuple[Optional[Path], str]: """Convenience function to get the folder name + from something that is either a Path (pointing to the folder) or a str of the folder name itself. """ @@ -313,13 +304,10 @@ def get_path_and_name(path_or_name: Path | str) -> Tuple[Optional[Path], str]: def replace_tags_in_regexp(regexp: str) -> str: - r"""Before validation, all tags in the names are converted to - their final values (e.g. @DATE@ -> _date-). We also want to - allow template to be formatted like `sub-\d\d_@DATE@` as it - is convenient for auto-completion in the TUI. + r"""Before validation, all tags in the names are converted to their final values (e.g. @DATE@ -> _date-). We also want to allow template to be formatted like `sub-\d\d_@DATE@` as it is convenient for auto-completion in the TUI. + + Therefore, we must replace the tags in the regexp with their actual regexp equivalent before comparison. - Therefore, we must replace the tags in the regexp with their - actual regexp equivalent before comparison. Note `replace_date_time_tags_in_name()` operates in place on a list. """ regexp_list = [regexp] @@ -338,9 +326,7 @@ def replace_tags_in_regexp(regexp: str) -> str: def name_begins_with_bad_key( name: str, prefix: Prefix, path_: Path | None ) -> List[str]: - """Check that a list of NeuroBlueprint names begin - with the required prefix (sub- or ses-). - """ + """Check that a list of NeuroBlueprint names begin with the required prefix (sub- or ses-).""" if name[:4] != f"{prefix}-": return [get_name_error(name, prefix, path_)] else: @@ -350,9 +336,9 @@ def name_begins_with_bad_key( def names_include_special_characters( name: str, path_: Path | None ) -> List[str]: - """Check that a list of NeuroBlueprint formatted - names do not contain special characters (i.e. characters - that are not integers, letters, dash or underscore). + """Check that a list of NeuroBlueprint formatted names do not contain special characters. + + Special characters are characters that are not integers, letters, dash or underscore. """ if name_has_special_character(name): return [get_special_char_error(name, path_)] @@ -369,9 +355,9 @@ def dashes_and_underscore_alternate_incorrectly( name: str, path_: Path | None ) -> List[str]: """Check a list of NeuroBlueprint formatted names - have the "-" and "-" ordered correctly. Names should be - key-value pairs separated by underscores e.g. - sub-001_ses-001. + + Names should have the "-" and "-" ordered correctly. Names should be + key-value pairs separated by underscores e.g. sub-001_ses-001. """ discrim = {"-": 1, "_": -1} @@ -397,9 +383,9 @@ def value_lengths_are_inconsistent( path_or_names_list: List[Path] | List[str] | List[Path | str], prefix: Prefix, ) -> List[str]: - """Given a list of NeuroBlueprint-formatted subject or session - names, determine if there are inconsistent value lengths for - the sub or ses key. e.g. ["sub-01", "sub-001"] is an error. + """Given a list of NeuroBlueprint-formatted subject or session names, + + Determine if there are inconsistent value lengths for the sub or ses key. e.g. ["sub-01", "sub-001"] is an error. """ names_list = [ path_or_name if isinstance(path_or_name, str) else path_or_name.name @@ -464,6 +450,7 @@ def raise_display_mode( message: str, display_mode: DisplayMode, log: bool ) -> None: """Show a message by raising an error, displaying warning, or printing. + Optionally log with the current datashuttle logger. """ if display_mode == "error": @@ -502,32 +489,32 @@ def validate_project( Parameters ---------- cfg - datashuttle Configs class. + datashuttle Configs class. top_level_folder_list - The top level folders to validate. + The top level folders to validate. include_central - If `False`, only project folders in the `local_path` will - be validated. Otherwise, project folders in both the `local_path` - and `central_path` will be validated. + If `False`, only project folders in the `local_path` will + be validated. Otherwise, project folders in both the `local_path` + and `central_path` will be validated. display_mode - Determine whether error or warning is raised. + Determine whether error or warning is raised. log - If `True`, errors or warnings are logged to "datashuttle" logger. + If `True`, errors or warnings are logged to "datashuttle" logger. name_templates - A `name_template` dictionary to validate against. See `set_name_templates()`. + A `name_template` dictionary to validate against. See `set_name_templates()`. strict_mode - If `True`, only allow NeuroBlueprint-formatted folders to exist in - the project. By default, non-NeuroBlueprint folders (e.g. a folder - called 'my_stuff' in the 'rawdata') are allowed, and only folders - starting with sub- or ses- prefix are checked. In `Strict Mode`, - any folder not prefixed with sub-, ses- or a valid datatype will - raise a validation issue. + If `True`, only allow NeuroBlueprint-formatted folders to exist in + the project. By default, non-NeuroBlueprint folders (e.g. a folder + called 'my_stuff' in the 'rawdata') are allowed, and only folders + starting with sub- or ses- prefix are checked. In `Strict Mode`, + any folder not prefixed with sub-, ses- or a valid datatype will + raise a validation issue. """ error_messages = [] @@ -597,45 +584,41 @@ def validate_names_against_project( log: bool = True, name_templates: Optional[Dict] = None, ) -> None: - """Given a list of subject and (optionally) session names, - check that these names are formatted consistently with the - rest of the project. Used for creating folders. + """Given a list of subject and (optionally) session names, check that these names are formatted consistently with the rest of the project. Used for creating folders. - Unfortunately this is quite fiddly, as it is important to only - validate the passed list of subject / session names while ignoring - validation errors that may already exist in the project. + Unfortunately this is quite fiddly, as it is important to only validate the passed list of subject / session names while ignoring validation errors that may already exist in the project. Parameters ---------- cfg - datashuttle Configs class. + datashuttle Configs class. top_level_folder - The top level folder to validate + The top level folder to validate sub_names - A list of subject-level names to validate against the - subject names that exist in the project. + A list of subject-level names to validate against the + subject names that exist in the project. ses_names - A list of session-level names to validate against the - session names that exist in the project. Note that - duplicate checks will only be performed for sessions within - the passed `sub_names`. + A list of session-level names to validate against the + session names that exist in the project. Note that + duplicate checks will only be performed for sessions within + the passed `sub_names`. include_central - If `True`, only project folders in the `local_path` will - be validated against. Otherwise, project folders in both the - `local_path` and `central_path` will be validated against. + If `True`, only project folders in the `local_path` will + be validated against. Otherwise, project folders in both the + `local_path` and `central_path` will be validated against. display_mode - Determine whether error or warning is raised. + Determine whether error or warning is raised. log - If `True`, errors or warnings are logged to "datashuttle" logger. + If `True`, errors or warnings are logged to "datashuttle" logger. name_templates - A `name_template` dictionary to validate against. See `set_name_templates()`. + A `name_template` dictionary to validate against. See `set_name_templates()`. """ error_messages = [] @@ -734,10 +717,10 @@ def validate_names_against_project( def check_high_level_project_structure( cfg: Configs, include_central: bool ) -> List[str]: - """Perform basic validation checks on the project structure, - that the project folder name is valid, and that the - project folder contains either a "rawdata" or "derivatives" - folder. + """Perform basic validation checks on the project structure. + + This includes checking that the project folder name is valid, and that the + project folder contains either a "rawdata" or "derivatives" folder. """ # To avoid circular imports from datashuttle.utils.folders import search_for_folders @@ -899,12 +882,9 @@ def strip_uncheckable_names( path_or_names_list: List[Path] | List[str], prefix: Prefix, ) -> List[Path] | List[str]: - """Convenience function to remove any name in which - the `prefix` value (sub or ses typically) cannot be - converted into an integer. This is necessary as some - validation steps (e.g. checking duplicate names) requires - conversion to `int` and will fail for this reason if these - bad names are not removed. + """Convenience function to remove any name in which the `prefix` value (sub or ses typically) cannot be converted into an integer. + + This is necessary as some validation steps (e.g. checking duplicate names) requires conversion to `int` and will fail for this reason if these bad names are not removed. """ new_list = [] @@ -934,9 +914,7 @@ def strip_uncheckable_names( def check_datatypes_are_valid( datatype: Union[List[str], str], allow_all: bool = False ) -> str | None: - """Check a datatype of list of datatypes is a valid - NeuroBlueprint datatype. - """ + """Check a datatype of list of datatypes is a valid NeuroBlueprint datatype.""" datatype_folders = canonical_folders.get_datatype_folders() if isinstance(datatype, str): diff --git a/pyproject.toml b/pyproject.toml index 825d6b1f4..0a9166652 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -108,10 +108,7 @@ fix = true ignore = ["E203","E501","E731","C901","W291","W293","E402","E722", "D100", # missing docstring in public module (not enforced FOR NOW) "D203", # one blank line before class - "D205", # 1 blank line required between summary line and description "D213", # multi-line-summary second line - "D401", # first line of docstrings should be in an imperative mood - "D107", # Missing docstring in `__init__` ] select = [ "I", # isort diff --git a/tests/tests_integration/test_filesystem_transfer.py b/tests/tests_integration/test_filesystem_transfer.py index 869dfa158..82b244459 100644 --- a/tests/tests_integration/test_filesystem_transfer.py +++ b/tests/tests_integration/test_filesystem_transfer.py @@ -213,7 +213,7 @@ def test_transfer_empty_folder_specific_data( ["behav", "ephys", "funcimg", "anat"], ], ) - @pytest.mark.parametrize("upload_or_download", ["uploaddownload"]) + @pytest.mark.parametrize("upload_or_download", ["upload", "download"]) def test_transfer_empty_folder_specific_subs( self, project, From f870aea84d86555c3f65c89da18629cd344d9fb7 Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Sun, 22 Jun 2025 01:43:38 +0100 Subject: [PATCH 30/70] Rewrite config_class.py --- datashuttle/configs/config_class.py | 68 ++++++++++++++++------------- 1 file changed, 38 insertions(+), 30 deletions(-) diff --git a/datashuttle/configs/config_class.py b/datashuttle/configs/config_class.py index 072297154..2f05c2e7d 100644 --- a/datashuttle/configs/config_class.py +++ b/datashuttle/configs/config_class.py @@ -25,7 +25,7 @@ class Configs(UserDict): - """Class to hold the datashuttle configs. + """Store and manage datashuttle configuration settings. The configs must match exactly the standard set in canonical_configs.py. If updating these configs, @@ -35,8 +35,13 @@ class Configs(UserDict): def __init__( self, project_name: str, file_path: Path, input_dict: Union[dict, None] ) -> None: - """Parameters + """Initialize the Configs class with project name, file path, and config dictionary. + + Parameters ---------- + project_name + Name of the datashuttle project. + file_path full filepath to save the config .yaml file to. @@ -48,6 +53,7 @@ def __init__( canonical standard by calling check_dict_values_raise_on_fail() project_name and all paths are set at runtime but not stored. + """ super(Configs, self).__init__(input_dict) @@ -60,15 +66,13 @@ def __init__( self.project_metadata_path: Path def setup_after_load(self) -> None: - """Setup the config after loading it.""" + """Set up the config after loading it.""" load_configs.convert_str_and_pathlib_paths(self, "str_to_path") self.ensure_local_and_central_path_end_in_project_name() self.check_dict_values_raise_on_fail() def ensure_local_and_central_path_end_in_project_name(self): - """Ensure that the local and central path end in the name of - the project. - """ + """Ensure that the local and central path end in the name of the project.""" for path_type in ["local_path", "central_path"]: if path_type == "central_path" and self[path_type] is None: continue @@ -82,8 +86,7 @@ def ensure_local_and_central_path_end_in_project_name(self): self[path_type] = self[path_type] / self.project_name def check_dict_values_raise_on_fail(self) -> None: - """Check the values of the current dictionary are set - correctly and will not cause downstream errors. + """Validate dictionary values against canonical config requirements. This will raise an error if the dictionary does not match the canonical keys and value types. @@ -117,9 +120,8 @@ def dump_to_file(self) -> None: def load_from_file(self) -> None: """Load a config dict saved at .yaml file. - Note this will - not automatically check the configs are valid, this - requires calling self.check_dict_values_raise_on_fail(). + Note this will not automatically check the configs are valid, + this requires calling self.check_dict_values_raise_on_fail(). """ with open(self.file_path) as config_file: config_dict = yaml.full_load(config_file) @@ -138,9 +140,10 @@ def build_project_path( sub_folders: Union[str, list], top_level_folder: TopLevelFolder, ) -> Path: - """Function for joining relative path to base dir. - If path already starts with base dir, the base - dir will not be joined. + """Build a path by joining a base directory with subfolders. + + If the path already starts with the base directory, + the base will not be joined again. Parameters ---------- @@ -177,15 +180,15 @@ def get_base_folder( base: str, top_level_folder: TopLevelFolder, ) -> Path: - """Convenience function to return the full base path. + """Return the full base path for the given top-level folder. Parameters ---------- base - base path, "local", "central" or "datashuttle" + Base path, "local", "central" or "datashuttle". top_level_folder - either "rawdata" or "derivatives" + Either "rawdata" or "derivatives". """ if base == "local": @@ -198,9 +201,9 @@ def get_base_folder( def get_rclone_config_name( self, connection_method: Optional[str] = None ) -> str: - """Convenience function to get the rclone config - name (these configs are created by datashuttle - but managed and stored by rclone). + """Generate the rclone configuration name for the project. + + These configs are created by datashuttle but managed and stored by rclone. """ if connection_method is None: connection_method = self["connection_method"] @@ -210,10 +213,11 @@ def get_rclone_config_name( def make_rclone_transfer_options( self, overwrite_existing_files: OverwriteExistingFiles, dry_run: bool ) -> Dict: - """Originally collected the relevant arguments - from configs. Now, all are passed via function arguments - However, now we fix the previously configurable arguments - `show_transfer_progress` and `dry_run` here. + """Create a dictionary of rclone transfer options. + + Originally these arguments were collected from configs, but now + they are passed via function arguments. The `show_transfer_progress` + and `dry_run` options are fixed here. """ allowed_overwrite = ["never", "always", "if_source_newer"] @@ -233,7 +237,7 @@ def make_rclone_transfer_options( } def init_paths(self) -> None: - """Initiate the datashuttle paths.""" + """Initialize paths used by datashuttle.""" self.project_metadata_path = self["local_path"] / ".datashuttle" datashuttle_path, _ = canonical_folders.get_project_datashuttle_path( @@ -249,8 +253,9 @@ def init_paths(self) -> None: def make_and_get_logging_path( self, ) -> Path: - """Build (and create if does not exist) the path where - logs are stored. + """Build and return the path where logs are stored. + + Create the directory if it does not already exist. """ logging_path = self.project_metadata_path / "logs" folders.create_folders(logging_path) @@ -259,8 +264,9 @@ def make_and_get_logging_path( def get_datatype_as_dict_items( self, datatype: Union[str, list] ) -> Union[ItemsView, zip]: - """Get the .items() structure of the datatype, either all of - the canonical datatypes or as a single item. + """Return canonical datatypes as dictionary items. + + Returns all datatype items or a subset if specified. """ if isinstance(datatype, str): datatype = [datatype] @@ -280,7 +286,9 @@ def get_datatype_as_dict_items( return items def is_local_project(self): - """A project is 'local-only' if it has no `central_path` and `connection_method`. + """Check if project is a local-only project. + + A project is 'local-only' if it has no `central_path` and `connection_method`. It can be used to make folders and validate, but not for transfer. """ canonical_configs.raise_on_bad_local_only_project_configs(self) From 29a94e8fe1604da726a941e3090f2f0bc7d66834 Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Sun, 22 Jun 2025 01:49:18 +0100 Subject: [PATCH 31/70] Update canonical_folders.py --- datashuttle/configs/canonical_folders.py | 34 +++++++++--------------- datashuttle/configs/canonical_tags.py | 7 +++-- 2 files changed, 16 insertions(+), 25 deletions(-) diff --git a/datashuttle/configs/canonical_folders.py b/datashuttle/configs/canonical_folders.py index c2c90065c..da4c92a65 100644 --- a/datashuttle/configs/canonical_folders.py +++ b/datashuttle/configs/canonical_folders.py @@ -11,8 +11,7 @@ def get_datatype_folders() -> dict: - """Holds the canonical folders - managed by datashuttle. + """Return the canonical folders managed by datashuttle. Notes ----- @@ -23,7 +22,7 @@ def get_datatype_folders() -> dict: kept in case this changes. The value is a Folder() class instance with - the required fields + the required fields. Parameters ---------- @@ -44,8 +43,9 @@ def get_datatype_folders() -> dict: def get_non_sub_names() -> List[str]: - """Get all arguments that are not allowed at the - subject level for data transfer, i.e. as sub_names. + """Return all arguments that are not allowed at the subject level. + + These are invalid as `sub_names` for data transfer. """ return [ "all_ses", @@ -56,9 +56,7 @@ def get_non_sub_names() -> List[str]: def get_non_ses_names() -> List[str]: - """Get all arguments that are not allowed at the - session level for data transfer, i.e. as ses_names. - """ + """Return all arguments that are not allowed at the session level.""" return [ "all_sub", "all_non_sub", @@ -68,32 +66,26 @@ def get_non_ses_names() -> List[str]: def canonical_reserved_keywords() -> List[str]: - """Key keyword arguments that are passed to `sub_names` or - `ses_names`. - """ + """Return key keyword arguments passed to `sub_names` or `ses_names`.""" return get_non_sub_names() + get_non_ses_names() def get_top_level_folders() -> List[TopLevelFolder]: - """Return a list of the different top level folder names.""" + """Return a list of canonical top level folder names.""" return ["rawdata", "derivatives"] def get_datashuttle_path() -> Path: - """Get the datashuttle path where all project - configs are stored. - """ + """Return the datashuttle path where all project configs are stored.""" return Path.home() / ".datashuttle" def get_project_datashuttle_path(project_name: str) -> Tuple[Path, Path]: - """Get the datashuttle path for the project, - where configuration files are stored. - Also, return a temporary path in this for logging in - some cases where local_path location is not clear. + """Return the datashuttle config path for the project. - The datashuttle configuration path is stored in the user home - folder. + Also returns a temporary logging path used in cases when + the `local_path` is not yet defined. The base configuration + path is the user home directory. """ base_path = get_datashuttle_path() / project_name temp_logs_path = base_path / "temp_logs" diff --git a/datashuttle/configs/canonical_tags.py b/datashuttle/configs/canonical_tags.py index 7cc274dd0..ee7295665 100644 --- a/datashuttle/configs/canonical_tags.py +++ b/datashuttle/configs/canonical_tags.py @@ -1,8 +1,7 @@ def tags(tag_name: str) -> str: - """Centralised function to get the tags used - in subject / session name processing. If changing - the formatting of these tags, it is only required - to change the dict values here. + """Return the formatting tag used for subject/session name parsing. + + If changing the formatting of these tags, update the dict values here. """ tags = { "date": "@DATE@", From 270abfbcb43044ce8fee3917d2ea08603368ebd9 Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Sun, 22 Jun 2025 01:57:03 +0100 Subject: [PATCH 32/70] Update canonical_configs.py --- datashuttle/configs/canonical_configs.py | 79 +++++++++++------------- 1 file changed, 37 insertions(+), 42 deletions(-) diff --git a/datashuttle/configs/canonical_configs.py b/datashuttle/configs/canonical_configs.py index f8700a610..bdd0ebeb9 100644 --- a/datashuttle/configs/canonical_configs.py +++ b/datashuttle/configs/canonical_configs.py @@ -1,11 +1,10 @@ -"""Contains all information for the required -format of the configs class. This is clearly defined -as configs can be provided from file or input dynamically -and so careful checks must be done. - -If adding a new config, first add the key to -get_canonical_configs() and type to -get_canonical_configs() +"""Contains all information defining the required format of the Configs class. + +This format is clearly specified because configs can be supplied +either from a file or dynamically, so careful validation is required. + +If adding a new config key: +- First add the key to `get_canonical_configs()` and define its type in the same function """ from __future__ import annotations @@ -30,9 +29,7 @@ def get_canonical_configs() -> dict: - """The only permitted types for DataShuttle - config values. - """ + """Define the only permitted types for DataShuttle config values.""" canonical_configs = { "local_path": Union[str, Path], "central_path": Optional[Union[str, Path]], @@ -45,9 +42,9 @@ def get_canonical_configs() -> dict: def keys_str_on_file_but_path_in_class() -> list[str]: - """All configs which are paths are converted to pathlib.Path - objects on load. This list indicates which config entries - are to be converted to Path. + """List all config keys that are paths but stored as str in the file. + + These are converted to pathlib.Path objects when loaded. """ return [ "local_path", @@ -61,12 +58,12 @@ def keys_str_on_file_but_path_in_class() -> list[str]: def check_dict_values_raise_on_fail(config_dict: Configs) -> None: - """Central function for performing checks on a - DataShuttle Configs UserDict class. This should - be run after any change to the configs (e.g. - make_config_file, update_config_file, supply_config_file). + """Perform checks on a DataShuttle Configs UserDict class. + + This should be run after any change to the configs + (e.g. make_config_file, update_config_file, supply_config_file). - This will raise assert if condition is not met. + This will raise an error if a condition is not met. Parameters ---------- @@ -139,9 +136,10 @@ def check_dict_values_raise_on_fail(config_dict: Configs) -> None: def raise_on_bad_local_only_project_configs(config_dict: Configs) -> None: - """There is no circumstance where one of `central_path` and `connection_method` - should be set and not the other. Either both are set ('full' project) or - neither are ('local only' project). Check this assumption here. + """Check that both or neither of `central_path` and `connection_method` are set. + + There is no circumstance where one is set and not the other. Either both are set + ('full' project) or both are `None` ('local only' project). """ params_are_none = local_only_configs_are_none(config_dict) @@ -155,9 +153,7 @@ def raise_on_bad_local_only_project_configs(config_dict: Configs) -> None: def local_only_configs_are_none(config_dict: Configs) -> list[bool]: - """Check if the central_path and connection_method config options - are set to `None`. - """ + """Check whether `central_path` and `connection_method` are both set to None.""" return [ config_dict[key] is None for key in ["central_path", "connection_method"] @@ -168,9 +164,7 @@ def raise_on_bad_path_syntax( path_name: str, path_type: str, ) -> None: - """Error if some common, unsupported patterns are observed - (e.g. ~, .) for path. - """ + """Raise error if path contains unsupported patterns (e.g. ~, .).""" if path_name[0] == "~": utils.log_and_raise_error( f"{path_type} must contain the full folder path with no ~ syntax.", @@ -210,13 +204,12 @@ def check_config_types(config_dict: Configs) -> None: def get_tui_config_defaults() -> Dict: - """Get the default settings for the datatype checkboxes - in the TUI. + """Return the default settings for the datatype checkboxes in the TUI. - Two sets are maintained (one for creating, - one for transfer) which have different defaults. + Two sets are maintained (one for checkboxes on the create tab, + the other for transfer tab) which have different defaults. By default, all broad datatype checkboxes are displayed, - and narrow are turned off. + and narrow datatypes are hidden and turned off. """ settings = { "tui": { @@ -268,13 +261,11 @@ def get_name_templates_defaults() -> Dict: def get_persistent_settings_defaults() -> Dict: - """Persistent settings are settings that are maintained - across sessions. Currently, persistent settings for - both the API and TUI are stored in the same place. + """Return the default persistent settings maintained across sessions. - Currently, settings for the working top level folder, - TUI checkboxes and name templates (i.e. regexp - validation for sub and ses names) are stored. + Currently, these include settings for both the API and TUI, such as the + working top level folder, TUI checkboxes, and name templates + (i.e. regexp validation for sub and ses names). """ settings = {} settings.update(get_tui_config_defaults()) @@ -298,6 +289,7 @@ def get_broad_datatypes(): def get_narrow_datatypes(): """Return the narrow datatype associated with each broad datatype. + The mapping between broad and narrow datatypes is required for validation. """ return { @@ -328,8 +320,9 @@ def get_narrow_datatypes(): def quick_get_narrow_datatypes(): - """A convenience wrapper around `get_narrow_datatypes()` - to quickly get a list of all narrow datatypes. + """Return a flat list of all narrow datatypes. + + This is a convenience wrapper around `get_narrow_datatypes()`. """ all_narrow_datatypes = get_narrow_datatypes() top_level_keys = list(all_narrow_datatypes.keys()) @@ -342,7 +335,9 @@ def quick_get_narrow_datatypes(): def in_place_update_narrow_datatypes_if_required(user_settings: dict): - """In versions < v0.6.0, only 'broad' datatypes were implemented + """Update legacy settings with the new version format. + + In versions < v0.6.0, only 'broad' datatypes were implemented and available in the TUI. Since, 'narrow' datatypes are introduced and datatype tui can be set to be both on / off but also displayed / not displayed. From 91a11100afb709b7e1b6fbd4b6a52f4799b1f01f Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Sun, 22 Jun 2025 01:58:07 +0100 Subject: [PATCH 33/70] Update load_configs.py --- datashuttle/configs/load_configs.py | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/datashuttle/configs/load_configs.py b/datashuttle/configs/load_configs.py index edcd4b6a9..3b1c00d1c 100644 --- a/datashuttle/configs/load_configs.py +++ b/datashuttle/configs/load_configs.py @@ -17,21 +17,21 @@ def attempt_load_configs( config_path: Path, verbose: bool = True, ) -> Optional[Configs]: - """Try to load an existing config file, that was previously - saved by Datashuttle. This should always work, unless - not already initialised (prompt) or these have been - changed manually. + """Try to load an existing config file previously saved by Datashuttle. + + This should always work unless the config is not initialized or has been + manually changed. Parameters ---------- project_name - name of project + Name of the project. config_path - path to datashuttle config .yaml file + Path to the datashuttle config .yaml file. verbose - warnings and error messages will be printed. + If True, warnings and error messages will be printed. """ exists = config_path.is_file() @@ -68,16 +68,18 @@ def attempt_load_configs( def convert_str_and_pathlib_paths( config_dict: Union["Configs", dict], direction: str ) -> None: - """Config paths are stored as str in the .yaml but used as Path - in the module, so make the conversion here. + """Convert config paths between strings and pathlib.Path objects. + + Paths are stored as strings in the .yaml file but used as Path objects in + the module. This function performs the conversion. Parameters ---------- config_dict - DataShuttle.cfg dict of configs + DataShuttle.cfg dict of configs. direction - "path_to_str" or "str_to_path" + Direction of conversion: "path_to_str" or "str_to_path". """ for path_key in canonical_configs.keys_str_on_file_but_path_in_class(): From 6aff334eaaa389b708f1c794ef47e0ab09ab6994 Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Sun, 22 Jun 2025 02:00:37 +0100 Subject: [PATCH 34/70] Update load_configs.py --- datashuttle/datashuttle_functions.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/datashuttle/datashuttle_functions.py b/datashuttle/datashuttle_functions.py index 39c154595..eb3522a77 100644 --- a/datashuttle/datashuttle_functions.py +++ b/datashuttle/datashuttle_functions.py @@ -29,11 +29,7 @@ def quick_validate_project( strict_mode: bool = False, name_templates: Optional[Dict] = None, ) -> List[str]: - """Perform validation on the project. - - This checks the subject - and session level folders to ensure there are not - NeuroBlueprint formatting issues. + """Perform validation on a NeuroBlueprint-formatted project. Parameters ---------- @@ -46,11 +42,11 @@ def quick_validate_project( perform validation. If `None`, both are checked. display_mode - The validation issues are displayed as ``"error"`` (raise error) - ``"warn"`` (show warning) or ``"print"``. + The validation issues are displayed as ``"error"`` (raise error), + ``"warn"`` (show warning), or ``"print"``. strict_mode - If `True`, only allow NeuroBlueprint-formatted folders to exist in + If ``True``, only allow NeuroBlueprint-formatted folders to exist in the project. By default, non-NeuroBlueprint folders (e.g. a folder called 'my_stuff' in the 'rawdata') are allowed, and only folders starting with sub- or ses- prefix are checked. In `Strict Mode`, @@ -105,8 +101,14 @@ def _format_top_level_folder( """Format the top level folder. Take a `top_level_folder` ("rawdata" or "derivatives" str) and - convert to list, if `None`, convert it to a list + convert it to a list. If `None`, convert it to a list of both possible top-level folders. + + Parameters + ---------- + top_level_folder + The top-level folder to format. Can be "rawdata", "derivatives", or None. + """ rawdata_and_derivatives: List[TopLevelFolder] = ["rawdata", "derivatives"] From 398b27974cb3609b6447415c35f39717dd240c84 Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Sun, 22 Jun 2025 04:04:55 +0100 Subject: [PATCH 35/70] Update datashuttle_class.py --- datashuttle/datashuttle_class.py | 436 ++++++++++++++----------------- 1 file changed, 192 insertions(+), 244 deletions(-) diff --git a/datashuttle/datashuttle_class.py b/datashuttle/datashuttle_class.py index 764cedc7e..23e7b42aa 100644 --- a/datashuttle/datashuttle_class.py +++ b/datashuttle/datashuttle_class.py @@ -62,42 +62,15 @@ class DataShuttle: - """DataShuttle is a tool for convenient scientific - project management and data transfer in BIDS format. - - The expected organisation is a central repository - on a central machine ('central') that contains all - project data. This is connected to multiple local - machines ('local'). These can each contain a subset of - the full project (e.g. machine for electrophysiology - collection, machine for behavioural collection). - - On first use on a new profile, show warning prompting - to set configurations with the function make_config_file(). - - Datashuttle will save logs to a .datashuttle folder - in the main local project. These logs contain - detailed information on folder creation / transfer. - To get the path to datashuttle logs, use - cfgs.make_and_get_logging_path(). - - For transferring data between a central data storage - with SSH, use setup setup_ssh_connection(). - This will allow you to check the server key, add host key to - profile if accepted, and setup ssh key pair. - """ + """DataShuttle is a tool for neuroscience project management and data transfer.""" def __init__(self, project_name: str, print_startup_message: bool = True): - """Parameters + """Initialise ``DataShuttle``. + + Parameters ---------- project_name - The project name to use the datashuttle - Folders containing all project files - and folders are specified in make_config_file(). - Datashuttle-related files are stored in - a .datashuttle folder in the user home - folder. Use get_datashuttle_path() to - see the path to this folder. + The project name. print_startup_message If `True`, a start-up message displaying the @@ -131,9 +104,7 @@ def __init__(self, project_name: str, print_startup_message: bool = True): rclone.prompt_rclone_download_if_does_not_exist() def _set_attributes_after_config_load(self) -> None: - """Once config file is loaded, update all private attributes - according to config contents. - """ + """Update all private attributes according to config contents.""" self.cfg.init_paths() self._make_project_metadata_if_does_not_exist() @@ -152,17 +123,15 @@ def create_folders( bypass_validation: bool = False, log: bool = True, ) -> Dict[str, List[Path]]: - """Create a subject / session folder tree in the project - folder. The passed subject / session names are - formatted and validated. If this succeeds, fully - validation against all subject / session folders in - the local project is performed before making the - folders. + """Create a folder tree in the project folder. + + The passed names are initially formatted and validated, + then folders are created. Parameters ---------- top_level_folder - Whether to make the folders in `rawdata` or + Whether to make the folders within `rawdata` or `derivatives`. sub_names @@ -172,7 +141,7 @@ def create_folders( "sub-") ses_names - (Optional). session name / list of session names. + session name / list of session names. (if not already, these will be prefixed with "ses-"). If no session is provided, no session-level folders are made. @@ -181,8 +150,7 @@ def create_folders( The datatype to make in the sub / ses folders. (e.g. "ephys", "behav", "anat"). If "" is passed no datatype will be created. Broad or - Narrow canonical NeuroBlueprint datatypes are - accepted. + Narrow NeuroBlueprint datatypes are accepted. bypass_validation If `True`, folders will be created even if they are not @@ -284,10 +252,7 @@ def _format_and_validate_names( bypass_validation: bool, log: bool = True, ) -> Tuple[List[str], List[str]]: - """A central method for the formatting and validation of subject / session - names for folder creation. This is called by both DataShuttle and - during TUI validation. - """ + """Central method to format and validate subject and session names.""" format_sub = formatting.check_and_format_names( sub_names, "sub", name_templates, bypass_validation ) @@ -329,47 +294,42 @@ def upload_custom( dry_run: bool = False, init_log: bool = True, ) -> None: - """Upload data from a local project to the central project - folder. In the case that a file / folder exists on - the central and local, the central will not be overwritten - even if the central file is an older version. Data - transfer logs are saved to the logging folder). + """Upload data from a local project to the central project folder. Parameters ---------- top_level_folder - The top-level folder (e.g. `"rawdata"`, `"derivatives"`) to transfer files - and folders within. + The top-level folder (e.g. `"rawdata"`, `"derivatives"`) to transfer within. sub_names - a subject name / list of subject names. These must - be prefixed with "sub-", or the prefix will be - automatically added. "@*@" can be used as a wildcard. + A subject name / list of subject names. These must + be prefixed with ``"sub-"``, or the prefix will be + automatically added. ``"@*@"`` can be used as a wildcard. "all" will search for all sub-folders in the datatype folder to upload. ses_names - a session name / list of session names, similar to - sub_names but requiring a "ses-" prefix. + A session name / list of session names, similar to + sub_names but requiring a ``"ses-"`` prefix. datatype The (broad or narrow) NeuroBlueprint datatypes to transfer. - If "all", any broad or narrow datatype folder will be transferred. + If ``"all"``, any broad or narrow datatype folder will be transferred. overwrite_existing_files - If `False`, files on central will never be overwritten - by files transferred from local. If `True`, central files - will be overwritten if there is any difference (date, size) - between central and local files. + If ``"never"`` files on target will never be overwritten by source. + If ``"always"`` files on target will be overwritten by source if + there is any difference in date or size. + If ``"if_source_newer"`` files on target will only be overwritten + by files on source with newer creation / modification datetime. dry_run - perform a dry-run of transfer. This will output as if file - transfer was taking place, but no files will be moved. Useful - to check which files will be moved on data transfer. + Perform a dry-run of transfer. This will output as if file + transfer was taking place, but no files will be moved. init_log - (Optional). Whether to handle logging. This should - always be True, unless logger is handled elsewhere + Whether to handle logging. This should + always be ``True``, unless logger is handled elsewhere (e.g. in a calling function). """ @@ -405,44 +365,42 @@ def download_custom( dry_run: bool = False, init_log: bool = True, ) -> None: - """Download data from the central project folder to the - local project folder. + """Download data from the central project to the local project folder. Parameters ---------- top_level_folder - The top-level folder (e.g. `rawdata`) to transfer files - and folders within. + The top-level folder (e.g. `"rawdata"`, `"derivatives"`) to transfer within. sub_names - a subject name / list of subject names. These must - be prefixed with "sub-", or the prefix will be - automatically added. "@*@" can be used as a wildcard. + A subject name / list of subject names. These must + be prefixed with ``"sub-"``, or the prefix will be + automatically added. ``"@*@"`` can be used as a wildcard. "all" will search for all sub-folders in the datatype folder to upload. ses_names - a session name / list of session names, similar to - sub_names but requiring a "ses-" prefix. + A session name / list of session names, similar to + sub_names but requiring a ``"ses-"`` prefix. datatype - see create_folders() + The (broad or narrow) NeuroBlueprint datatypes to transfer. + If ``"all"``, any broad or narrow datatype folder will be transferred. overwrite_existing_files - If "never" files on target will never be overwritten by source. - If "always" files on target will be overwritten by source if + If ``"never"`` files on target will never be overwritten by source. + If ``"always"`` files on target will be overwritten by source if there is any difference in date or size. - If "if_source_newer" files on target will only be overwritten + If ``"if_source_newer"`` files on target will only be overwritten by files on source with newer creation / modification datetime. dry_run - perform a dry-run of transfer. This will output as if file - transfer was taking place, but no files will be moved. Useful - to check which files will be moved on data transfer. + Perform a dry-run of transfer. This will output as if file + transfer was taking place, but no files will be moved. init_log - (Optional). Whether to handle logging. This should - always be True, unless logger is handled elsewhere + Whether to handle logging. This should + always be ``True``, unless logger is handled elsewhere (e.g. in a calling function). """ @@ -478,21 +436,20 @@ def upload_rawdata( overwrite_existing_files: OverwriteExistingFiles = "never", dry_run: bool = False, ): - """Upload files in the `rawdata` top level folder. + """Upload all files in the `rawdata` top level folder. Parameters ---------- overwrite_existing_files - If "never" files on target will never be overwritten by source. - If "always" files on target will be overwritten by source if + If ``"never"`` files on target will never be overwritten by source. + If ``"always"`` files on target will be overwritten by source if there is any difference in date or size. - If "if_source_newer" files on target will only be overwritten + If ``"if_source_newer"`` files on target will only be overwritten by files on source with newer creation / modification datetime. dry_run - perform a dry-run of transfer. This will output as if file - transfer was taking place, but no files will be moved. Useful - to check which files will be moved on data transfer. + Perform a dry-run of transfer. This will output as if file + transfer was taking place, but no files will be moved. """ self._transfer_top_level_folder( @@ -509,21 +466,20 @@ def upload_derivatives( overwrite_existing_files: OverwriteExistingFiles = "never", dry_run: bool = False, ): - """Upload files in the `derivatives` top level folder. + """Upload all files in the `derivatives` top level folder. Parameters ---------- overwrite_existing_files - If "never" files on target will never be overwritten by source. - If "always" files on target will be overwritten by source if + If ``"never"`` files on target will never be overwritten by source. + If ``"always"`` files on target will be overwritten by source if there is any difference in date or size. - If "if_source_newer" files on target will only be overwritten + If ``"if_source_newer"`` files on target will only be overwritten by files on source with newer creation / modification datetime. dry_run - perform a dry-run of transfer. This will output as if file - transfer was taking place, but no files will be moved. Useful - to check which files will be moved on data transfer. + Perform a dry-run of transfer. This will output as if file + transfer was taking place, but no files will be moved. """ self._transfer_top_level_folder( @@ -540,21 +496,20 @@ def download_rawdata( overwrite_existing_files: OverwriteExistingFiles = "never", dry_run: bool = False, ): - """Download files in the `rawdata` top level folder. + """Download all files in the `rawdata` top level folder. Parameters ---------- overwrite_existing_files - If "never" files on target will never be overwritten by source. - If "always" files on target will be overwritten by source if + If ``"never"`` files on target will never be overwritten by source. + If ``"always"`` files on target will be overwritten by source if there is any difference in date or size. - If "if_source_newer" files on target will only be overwritten + If ``"if_source_newer"`` files on target will only be overwritten by files on source with newer creation / modification datetime. dry_run - perform a dry-run of transfer. This will output as if file - transfer was taking place, but no files will be moved. Useful - to check which files will be moved on data transfer. + Perform a dry-run of transfer. This will output as if file + transfer was taking place, but no files will be moved.. """ self._transfer_top_level_folder( @@ -571,21 +526,20 @@ def download_derivatives( overwrite_existing_files: OverwriteExistingFiles = "never", dry_run: bool = False, ): - """Download files in the `derivatives` top level folder. + """Download all files in the `derivatives` top level folder. Parameters ---------- overwrite_existing_files - If "never" files on target will never be overwritten by source. - If "always" files on target will be overwritten by source if + If ``"never"`` files on target will never be overwritten by source. + If ``"always"`` files on target will be overwritten by source if there is any difference in date or size. - If "if_source_newer" files on target will only be overwritten + If ``"if_source_newer"`` files on target will only be overwritten by files on source with newer creation / modification datetime. dry_run - perform a dry-run of transfer. This will output as if file - transfer was taking place, but no files will be moved. Useful - to check which files will be moved on data transfer. + Perform a dry-run of transfer. This will output as if file + transfer was taking place, but no files will be moved. """ self._transfer_top_level_folder( @@ -602,23 +556,22 @@ def upload_entire_project( overwrite_existing_files: OverwriteExistingFiles = "never", dry_run: bool = False, ) -> None: - """Upload the entire project (from 'local' to 'central'), - i.e. including every top level folder (e.g. 'rawdata', - 'derivatives', 'code', 'analysis'). + """Upload the entire project. + + Includes every top level folder (e.g. ``rawdata``, ``derivatives``). Parameters ---------- overwrite_existing_files - If "never" files on target will never be overwritten by source. - If "always" files on target will be overwritten by source if + If ``"never"`` files on target will never be overwritten by source. + If ``"always"`` files on target will be overwritten by source if there is any difference in date or size. - If "if_source_newer" files on target will only be overwritten + If ``"if_source_newer"`` files on target will only be overwritten by files on source with newer creation / modification datetime. dry_run - perform a dry-run of transfer. This will output as if file - transfer was taking place, but no files will be moved. Useful - to check which files will be moved on data transfer. + Perform a dry-run of transfer. This will output as if file + transfer was taking place, but no files will be moved. """ self._start_log("upload-entire-project", local_vars=locals()) @@ -634,23 +587,22 @@ def download_entire_project( overwrite_existing_files: OverwriteExistingFiles = "never", dry_run: bool = False, ) -> None: - """Download the entire project (from 'central' to 'local'), - i.e. including every top level folder (e.g. 'rawdata', - 'derivatives', 'code', 'analysis'). + """Download the entire project. + + Includes every top level folder (e.g. ``rawdata``, ``derivatives``). Parameters ---------- overwrite_existing_files - If "never" files on target will never be overwritten by source. - If "always" files on target will be overwritten by source if + If ``"never"`` files on target will never be overwritten by source. + If ``"always"`` files on target will be overwritten by source if there is any difference in date or size. - If "if_source_newer" files on target will only be overwritten + If ``"if_source_newer"`` files on target will only be overwritten by files on source with newer creation / modification datetime. dry_run - perform a dry-run of transfer. This will output as if file - transfer was taking place, but no files will be moved. Useful - to check which files will be moved on data transfer. + Perform a dry-run of transfer. This will output as if file + transfer was taking place, but no files will be moved. """ self._start_log("download-entire-project", local_vars=locals()) @@ -667,12 +619,11 @@ def upload_specific_folder_or_file( overwrite_existing_files: OverwriteExistingFiles = "never", dry_run: bool = False, ) -> None: - """Upload a specific file or folder. If transferring - a single file, the path including the filename is - required (see 'filepath' input). If a folder, - wildcards "*" or "**" must be used to transfer - all files in the folder ("*") or all files - and sub-folders ("**"). + """Upload a specific file or folder. + + If transferring a single file, the path including the filename is + required (see 'filepath' input). If a folder, wildcards "*" or "**" must be used to transfer + all files in the folder ("*") or all files and sub-folders ("**"). Parameters ---------- @@ -680,16 +631,15 @@ def upload_specific_folder_or_file( a string containing the full filepath. overwrite_existing_files - If "never" files on target will never be overwritten by source. - If "always" files on target will be overwritten by source if + If ``"never"`` files on target will never be overwritten by source. + If ``"always"`` files on target will be overwritten by source if there is any difference in date or size. - If "if_source_newer" files on target will only be overwritten + If ``"if_source_newer"`` files on target will only be overwritten by files on source with newer creation / modification datetime. dry_run - perform a dry-run of transfer. This will output as if file - transfer was taking place, but no files will be moved. Useful - to check which files will be moved on data transfer. + Perform a dry-run of transfer. This will output as if file + transfer was taking place, but no files will be moved. """ self._start_log("upload-specific-folder-or-file", local_vars=locals()) @@ -708,11 +658,11 @@ def download_specific_folder_or_file( overwrite_existing_files: OverwriteExistingFiles = "never", dry_run: bool = False, ) -> None: - """Download a specific file or folder. If transferring - a single file, the path including the filename is - required (see 'filepath' input). If a folder, - wildcards "*" or "**" must be used to transfer - all files in the folder ("*") or all files + """Download a specific file or folder. + + If transferring a single file, the path including the filename is + required (see 'filepath' input). If a folder, wildcards "*" or "**" + must be used to transfer all files in the folder ("*") or all files and sub-folders ("**"). Parameters @@ -721,16 +671,15 @@ def download_specific_folder_or_file( a string containing the full filepath. overwrite_existing_files - If "never" files on target will never be overwritten by source. - If "always" files on target will be overwritten by source if + If ``"never"`` files on target will never be overwritten by source. + If ``"always"`` files on target will be overwritten by source if there is any difference in date or size. - If "if_source_newer" files on target will only be overwritten + If ``"if_source_newer"`` files on target will only be overwritten by files on source with newer creation / modification datetime. dry_run - perform a dry-run of transfer. This will output as if file - transfer was taking place, but no files will be moved. Useful - to check which files will be moved on data transfer. + Perform a dry-run of transfer. This will output as if file + transfer was taking place, but no files will be moved. """ self._start_log( @@ -751,8 +700,10 @@ def _transfer_top_level_folder( dry_run: bool = False, init_log: bool = True, ): - """Core function to upload / download files within a - particular top-level-folder. e.g. `upload_rawdata()`. + """Upload or download files within a particular top-level-folder. + + A centralised function to upload or download data within + a particular top level folder (e.g. ``rawdata``, ``derivatives``). """ if init_log: self._start_log( @@ -822,10 +773,10 @@ def _transfer_specific_file_or_folder( @requires_ssh_configs @check_is_not_local_project def setup_ssh_connection(self) -> None: - """Setup a connection to the central server using SSH. + """Set up a connection to the central server using SSH. + Assumes the central_host_id and central_host_username are set in configs (see make_config_file() and update_config_file()). - First, the server key will be displayed, requiring verification of the server ID. This will store the hostkey for all future use. @@ -853,14 +804,16 @@ def setup_ssh_connection(self) -> None: @requires_ssh_configs @check_is_not_local_project def write_public_key(self, filepath: str) -> None: - """By default, the SSH private key only is stored, in - the datashuttle configs folder. Use this function - to save the public key. + """Save the public SSH key to a specified filepath. + + By default, only the SSH private key is stored in the + datashuttle configs folder. Use this function to save + the public key. Parameters ---------- filepath - full filepath (inc filename) to write the + Full filepath (including filename) to write the public key to. """ @@ -885,17 +838,17 @@ def make_config_file( central_host_id: Optional[str] = None, central_host_username: Optional[str] = None, ) -> None: - """Initialise the configurations for datashuttle to use on the - local machine. Once initialised, these settings will be - used each time the datashuttle is opened. This method - can also be used to completely overwrite existing configs. + """Initialize the configurations for datashuttle on the local machine. + + Once initialised, these settings will be used each + time the datashuttle is opened. These settings are stored in a config file on the - datashuttle path (not in the project folder) - on the local machine. Use get_config_path() to - get the full path to the saved config file. + datashuttle path (not in the project folder) on the + local machine. Use ``get_config_path()`` to get the + full path to the saved config file. - Use update_config_file() to selectively update settings. + Use ``update_config_file()`` to selectively update settings. Parameters ---------- @@ -904,25 +857,25 @@ def make_config_file( central_path Filepath to central project. - If this is local (i.e. connection_method = "local_filesystem"), + If this is local (i.e. ``connection_method = "local_filesystem"``), this is the full path on the local filesystem - Otherwise, if this is via ssh (i.e. connection method = "ssh"), + Otherwise, if this is via ssh (i.e. ``connection method = "ssh"``), this is the path to the project folder on central machine. This should be a full path to central folder i.e. this cannot include ~ home folder syntax, must contain the full path - (e.g. /nfs/nhome/live/jziminski) + (e.g. ``/nfs/nhome/live/jziminski``) connection_method The method used to connect to the central project filesystem, - e.g. "local_filesystem" (e.g. mounted drive) or "ssh" + e.g. ``"local_filesystem"`` (e.g. mounted drive) or ``"ssh"`` central_host_id server address for central host for ssh connection - e.g. "ssh.swc.ucl.ac.uk" + e.g. ``"ssh.swc.ucl.ac.uk"`` central_host_username username for which to log in to central host. - e.g. "jziminski" + e.g. ``"jziminski"`` """ self._start_log( @@ -1009,9 +962,9 @@ def get_central_path(self) -> Path: return self.cfg["central_path"] def get_datashuttle_path(self) -> Path: - """Get the path to the local datashuttle - folder where configs and other - datashuttle files are stored. + """Return the path to the local datashuttle folder. + + This is where configs and other datashuttle files are stored. """ return self._datashuttle_path @@ -1033,6 +986,7 @@ def get_logging_path(self) -> Path: @staticmethod def get_existing_projects() -> List[Path]: """Get a list of existing project names found on the local machine. + This is based on project folders in the "home / .datashuttle" folder that contain valid config.yaml files. """ @@ -1045,7 +999,7 @@ def get_next_sub( return_with_prefix: bool = True, include_central: bool = False, ) -> str: - """Convenience function for `get_next_sub_or_ses` to find the next subject number. + """Return the next subject number. Parameters ---------- @@ -1087,8 +1041,7 @@ def get_next_ses( return_with_prefix: bool = True, include_central: bool = False, ) -> str: - """Convenience function for get_next_sub_or_ses - to find the next session number. + """Return the next session number. Parameters ---------- @@ -1102,8 +1055,8 @@ def get_next_ses( If `True`, return with the "ses-" prefix. include_central - If `False, only get names from `local_path`, otherwise from - `local_path` and `central_path`. If in local-project mode, + If ``False``, only get names from ``local_path``, otherwise from + ``local_path`` and ``central_path``. If in local-project mode, this flag is ignored. """ @@ -1127,7 +1080,9 @@ def get_next_ses( @check_configs_set def is_local_project(self) -> bool: - """A project is 'local-only' if it has no `central_path` and `connection_method`. + """Return a bool indicating whether the project is 'local only'. + + A project is 'local-only' if it has no ``central_path`` and ``connection_method``. It can be used to make folders and validate, but not for transfer. """ return self.cfg.is_local_project() @@ -1136,8 +1091,9 @@ def is_local_project(self) -> bool: # ------------------------------------------------------------------------- def get_name_templates(self) -> Dict: - """Get the regexp templates used for validation. If - the "on" key is set to `False`, template validation is not performed. + """Return the regexp templates used for validation. + + If the "on" key is set to `False`, template validation is not performed. Returns ------- @@ -1151,15 +1107,15 @@ def get_name_templates(self) -> Dict: def set_name_templates(self, new_name_templates: Dict) -> None: """Update the persistent settings with new name templates. - Name templates are regexp for that, when name_templates["on"] is - set to `True`, "sub" and "ses" names are validated against + Name templates are regexp for that, when ``name_templates["on"]`` is + set to ``True``, ``"sub"`` and ``"ses"`` names are validated against the regexp contained in the dict. Parameters ---------- new_name_templates - e.g. {"name_templates": {"on": False, "sub": None, "ses": None}} - where "sub" or "ses" can be a regexp that subject and session + e.g. ``{"name_templates": {"on": False, "sub": None, "ses": None}}`` + where ``"sub"`` or ``"ses"`` can be a regexp that subject and session names respectively are validated against. """ @@ -1186,14 +1142,15 @@ def validate_project( include_central: bool = False, strict_mode: bool = False, ) -> List[str]: - """Perform validation on the project. This checks the subject - and session level folders to ensure there are no NeuroBlueprint - formatting issues. + """Perform validation on the project. + + This checks the subject and session level folders to + ensure there are no NeuroBlueprint formatting issues. Parameters ---------- top_level_folder - Folder to check, either "rawdata" or "derivatives". If ``None``, + Folder to check, either ``"rawdata"`` or ``"derivatives"``. If ``None``, will check both folders. display_mode @@ -1206,10 +1163,10 @@ def validate_project( this flag is ignored. strict_mode - If `True`, only allow NeuroBlueprint-formatted folders to exist in + If ``True``, only allow NeuroBlueprint-formatted folders to exist in the project. By default, non-NeuroBlueprint folders (e.g. a folder - called 'my_stuff' in the 'rawdata') are allowed, and only folders - starting with sub- or ses- prefix are checked. In `Strict Mode`, + called '`my_stuff'` in the '`rawdata'`) are allowed, and only folders + starting with sub- or ses- prefix are checked. In ``Strict Mode``, any folder not prefixed with sub-, ses- or a valid datatype will raise a validation issue. @@ -1254,12 +1211,13 @@ def validate_project( @staticmethod def check_name_formatting(names: Union[str, list], prefix: Prefix) -> None: - """Pass list of names to check how these will be auto-formatted, - for example as when passed to create_folders() or upload_custom() - or download(). + """Format a list of subject or session names. + + Pass list of names to check how these will be auto-formatted, + for example as when passed to ``create_folders()`` or ``upload_custom()`` Useful for checking tags e.g. @TO@, @DATE@, @DATETIME@, @DATE@. - This method will print the formatted list of names, + This method will print the formatted list of names. Parameters ---------- @@ -1268,7 +1226,7 @@ def check_name_formatting(names: Union[str, list], prefix: Prefix) -> None: prefix The relevant subject or session prefix, - e.g. "sub-" or "ses-" + e.g. ``"sub-"`` or ``"ses-"`` """ if prefix not in ["sub", "ses"]: @@ -1293,23 +1251,10 @@ def _transfer_entire_project( overwrite_existing_files: OverwriteExistingFiles, dry_run: bool, ) -> None: - """Transfer (i.e. upload or download) the entire project (i.e. - every 'top level folder' (e.g. 'rawdata', 'derivatives'). - - Parameters - ---------- - upload_or_download - direction to transfer the data, either "upload" (from - local to central) or "download" (from central to local). - - overwrite_existing_files - determines whether or not to overwrite existing files - - dry_run - perform a dry-run of transfer. This will output as if file - transfer was taking place, but no files will be moved. Useful - to check which files will be moved on data transfer. + """Transfer the entire project. + i.e. every 'top level folder' (e.g. 'rawdata', 'derivatives'). + See ``upload_custom()`` or ``download_custom()`` for parameters. """ for top_level_folder in canonical_folders.get_top_level_folders(): utils.log_and_message(f"Transferring `{top_level_folder}`") @@ -1329,25 +1274,26 @@ def _start_log( store_in_temp_folder: bool = False, verbose: bool = True, ) -> None: - """Initialize the logger. This is typically called at - the start of public methods to initialize logging - for a specific function call. + """Initialize the logger. + + This is typically called at the start of public + methods to initialize logging for a specific function call. Parameters ---------- command_name - name of the command, for the log output files. + Name of the command, for the log output files. local_vars local_vars are passed to fancylog variables argument. see ds_logger.wrap_variables_for_fancylog for more info store_in_temp_folder - if `False`, existing logging path will be used + If `False`, existing logging path will be used (local project .datashuttle). verbose - print warnings and error messages. + Print warnings and error messages. """ if local_vars is None: @@ -1368,7 +1314,9 @@ def _start_log( ds_logger.start(path_to_save, command_name, variables, verbose) def _move_logs_from_temp_folder(self) -> None: - """Logs are stored within the project folder. Although + """Create a temporary logging folder when the project folder is unknown. + + Logs are stored within the project folder. Although in some instances, when setting configs, we do not know what the project folder is. In this case, make the logs in a temp folder in the .datashuttle config folder, @@ -1406,6 +1354,7 @@ def _error_on_base_project_name(self, project_name): def _log_successful_config_change(self, message: bool = False) -> None: """Log the entire config at the time of config change. + If messaged, just message "update successful" rather than print the entire configs as it becomes confusing. """ @@ -1417,15 +1366,15 @@ def _log_successful_config_change(self, message: bool = False) -> None: ) def _get_json_dumps_config(self) -> str: - """Get the config dictionary formatted as json.dumps() - which allows well formatted printing. - """ + """Get the config dictionary formatted as json.dumps() which allows well formatted printing.""" copy_dict = copy.deepcopy(self.cfg.data) load_configs.convert_str_and_pathlib_paths(copy_dict, "path_to_str") return json.dumps(copy_dict, indent=4) def _make_project_metadata_if_does_not_exist(self) -> None: - """Within the project local_path is also a .datashuttle + """Locate the .datashuttle folder within the project local_path. + + Within the project local_path is also a .datashuttle folder that contains additional information, e.g. logs. """ folders.create_folders(self.cfg.project_metadata_path, log=False) @@ -1449,8 +1398,9 @@ def _setup_rclone_central_local_filesystem_config(self) -> None: def _update_persistent_setting( self, setting_name: str, setting_value: Any ) -> None: - """Load settings that are stored persistently across datashuttle - sessions. These are stored in yaml dumped to dictionary. + """Load settings that are stored persistently across datashuttle sessions. + + These are stored in yaml dumped to dictionary. Parameters ---------- @@ -1474,9 +1424,7 @@ def _update_persistent_setting( self._save_persistent_settings(settings) def _init_persistent_settings(self) -> None: - """Initialise the default persistent settings - and save to file. - """ + """Initialise the default persistent settings and save to file.""" settings = canonical_configs.get_persistent_settings_defaults() self._save_persistent_settings(settings) @@ -1486,9 +1434,7 @@ def _save_persistent_settings(self, settings: Dict) -> None: yaml.dump(settings, settings_file, sort_keys=False) def _load_persistent_settings(self) -> Dict: - """Load settings that are stored persistently across - datashuttle sessions. - """ + """Load settings that are stored persistently across datashuttle sessions.""" if not self._persistent_settings_path.is_file(): self._init_persistent_settings() @@ -1500,8 +1446,10 @@ def _load_persistent_settings(self) -> Dict: return settings def _update_settings_with_new_canonical_keys(self, settings: Dict): - """Perform a check on the keys within persistent settings. - If they do not exist, persistent settings is from older version + """Check and update keys within persistent settings if missing. + + Perform a check on the keys within persistent settings. + If they do not exist, persistent settings is from an older version and the new keys need adding. If changing keys within the top level (e.g. a dict entry in "tui") this method will need to be extended. From 16a87257288f2aff836295627226b984c544289a Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Sun, 22 Jun 2025 04:09:25 +0100 Subject: [PATCH 36/70] Update ds_logger.py --- datashuttle/utils/ds_logger.py | 39 ++++++++++++++++++++++++++++------ 1 file changed, 33 insertions(+), 6 deletions(-) diff --git a/datashuttle/utils/ds_logger.py b/datashuttle/utils/ds_logger.py index 140ab7c54..f5dd08633 100644 --- a/datashuttle/utils/ds_logger.py +++ b/datashuttle/utils/ds_logger.py @@ -41,7 +41,24 @@ def start( variables: Optional[List[Any]], verbose: bool = True, ) -> None: - """Call fancylog to initialise logging.""" + """Call fancylog to initialise logging. + + Parameters + ---------- + path_to_log + Path to save the log file to. + + command_name + Name of the datashuttle command run, which is included + in the log filename. + + variables + Local variables to log. + + verbose + Verbosity passed to ``fancylog``. + + """ filename = get_logging_filename(command_name) fancylog.start_logging( @@ -61,9 +78,17 @@ def start( def get_logging_filename(command_name: str) -> str: - """Get the filename to which the log will be saved. This - starts with ISO8601-formatted datetime, so logs are stored - in datetime order. + """Return the log filename. + + This starts with ISO8601-formatted datetime, so logs + are stored in datetime order. + + Parameters + ---------- + command_name + Name of the datashuttle command run, which is included + in the log filename. + """ filename = datetime.now().strftime(f"%Y%m%dT%H%M%S_{command_name}") return filename @@ -87,8 +112,10 @@ def log_names(list_of_headers: List[Any], list_of_names: List[Any]) -> None: def wrap_variables_for_fancylog(local_vars: dict, cfg: Configs) -> List: - """Wrap the locals from the original function call to log - and the datashuttle.cfg in a wrapper class with __dict__ + """Wrap the locals from the original function call for fancylog. + + Fancylog will log these variables as well as the + datashuttle.cfg in a wrapper class with __dict__ attribute for fancylog writing. Delete the self attribute (which is DataShuttle class) From 6e1c1b2f4883a68aa867adaa1fdd0e3ea0e05d56 Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Sun, 22 Jun 2025 04:13:43 +0100 Subject: [PATCH 37/70] Update decorators.py --- datashuttle/utils/decorators.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/datashuttle/utils/decorators.py b/datashuttle/utils/decorators.py index 19f3794b2..6fecb6ce0 100644 --- a/datashuttle/utils/decorators.py +++ b/datashuttle/utils/decorators.py @@ -5,8 +5,10 @@ def requires_ssh_configs(func): - """Decorator to check file is loaded. Used on Mainwindow class - methods only as first arg is assumed to be self (containing cfgs). + """Check ssh configs have been set. + + Used on Mainwindow class methods only as first + arg is assumed to be self (containing cfgs). """ @wraps(func) @@ -28,10 +30,7 @@ def wrapper(*args, **kwargs): def check_configs_set(func): - """Check that configs have been loaded (i.e. - project.cfg is not None) before the - func is run. - """ + """Check configs have been set.""" @wraps(func) def wrapper(*args, **kwargs): @@ -48,8 +47,7 @@ def wrapper(*args, **kwargs): def check_is_not_local_project(func): - """Decorator to check that the project is not - a local project. If it is, raise. + """Check that the project is not a local project. This decorator should be placed above methods which require `central_path` and `connection_method` to be set. From 4bfe65fff25ed2fcd62db1fd038ace831646d2f3 Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Sun, 22 Jun 2025 04:20:46 +0100 Subject: [PATCH 38/70] Update data_transfer.py --- datashuttle/utils/data_transfer.py | 60 +++++++++++++++--------------- 1 file changed, 31 insertions(+), 29 deletions(-) diff --git a/datashuttle/utils/data_transfer.py b/datashuttle/utils/data_transfer.py index e43fd4540..5a855a50c 100644 --- a/datashuttle/utils/data_transfer.py +++ b/datashuttle/utils/data_transfer.py @@ -12,8 +12,10 @@ class TransferData: - """Class to perform data transfers. This works by first building - a large list of all files to transfer. Then, rclone is called + """Class to perform data transfers. + + This works by first building a large list of all + files to transfer. Then, rclone is called once with this list to perform the transfer. The properties on this class are to be read during generation @@ -33,7 +35,9 @@ def __init__( dry_run: bool, log: bool, ): - """Parameters + """Initialise TransferData. + + Parameters ---------- cfg datashuttle configs UserDict. @@ -57,18 +61,19 @@ def __init__( specified. Can include datatype-level tranfser keywords. overwrite_existing_files - If "never" files on target will never be overwritten by source. - If "always" files on target will be overwritten by source if + If ``"never"`` files on target will never be overwritten by source. + If ``"always"`` files on target will be overwritten by source if there is any difference in date or size. - If "if_source_newer" files on target will only be overwritten + If ``"if_source_newer"`` files on target will only be overwritten by files on source with newer creation / modification datetime. dry_run - If `True`, transfer will not actually occur but will be logged - as if it did (to see what would happen for a transfer). + Perform a dry-run of transfer. This will output as if file + transfer was taking place, but no files will be moved. log if `True`, log and print the transfer output. + """ self.__cfg = cfg self.__upload_or_download = upload_or_download @@ -110,8 +115,7 @@ def __init__( # ------------------------------------------------------------------------- def build_a_list_of_all_files_and_folders_to_transfer(self) -> List[str]: - """Build a list of every file to transfer based on the user-passed - arguments. + """Build a list of every file to transfer based on the user-passed arguments. This cycles through every subject, session and datatype and adds the outputs to three lists: @@ -188,9 +192,7 @@ def build_a_list_of_all_files_and_folders_to_transfer(self) -> List[str]: def make_include_arg( self, list_of_paths: List[str], recursive: bool = True ) -> List[str]: - """Format the list of paths to rclone's required - `--include` flag format. - """ + """Format the list of paths to rclone's required `--include` flag format.""" if not any(list_of_paths): return [] @@ -213,8 +215,9 @@ def include_arg(ele: str) -> str: def update_list_with_non_sub_top_level_folders( self, extra_folder_names: List[str], extra_filenames: List[str] ) -> None: - """Search the subject level for all files and folders in the - top-level-folder. Split the output based onto files / folders + """Search the subject level for all files and folders in the top-level-folder. + + Splits the output based onto files / folders within "sub-" prefixed folders or not. """ top_level_folders: List[str] @@ -240,9 +243,7 @@ def update_list_with_non_ses_sub_level_folders( extra_filenames: List[str], sub: str, ) -> None: - """For the subject, get a list of files / folders that are - not within "ses-" prefixed folders. - """ + """Get a list of files / folders that for the subject that are not within "ses-" prefixed folders.""" sub_level_folders: List[str] sub_level_folders, sub_level_files = folders.search_sub_or_ses_level( # type: ignore self.__cfg, @@ -276,7 +277,9 @@ def update_list_with_non_dtype_ses_level_folders( sub: str, ses: str, ) -> None: - """For a specific subject and session, get a list of files / folders + """Return non-datatype files and folders. + + Returns a list of files / folders for a specific subject and session that are not in canonical datashuttle datatype folders. """ ses_level_folders: List[str] @@ -320,9 +323,7 @@ def update_list_with_dtype_paths( sub: str, ses: Optional[str] = None, ) -> None: - """Given a particular subject and session, get a list of all - canonical datatype folders. - """ + """Get a list of all canonical datatype folders for a particular subject and session.""" datatype = list(filter(lambda x: x != "all_non_datatype", datatype)) datatype_items = folders.items_from_datatype_input( @@ -358,10 +359,12 @@ def to_list(self, names: Union[str, List[str]]) -> List[str]: def check_input_arguments( self, ) -> None: - """Check the sub / session names passed. The checking here - is stricter than for create_folders / formatting.check_and_format_names - because we want to ensure that a) non-datatype arguments are not - passed at the wrong input (e.g. all_non_ses as a subject name). + """Check the sub / session names passed. + + The checking here is stricter than for create_folders and + formatting.check_and_format_names because we want to ensure that + non-datatype arguments are not passed at the wrong input + (e.g. all_non_ses as a subject name). We also want to limit the possible combinations of inputs, such that is a user inputs "all" subjects, or "all_sub", they should @@ -420,6 +423,7 @@ def get_processed_names( sub: Optional[str] = None, ) -> List[str]: """Process the list of subject session names. + If they are pre-defined (e.g. ["sub-001", "sub-002"]) they will be checked and formatted as per formatting.check_and_format_names() and @@ -471,9 +475,7 @@ def get_processed_names( return processed_names def transfer_non_datatype(self, datatype_checked: List[str]) -> bool: - """Convenience function, bool if all non-datatype folders - are to be transferred. - """ + """Return bool if all non-datatype folders are to be transferred.""" return any( [name in ["all_non_datatype", "all"] for name in datatype_checked] ) From 0be456cfec1cf484a2d565e10b07cd31d0e5a265 Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Sun, 22 Jun 2025 04:35:28 +0100 Subject: [PATCH 39/70] Update rlcone.py. --- datashuttle/utils/rclone.py | 39 ++++++++++++++++++++++--------------- 1 file changed, 23 insertions(+), 16 deletions(-) diff --git a/datashuttle/utils/rclone.py b/datashuttle/utils/rclone.py index 7913cf863..8958c4824 100644 --- a/datashuttle/utils/rclone.py +++ b/datashuttle/utils/rclone.py @@ -12,8 +12,7 @@ def call_rclone(command: str, pipe_std: bool = False) -> CompletedProcess: - """Call rclone with the specified command. Current mode is double-verbose. - Return the completed process from subprocess. + """Call rclone with the specified command. Parameters ---------- @@ -36,9 +35,10 @@ def call_rclone(command: str, pipe_std: bool = False) -> CompletedProcess: def call_rclone_through_script(command: str) -> CompletedProcess: - """Call rclone through a script, to avoid limits on command-line calls - (in particular on Windows). Used for transfers due to generation of - large call strings. + """Call rclone through a script. + + This is to avoid limits on command-line calls (in particular on Windows). + Used for transfers due to generation of large call strings. """ system = platform.system() @@ -82,7 +82,9 @@ def setup_rclone_config_for_local_filesystem( rclone_config_name: str, log: bool = True, ): - """RClone sets remote targets in a config file that are + """Set the RClone remote config for local filesystem. + + RClone sets remote targets in a config file that are used at transfer. For local filesystem, this is essentially a placeholder and that is not linked to a particular filepath. It just tells rclone to use the local filesystem - then we @@ -116,7 +118,9 @@ def setup_rclone_config_for_ssh( ssh_key_path: Path, log: bool = True, ): - """RClone sets remote targets in a config file that are + """Set the RClone remote config for ssh. + + RClone sets remote targets in a config file that are used at transfer. For SSH, this must contain the central path, username and ssh key. The relative path is supplied at transfer time. @@ -170,9 +174,7 @@ def check_rclone_with_default_call() -> bool: def prompt_rclone_download_if_does_not_exist() -> None: - """Check that rclone is installed. If it does not - (e.g. first time using datashuttle) then download. - """ + """Check that rclone is installed.""" if not check_rclone_with_default_call(): raise BaseException( "RClone installation not found. Install by entering " @@ -249,13 +251,15 @@ def get_local_and_central_file_differences( cfg: Configs, top_level_folders_to_check: List[TopLevelFolder], ) -> Dict: - """Convert the output of rclone's check (with `--combine`) flag - to a dictionary separating each case. + """Format a structure of all changes between local and central. Rclone output comes as a list of files, separated by newlines, with symbols indicating whether the file paths are same across local and central, different, or found in local / central only. + Convert the output of Rclone's check (with `--combine`) flag + to a dictionary separating each case. + Parameters ---------- cfg @@ -305,9 +309,10 @@ def get_local_and_central_file_differences( def assert_rclone_check_output_is_as_expected(result, symbol, convert_symbols): - """Ensure the output of Rclone check is as expected. Currently, the "error" - case is untested and a test case is required. Once the test case is - obtained this should most likely be moved to tests. + """Ensure the output of Rclone check is as expected. + + Currently, the "error" case is untested and a test case is required. + Once the test case is obtained this should most likely be moved to tests. """ assert result[1] == " ", ( "`rclone check` output does not contain a " @@ -324,7 +329,9 @@ def assert_rclone_check_output_is_as_expected(result, symbol, convert_symbols): def perform_rclone_check( cfg: Configs, top_level_folder: TopLevelFolder ) -> str: - r"""Use Rclone's `check` command to build a list of files that + r"""Run RClone check to find differences in files between local and central. + + Use Rclone's `check` command to build a list of files that are the same ("="), different ("*"), found in local only ("+") or central only ("-"). The output is formatted as "\ \\n". """ From 14c8631134d27e624cc847b0326e17db6680195f Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Sun, 22 Jun 2025 04:42:08 +0100 Subject: [PATCH 40/70] Update getters.py --- datashuttle/utils/getters.py | 50 ++++++++++++++++++++++-------------- 1 file changed, 31 insertions(+), 19 deletions(-) diff --git a/datashuttle/utils/getters.py b/datashuttle/utils/getters.py index 44e58a499..3af513b51 100644 --- a/datashuttle/utils/getters.py +++ b/datashuttle/utils/getters.py @@ -37,9 +37,11 @@ def get_next_sub_or_ses( default_num_value_digits: int = 3, name_template_regexp: Optional[str] = None, ) -> str: - """Suggest the next available subject or session number. This function will - search the local repository, and the central repository, for all subject - or session folders (subject or session depending on inputs). + """Suggest the next available subject or session number. + + This function will search the local repository, and the central + repository, for all subject or session folders (subject or session + depending on inputs). It will take the union of all folder names, find the relevant key-value pair values, and return the maximum value + 1 as the new number. @@ -56,11 +58,11 @@ def get_next_sub_or_ses( The top-level folder (e.g. `"rawdata"`, `"derivatives"`) sub - subject name to search within if searching for sessions, otherwise None + Subject name to search within if searching for sessions, otherwise None to search for subjects search_str - the string to search for within the top-level or subject-level + The string to search for within the top-level or subject-level folder ("sub-*") or ("ses-*") are suggested, respectively. include_central @@ -126,10 +128,10 @@ def get_max_sub_or_ses_num_and_value_length( default_num_value_digits: Optional[int] = None, name_template_regexp: Optional[str] = None, ) -> Tuple[int, int]: - """Given a list of BIDS-style folder names, find the maximum subject or - session value (sub or ses depending on `prefix`). Also, find the - number of value digits across the project, so a new suggested number - can be formatted consistency. If the list is empty, set the value + """Find the maximum subject or session value given a list of BIDS-style folder names. + + Also, find the number of value digits across the project, so a new suggested + number can be formatted consistency. If the list is empty, set the value to 0 and a default number of value digits. Parameters @@ -215,8 +217,15 @@ def get_num_value_digits_from_project( all_values_str: List[str], prefix: Prefix ) -> int: """Find the number of digits for the sub or ses key within the project. - `all_values_str` is a list of all the sub or ses values from within - the project. + + Parameters + ---------- + all_values_str + A list of all the sub or ses values from within the project. + + prefix + "sub" or "ses". + """ all_num_value_digits = [len(value) for value in all_values_str] @@ -234,8 +243,9 @@ def get_num_value_digits_from_project( def get_num_value_digits_from_regexp( prefix: Prefix, name_template_regexp: str ) -> Union[Literal[False], int]: - r"""Given a name template regexp, find the number of values for the - sub or ses key. These will be fixed with "\d" (digit) or ".?" (wildcard). + r"""Given a name template regexp, find the number of values for the sub or ses key. + + These will be fixed with "\d" (digit) or ".?" (wildcard). If there is length-unspecific wildcard (.*) in the sub key, then skip. In practice, there should never really be a .* in the sub or ses key of a name template, but handle it just in case. @@ -260,8 +270,9 @@ def get_num_value_digits_from_regexp( def get_existing_project_paths() -> List[Path]: - """Return full path and names of datashuttle projects on - this local machine. A project is determined by a project + """Return full path and names of datashuttle projects on this local machine. + + A project is determined by a project folder in the home / .datashuttle folder that contains a config.yaml file. Returns in order of most recently modified first. @@ -298,9 +309,10 @@ def get_all_sub_and_ses_paths( top_level_folder: TopLevelFolder, include_central: bool, ) -> Dict: - """Get a list of every subject and session name in the - local and central project folders. Local and central names are combined - into a single list, separately for subject and sessions. + """Return a dict including filepaths to all subjects and sessions. + + Local and central names are combined into a single list, + separately for subject and sessions. Note this only finds local sub and ses names on this machine. Other local machines are not searched. @@ -308,7 +320,7 @@ def get_all_sub_and_ses_paths( Parameters ---------- cfg - datashuttle Configs + Datashuttle Configs. top_level_folder The top-level folder (e.g. `"rawdata"`, `"derivatives"`) From bd94bce9e81ce119b153100c5127678f7540dfbe Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Sun, 22 Jun 2025 04:50:38 +0100 Subject: [PATCH 41/70] update formatting.py --- datashuttle/utils/formatting.py | 28 +++++++++++++--------------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/datashuttle/utils/formatting.py b/datashuttle/utils/formatting.py index 5177ff35e..f3504c26e 100644 --- a/datashuttle/utils/formatting.py +++ b/datashuttle/utils/formatting.py @@ -22,8 +22,9 @@ def check_and_format_names( name_templates: Optional[Dict] = None, bypass_validation: bool = False, ) -> List[str]: - """Format a list of subject or session names, e.g. - by ensuring all have sub- or ses- prefix, checking + """Format a list of subject or session names. + + This ensures all have sub- or ses- prefix, checks for tags, that names do not include spaces and that there are not duplicates. @@ -116,6 +117,7 @@ def update_names_with_range_to_flag( names: List[str], prefix: str ) -> List[str]: """Given a list of names, check if they contain the @TO@ keyword. + If so, expand to a range of names. Names including the @TO@ keyword must be in the form prefix-num1@num2. The maximum number of leading zeros are used to pad the output @@ -171,9 +173,7 @@ def update_names_with_range_to_flag( def check_name_with_to_tag_is_formatted_correctly( name: str, prefix: str ) -> None: - """Check the input string is formatted with the @TO@ key - as expected. - """ + """Check the input string is formatted with the @TO@ key as expected.""" first_key_value_pair = name.split("_")[0] expected_format = re.compile(f"{prefix}-[0-9]+{tags('to')}[0-9]+") @@ -189,7 +189,9 @@ def check_name_with_to_tag_is_formatted_correctly( def make_list_of_zero_padded_names_across_range( left_number: str, right_number: str, name_start_str: str, name_end_str: str ) -> List[str]: - """Numbers formatted with the @TO@ keyword need to have + """Make a list of subject or session names across a range. + + Numbers formatted with the @TO@ keyword need to have standardised leading zeros on the output. Here we take the maximum number of leading zeros and apply for all numbers in the range. Note int() will strip @@ -257,9 +259,7 @@ def replace_date_time_tags_in_name( date_with_key: str, time_with_key: str, ): - """For all names in the list, do the replacement of tags - with their final values. - """ + """Replace tags with their final value for every name in a list.""" for i, name in enumerate(names): # datetime conditional must come first. if tags("datetime") in name: @@ -293,7 +293,9 @@ def format_datetime(date: str, time_: str) -> str: def add_underscore_before_after_if_not_there(string: str, key: str) -> str: - """If names are passed with @DATE@, @TIME@, or @DATETIME@ + """Handle tags that are not perfectly formatted with underscores. + + If names are passed with @DATE@, @TIME@, or @DATETIME@ but not surrounded by underscores, check and insert if required. e.g. sub-001@DATE@ becomes sub-001_@DATE@ or sub-001@DATEid-101 becomes sub-001_@DATE_id-101. @@ -322,11 +324,7 @@ def add_underscore_before_after_if_not_there(string: str, key: str) -> str: def add_missing_prefixes_to_names( all_names: Union[List[str], str], prefix: str ) -> List[str]: - """Make sure all elements in the list of names are - prefixed with the prefix, typically "sub-" or "ses-". - - Use expanded list for readability - """ + """Ensure all elements in the list of names are prefixed with "sub-" or "ses-".""" prefix = prefix + "-" n_chars = len(prefix) From 5d2ee85f59ec9c409110f28ba768247e192330cb Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Sun, 22 Jun 2025 04:52:18 +0100 Subject: [PATCH 42/70] update folders.py --- datashuttle/utils/folders.py | 49 +++++++++++++++++++----------------- 1 file changed, 26 insertions(+), 23 deletions(-) diff --git a/datashuttle/utils/folders.py b/datashuttle/utils/folders.py index fb0e3e0f1..74fe2575e 100644 --- a/datashuttle/utils/folders.py +++ b/datashuttle/utils/folders.py @@ -36,8 +36,9 @@ def create_folder_trees( datatype: Union[List[str], str], log: bool = True, ) -> Dict[str, List[Path]]: - """Entry method to make a full folder tree. It will - iterate through all passed subjects, then sessions, then + """Entry method to make a full folder tree. + + Iterate through all passed subjects, then sessions, then subfolders within a datatype folder. This permits flexible creation of folders (e.g. to make subject only, do not pass session name. @@ -128,9 +129,10 @@ def make_datatype_folders( save_paths: Dict, log: bool = True, ): - """Make datatype folder (e.g. behav) at the sub or ses - level. Checks folder_class.Folders attributes, - whether the datatype is used and at the current level. + """Make datatype folder (e.g. behav) at the sub or ses level. + + Checks folder_class.Folders attributes, whether the datatype + is used and at the current level. Parameters ---------- @@ -180,8 +182,7 @@ def make_datatype_folders( def create_folders(paths: Union[Path, List[Path]], log: bool = True) -> None: - """For path or list of paths, make them if - they do not already exist. + """Make a path or list of paths if they do not already exist. Parameters ---------- @@ -219,10 +220,11 @@ def search_project_for_sub_or_ses_names( include_central: bool, return_full_path: bool = False, ) -> Dict: - """If sub is None, the top-level level folder will be - searched (i.e. for subjects). The search string "sub-*" is suggested - in this case. Otherwise, the subject, level folder for the specified - subject will be searched. The search_str "ses-*" is suggested in this case. + """If sub is None, the top-level level folder will be searched (i.e. for subjects). + + The search string "sub-*" is suggested in this case. Otherwise, the subject, + level folder for the specified subject will be searched. + The search_str "ses-*" is suggested in this case. Note `verbose` argument of `search_sub_or_ses_level()` is set to `False`, as session folders for local subjects that are not yet on central @@ -270,12 +272,12 @@ def items_from_datatype_input( sub: str, ses: Optional[str] = None, ) -> Union[ItemsView, zip]: - """Get the list of datatypes to transfer, either - directly from user input, or by searching + """Get the list of datatypes to transfer. + + Take these directly from user input, or by searching what is available if "all" is passed. - see _transfer_datatype() for full - parameters list. + see _transfer_datatype() for full parameters list. """ base_folder = cfg.get_base_folder(local_or_central, top_level_folder) @@ -307,8 +309,9 @@ def search_for_datatype_folders( sub: str, ses: Optional[str] = None, ) -> zip: - """Search a subject or session folder specifically - for datatypes. First searches for all folders / files + """Search a subject or session folder specifically for datatypes. + + First searches for all folders / files in the folder, and then returns any folders that match datatype name. @@ -336,9 +339,9 @@ def process_glob_to_find_datatype_folders( folder_names: list, datatype_folders: dict, ) -> zip: - """Process the results of glob on a sub or session level, - which could contain any kind of folder / file. + """Process the results of glob on a sub or session level. + The results could contain any type of folder / file. see project.search_sub_or_ses_level() for inputs. Returns @@ -456,7 +459,6 @@ def search_sub_or_ses_level( return_full_path: bool = False, ) -> Tuple[List[str] | List[Path], List[str]]: """Search project folder at the subject or session level. - Only returns folders. Parameters ---------- @@ -527,8 +529,7 @@ def search_for_folders( verbose: bool = True, return_full_path: bool = False, ) -> Tuple[List[Any], List[Any]]: - """Wrapper to determine the method used to search for search - prefix folders in the search path. + """Determine the method used to search for search prefix folders in the search path. Parameters ---------- @@ -578,7 +579,9 @@ def search_for_folders( def search_filesystem_path_for_folders( search_path_with_prefix: Path, return_full_path: bool = False ) -> Tuple[List[Path | str], List[Path | str]]: - """Use glob to search the full search path (including prefix) with glob. + """Search a folder through the local filesystem. + + Use glob to search the full search path (including prefix) with glob. Files are filtered out of results, returning folders only. """ all_folder_names = [] From fea2b6041266454c090b74e82bcb701d30fad39f Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Sun, 22 Jun 2025 04:53:10 +0100 Subject: [PATCH 43/70] Update folder_class.py --- datashuttle/utils/folder_class.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/datashuttle/utils/folder_class.py b/datashuttle/utils/folder_class.py index 87c036bab..71fab22a5 100644 --- a/datashuttle/utils/folder_class.py +++ b/datashuttle/utils/folder_class.py @@ -1,6 +1,5 @@ class Folder: - """Folder class used to contain details of canonical - folders in the project folder tree. + """Contains details of canonical folders in the project folder tree. see configs.canonical_folders.py for details. """ @@ -10,9 +9,10 @@ def __init__( name: str, level: str, ): - """Parameters - ------------- + """Initialise the Folder class. + Parameters + ---------- name the name of the folder. From a6323162d90c57c404164033c3a54a04c5741e93 Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Sun, 22 Jun 2025 05:01:31 +0100 Subject: [PATCH 44/70] Update folder_utils.py --- datashuttle/utils/utils.py | 81 ++++++++++++++++++++++++++++---------- 1 file changed, 61 insertions(+), 20 deletions(-) diff --git a/datashuttle/utils/utils.py b/datashuttle/utils/utils.py index 8e51d9185..6c102279b 100644 --- a/datashuttle/utils/utils.py +++ b/datashuttle/utils/utils.py @@ -19,9 +19,7 @@ def log(message: str) -> None: - """Log the message to the main initialised - logger. - """ + """Log the message to the main initialised logger.""" if ds_logger.logging_is_active(): logger = ds_logger.get_logger() logger.debug(message) @@ -29,7 +27,15 @@ def log(message: str) -> None: def log_and_message(message: str, use_rich: bool = False) -> None: """Log the message and send it to user. - use_rich : is True, use rich's print() function. + + Parameters + ---------- + message + Message to log and print to user. + + use_rich + If True, use rich's print() function. + """ log(message) print_message_to_user(message, use_rich) @@ -45,7 +51,17 @@ def log_and_raise_error(message: str, exception: Any) -> None: def warn(message: str, log: bool) -> None: - """PLACEHOLDER.""" + """Send a warning. + + Parameters + ---------- + message + Message to warn. + + log + If True, log at WARNING level. + + """ if log and ds_logger.logging_is_active(): logger = ds_logger.get_logger() logger.warning(message) @@ -53,9 +69,10 @@ def warn(message: str, log: bool) -> None: def raise_error(message: str, exception) -> None: - """Centralized way to raise an error. The logger is closed - to ensure it is not still running if a function call - raises an exception in a python environment. + """Centralized way to raise an error. + + The logger is closed to ensure it is not still running + if a function call raises an exception in a python environment. """ ds_logger.close_log_filehandler() raise exception(message) @@ -65,7 +82,15 @@ def print_message_to_user( message: Union[str, list], use_rich: bool = False ) -> None: """Centralised way to send message. - use_rich : use rich's print() function. + + Parameters + ---------- + message + Message to print. + + use_rich + If True, use rich's print() function. + """ if use_rich: rich_print(message) @@ -85,7 +110,7 @@ def get_user_input(message: str) -> str: def path_starts_with_base_folder(base_folder: Path, path_: Path) -> bool: - """PLACEHOLDER.""" + """Check whether the path starts with the base folder path.""" return path_.as_posix().startswith(base_folder.as_posix()) @@ -118,9 +143,21 @@ def get_values_from_bids_formatted_name( return_as_int: bool = False, sort: bool = False, ) -> Union[List[int], List[str]]: - """Find the values associated with a key from a list of all - BIDS-formatted file / folder names. This is typically used to - find sub / ses values. + """Find the values associated with a key in a BIDS-style name. + + Parameters + ---------- + all_names + A list of names from which to find the value associated with the key. + + key + Key from which to associate the values e.g. "sub") + + return_as_int + If True and the value can be cast to int (e.g. `sub-001`), return as `int`. + + sort + If True, results are sorted before being returned. Notes ----- @@ -170,10 +207,9 @@ def sub_or_ses_value_to_int(value: str) -> int: def get_value_from_key_regexp(name: str, key: str) -> List[str]: - """Find the value related to the key in a - BIDS-style key-value pair name. - e.g. sub-001_ses-312 would find - 312 for key "ses". + """Find the value related to the key in a BIDS-style key-value pair name. + + e.g. sub-001_ses-312 would find 312 for key "ses". """ return re.findall(f"{key}-(.*?)(?=_|$)", name) @@ -190,14 +226,19 @@ def integers_are_consecutive(list_of_ints: List[int]) -> bool: def diff(x: List) -> List: - """slow, custom differentiator for small inputs, to avoid - adding numpy as a dependency. + """Differentiate list of numbers. + + Slow, only to avoid adding numpy as a dependency. """ return [x[i + 1] - x[i] for i in range(len(x) - 1)] def num_leading_zeros(string: str) -> int: - """int() strips leading zeros.""" + """Return the number of leading zeros in a sub- or ses- id. + + e.g. sub-001 has 2 leading zeros. + int() strips leading zeros. + """ if string[:4] in ["sub-", "ses-"]: string = string[4:] From e410907252488435b5a962dc674315ef5436c04d Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Sun, 22 Jun 2025 05:07:09 +0100 Subject: [PATCH 45/70] Update ssh.py --- datashuttle/utils/ssh.py | 95 +++++++++++++++++++++++++++++++++------- 1 file changed, 80 insertions(+), 15 deletions(-) diff --git a/datashuttle/utils/ssh.py b/datashuttle/utils/ssh.py index 568773438..bd418a62d 100644 --- a/datashuttle/utils/ssh.py +++ b/datashuttle/utils/ssh.py @@ -29,7 +29,22 @@ def connect_client_core( cfg: Configs, password: Optional[str] = None, ): - """PLACEHOLDER.""" + """Connect to the client. + + A centralised function to connect to a paramiko client. + + Parameters + ---------- + client + Paramiko client to connect to. + + cfg + Datashuttle Configs. + + password + Password (if required) to establish the connection. + + """ client.get_host_keys().load(cfg.hostkeys_path.as_posix()) client.set_missing_host_key_policy(paramiko.RejectPolicy()) @@ -72,14 +87,26 @@ def add_public_key_to_central_authorized_keys( def generate_and_write_ssh_key(ssh_key_path: Path) -> None: - """PLACEHOLDER.""" + """Generate an RSA SSH key and save it to the specified file path. + + Parameters + ---------- + ssh_key_path + The full file path where the private SSH key will be saved. + + """ key = paramiko.RSAKey.generate(4096) key.write_private_key_file(ssh_key_path.as_posix()) def get_remote_server_key(central_host_id: str): - """Get the remove server host key for validation before - connection. + """Get the remove server host key for validation before connection. + + Parameters + ---------- + central_host_id + The hostname or IP address of the central host. + """ transport: paramiko.Transport with paramiko.Transport(central_host_id) as transport: @@ -89,7 +116,24 @@ def get_remote_server_key(central_host_id: str): def save_hostkey_locally(key, central_host_id, hostkeys_path) -> None: - """PLACEHOLDER.""" + """Save the SSH host key locally to the specified hostkeys file. + + The host key uniquely identifies the SSH server to prevent + man-in-the-middle attacks by verifying the server's identity + on future connections. + + Parameters + ---------- + key + The SSH host key to save. + + central_host_id + The hostname or IP address of the central host. + + hostkeys_path + The file path where host keys are stored locally. + + """ client = paramiko.SSHClient() client.get_host_keys().add(central_host_id, key.get_name(), key) client.get_host_keys().save(hostkeys_path.as_posix()) @@ -106,11 +150,10 @@ def setup_ssh_key( cfg: Configs, log: bool = True, ) -> None: - """Set up an SSH private / public key pair with - central server. First, a private key is generated - and saved in the .datashuttle config path. - Next a connection requiring input - password made, and the public part of the key + """Set up an SSH private / public key pair with central server. + + First, a private key is generated and saved in the .datashuttle config path. + Next a connection requiring input password made, and the public part of the key added to ~/.ssh/authorized_keys. Parameters @@ -175,6 +218,7 @@ def connect_client_with_logging( message_on_sucessful_connection: bool = True, ) -> None: """Connect client to central server using paramiko. + Accept either password or path to private key, but not both. Paramiko does not support pathlib. """ @@ -201,9 +245,28 @@ def connect_client_with_logging( def verify_ssh_central_host( central_host_id: str, hostkeys_path: Path, log: bool = True ) -> bool: - """Similar to connecting with other SSH manager e.g. putty, - get the server key and present when connecting - for manual validation. + """Prompt the user to verify and cache the SSH server's host key. + + This function retrieves the SSH server's key and asks the user to + manually validate and accept it. Accepting the key caches it locally + to ensure secure future connections. + + Parameters + ---------- + central_host_id + Hostname or IP address of the SSH server. + + hostkeys_path + Path to the local file where known host keys are stored. + + log + Whether to log the verification messages. + + Returns + ------- + bool + True if the host key was accepted and saved, False otherwise. + """ key = get_remote_server_key(central_host_id) @@ -248,6 +311,7 @@ def search_ssh_central_for_folders( return_full_path: bool = False, ) -> Tuple[List[Any], List[Any]]: """Search for the search prefix in the search path over SSH. + Returns the list of matching folders, files are filtered out. Parameters @@ -295,8 +359,9 @@ def get_list_of_folder_names_over_sftp( verbose: bool = True, return_full_path: bool = False, ) -> Tuple[List[Any], List[Any]]: - """Use paramiko's sftp to search a path - over ssh for folders. Return the folder names. + """Use paramiko's sftp to search a path over ssh for folders. + + Return the folder names. Parameters ---------- From d51da9f2a9b1224de6ad3b03922648f3b58c4078 Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Sun, 22 Jun 2025 05:19:04 +0100 Subject: [PATCH 46/70] Update validate.py --- datashuttle/utils/validation.py | 154 +++++++++++++++++--------------- 1 file changed, 84 insertions(+), 70 deletions(-) diff --git a/datashuttle/utils/validation.py b/datashuttle/utils/validation.py index 88fc02a77..390d82171 100644 --- a/datashuttle/utils/validation.py +++ b/datashuttle/utils/validation.py @@ -34,7 +34,7 @@ def get_missing_prefix_error(name: str, prefix, path_: Path | None) -> str: - """PLACEHOLDER.""" + """Return error message when a required prefix is missing from a name.""" return handle_path( f"MISSING_PREFIX: The prefix {prefix} was not found in the name: {name}", path_, @@ -42,7 +42,7 @@ def get_missing_prefix_error(name: str, prefix, path_: Path | None) -> str: def get_bad_value_error(name: str, prefix, path_: Path | None) -> str: - """PLACEHOLDER.""" + """Return error message when the value for a prefix is not an integer.""" return handle_path( f"BAD_VALUE: The value for prefix {prefix} in name {name} is not an integer.", path_, @@ -50,22 +50,22 @@ def get_bad_value_error(name: str, prefix, path_: Path | None) -> str: def get_duplicate_prefix_error(name: str, prefix, path_: Path | None) -> str: - """PLACEHOLDER.""" + """Return error message when a name contains multiple instances of the same prefix.""" return handle_path( - f"DUPLICATE_PREFIX: The name: {name} of contains more than one instance of the prefix {prefix}.", + f"DUPLICATE_PREFIX: The name: {name} contains more than one instance of the prefix {prefix}.", path_, ) def get_name_error(name: str, prefix: Prefix, path_: Path | None) -> str: - """PLACEHOLDER.""" + """Return error message when a name is invalid for a given prefix.""" return handle_path( f"BAD_NAME: The name: {name} of type: {prefix} is not valid.", path_ ) def get_special_char_error(name: str, path_: Path | None) -> str: - """PLACEHOLDER.""" + """Return error message when a name contains invalid characters.""" return handle_path( f"SPECIAL_CHAR: The name: {name}, contains characters which are not alphanumeric, dash or underscore.", path_, @@ -73,7 +73,7 @@ def get_special_char_error(name: str, path_: Path | None) -> str: def get_name_format_error(name: str, path_: Path | None) -> str: - """PLACEHOLDER.""" + """Return error message when a name does not follow key-value pair format.""" return handle_path( f"NAME_FORMAT: The name {name} does not consist of key-value pairs separated by underscores.", path_, @@ -81,12 +81,12 @@ def get_name_format_error(name: str, path_: Path | None) -> str: def get_value_length_error(prefix: Prefix) -> str: - """PLACEHOLDER.""" + """Return error message for inconsistent value lengths for a prefix.""" return f"VALUE_LENGTH: Inconsistent value lengths for the prefix: {prefix} were found in the project." def get_datetime_error(key, name: str, strfmt: str, path_: Path | None) -> str: - """PLACEHOLDER.""" + """Return error message when a datetime value is not in the expected ISO format.""" return handle_path( f"DATETIME: Name {name} contains an invalid {key}. It should be ISO format: {strfmt}.", path_, @@ -94,7 +94,10 @@ def get_datetime_error(key, name: str, strfmt: str, path_: Path | None) -> str: def get_template_error(name: str, regexp: str, path_: Path | None) -> str: - """The missing full-stop at the end is intentional, to avoid confusion when reading the regexp.""" + """Return error message when a name does not match a given template. + + The missing full-stop at the end is intentional, to avoid confusion when reading the regexp. + """ return handle_path( f"TEMPLATE: The name: {name} does not match the template: {regexp}", path_, @@ -104,7 +107,7 @@ def get_template_error(name: str, regexp: str, path_: Path | None) -> str: def get_missing_top_level_folder_error( path_: Path | None, local_or_central: Literal["local", "central"] ) -> str: - """PLACEHOLDER.""" + """Return error message when the top level folder is missing from the project.""" return handle_path( f"TOP_LEVEL_FOLDER: The {local_or_central} project must contain a 'rawdata' or 'derivatives' folder.", path_, @@ -114,7 +117,7 @@ def get_missing_top_level_folder_error( def get_duplicate_name_error( new_name: str, exist_name: str, exist_path: Path | None ) -> str: - """PLACEHOLDER.""" + """Return error message when a new name duplicates an existing name.""" return handle_path( f"DUPLICATE_NAME: The prefix for {new_name} duplicates the name: {exist_name}.", exist_path, @@ -122,14 +125,14 @@ def get_duplicate_name_error( def get_datatype_error(datatype_name: str, path_: Path | None) -> str: - """PLACEHOLDER.""" + """Return error message when an invalid datatype name is encountered.""" return handle_path( f"DATATYPE: {datatype_name} is not a valid datatype name.", path_ ) def handle_path(message: str, path_: Path | None) -> str: - """PLACEHOLDER.""" + """Append the file path to the error message if available.""" if path_: message += f" Path: {path_.as_posix()}" return message @@ -146,23 +149,22 @@ def validate_list_of_names( name_templates: Optional[Dict] = None, check_value_lengths: bool = True, ) -> List[str]: - """Validate a list of subject or session names, ensuring - they are formatted as per NeuroBlueprint. + """Validate a list of subject or session names against NeuroBlueprint. Parameters ---------- path_or_name_list - A list of pathlib.Path to NeuroBlueprint-formatted folders to validate + A list of pathlib.Path to NeuroBlueprint-formatted folders to validate prefix - Whether these are subject (sub) or session (ses) level names + Whether these are subject (sub) or session (ses) level names name_templates - A `name_template` dictionary to validate against. See `set_name_templates()`. + A `name_template` dictionary to validate against. See `set_name_templates()`. check_value_lengths - If `True`, check that the prefix- value lengths - are consistent across the passed list. + If `True`, check that the prefix- value lengths + are consistent across the passed list. """ if len(path_or_name_list) == 0: @@ -214,7 +216,11 @@ def validate_list_of_names( def prefix_is_duplicate_or_has_bad_values( name: str, prefix: Prefix, path_: Path | None ) -> List[str]: - """Check that the prefix (sub- or ses-) is found only once in the name and that its value can be converted to integer.""" + """Check the sub- or ses- prefix. + + Ensure it is found only once in the name and + that its value can be converted to integer. + """ value = re.findall(f"{prefix}(.*?)(?=_|$)", name) if len(value) == 0: @@ -237,7 +243,11 @@ def new_name_duplicates_existing( ) -> List[str]: """Check that a subject or session value does not duplicate an existing value. - The only case this is allowed is when the names match exactly. For example, if "sub-001" exists, we can pass "sub-001" as a valid subject name (for example, when making sessions). However, if "sub-001_another-tag" exists, we should throw an error, because this shares the same subject id but refers to a different subject. + The only case this is allowed is when the names match exactly. + For example, if "sub-001" exists, we can pass "sub-001" as a valid subject name + (for example, when making sessions). However, if "sub-001_another-tag" exists, + we should throw an error, because this shares the same subject id but refers + to a different subject. """ # Make a list of matches between `new_name` and any in `existing_names` new_name_id = utils.get_values_from_bids_formatted_name( @@ -268,10 +278,7 @@ def names_dont_match_templates( prefix: Prefix, name_templates: Optional[Dict] = None, ) -> List[str]: - """Test a list of subject or session names against - the respective `name_templates`, a regexp template. - - """ + """Validate a list of sub/ses names against the respective regexp `name_templates`.""" if name_templates is None: return [] @@ -292,11 +299,7 @@ def names_dont_match_templates( def get_path_and_name(path_or_name: Path | str) -> Tuple[Optional[Path], str]: - """Convenience function to get the folder name - - from something that is either a Path (pointing - to the folder) or a str of the folder name itself. - """ + """Return the folder name from a Path (to the folder) or a str of the name itself.""" if isinstance(path_or_name, Path): return path_or_name, path_or_name.name else: @@ -304,9 +307,12 @@ def get_path_and_name(path_or_name: Path | str) -> Tuple[Optional[Path], str]: def replace_tags_in_regexp(regexp: str) -> str: - r"""Before validation, all tags in the names are converted to their final values (e.g. @DATE@ -> _date-). We also want to allow template to be formatted like `sub-\d\d_@DATE@` as it is convenient for auto-completion in the TUI. + r"""Before validation, all tags in the names are converted to their final values. - Therefore, we must replace the tags in the regexp with their actual regexp equivalent before comparison. + For example, (e.g. @DATE@ -> _date-). + We also want to allow template to be formatted like `sub-\d\d_@DATE@` as it is + convenient for auto-completion in the TUI. Therefore, we must replace the tags + in the regexp with their actual regexp equivalent before comparison. Note `replace_date_time_tags_in_name()` operates in place on a list. """ @@ -354,7 +360,7 @@ def name_has_special_character(name: str) -> bool: def dashes_and_underscore_alternate_incorrectly( name: str, path_: Path | None ) -> List[str]: - """Check a list of NeuroBlueprint formatted names + """Check a list of NeuroBlueprint formatted names. Names should have the "-" and "-" ordered correctly. Names should be key-value pairs separated by underscores e.g. sub-001_ses-001. @@ -383,9 +389,9 @@ def value_lengths_are_inconsistent( path_or_names_list: List[Path] | List[str] | List[Path | str], prefix: Prefix, ) -> List[str]: - """Given a list of NeuroBlueprint-formatted subject or session names, + """Determine if there are inconsistent value lengths for the sub or ses key in a list of names. - Determine if there are inconsistent value lengths for the sub or ses key. e.g. ["sub-01", "sub-001"] is an error. + For example, ["sub-01", "sub-001"] is an error. """ names_list = [ path_or_name if isinstance(path_or_name, str) else path_or_name.name @@ -489,32 +495,32 @@ def validate_project( Parameters ---------- cfg - datashuttle Configs class. + datashuttle Configs class. top_level_folder_list - The top level folders to validate. + The top level folders to validate. include_central - If `False`, only project folders in the `local_path` will - be validated. Otherwise, project folders in both the `local_path` - and `central_path` will be validated. + If `False`, only project folders in the `local_path` will + be validated. Otherwise, project folders in both the `local_path` + and `central_path` will be validated. display_mode - Determine whether error or warning is raised. + Determine whether error or warning is raised. log - If `True`, errors or warnings are logged to "datashuttle" logger. + If `True`, errors or warnings are logged to "datashuttle" logger. name_templates - A `name_template` dictionary to validate against. See `set_name_templates()`. + A `name_template` dictionary to validate against. See `set_name_templates()`. strict_mode - If `True`, only allow NeuroBlueprint-formatted folders to exist in - the project. By default, non-NeuroBlueprint folders (e.g. a folder - called 'my_stuff' in the 'rawdata') are allowed, and only folders - starting with sub- or ses- prefix are checked. In `Strict Mode`, - any folder not prefixed with sub-, ses- or a valid datatype will - raise a validation issue. + If `True`, only allow NeuroBlueprint-formatted folders to exist in + the project. By default, non-NeuroBlueprint folders (e.g. a folder + called 'my_stuff' in the 'rawdata') are allowed, and only folders + starting with sub- or ses- prefix are checked. In `Strict Mode`, + any folder not prefixed with sub-, ses- or a valid datatype will + raise a validation issue. """ error_messages = [] @@ -584,41 +590,45 @@ def validate_names_against_project( log: bool = True, name_templates: Optional[Dict] = None, ) -> None: - """Given a list of subject and (optionally) session names, check that these names are formatted consistently with the rest of the project. Used for creating folders. + """Check that sub / ses names are formatted consistently with the rest of the project. - Unfortunately this is quite fiddly, as it is important to only validate the passed list of subject / session names while ignoring validation errors that may already exist in the project. + This is used for creating folders. + + Unfortunately this is quite fiddly, as it is important to only + validate the passed list of subject / session names while ignoring + validation errors that may already exist in the project. Parameters ---------- cfg - datashuttle Configs class. + datashuttle Configs class. top_level_folder - The top level folder to validate + The top level folder to validate sub_names - A list of subject-level names to validate against the - subject names that exist in the project. + A list of subject-level names to validate against the + subject names that exist in the project. ses_names - A list of session-level names to validate against the - session names that exist in the project. Note that - duplicate checks will only be performed for sessions within - the passed `sub_names`. + A list of session-level names to validate against the + session names that exist in the project. Note that + duplicate checks will only be performed for sessions within + the passed `sub_names`. include_central - If `True`, only project folders in the `local_path` will - be validated against. Otherwise, project folders in both the - `local_path` and `central_path` will be validated against. + If `True`, only project folders in the `local_path` will + be validated against. Otherwise, project folders in both the + `local_path` and `central_path` will be validated against. display_mode - Determine whether error or warning is raised. + Determine whether error or warning is raised. log - If `True`, errors or warnings are logged to "datashuttle" logger. + If `True`, errors or warnings are logged to "datashuttle" logger. name_templates - A `name_template` dictionary to validate against. See `set_name_templates()`. + A `name_template` dictionary to validate against. See `set_name_templates()`. """ error_messages = [] @@ -780,7 +790,9 @@ def check_high_level_project_structure( def check_strict_mode( cfg: Configs, top_level_folder: TopLevelFolder, include_central: bool ) -> List[str]: - """`strict_mode` does not allow any non-NeuroBlueprint folder to exist + """Perform `strict_mode` validation. + + `strict_mode` does not allow any non-NeuroBlueprint folder to exist in the project outside the datatype folder. NeuroBlueprint folders are top-level folder or folder with sub-, ses- or datatype. @@ -882,9 +894,11 @@ def strip_uncheckable_names( path_or_names_list: List[Path] | List[str], prefix: Prefix, ) -> List[Path] | List[str]: - """Convenience function to remove any name in which the `prefix` value (sub or ses typically) cannot be converted into an integer. + """Remove any name in which the `prefix` value (sub or ses typically) cannot be converted into an integer. - This is necessary as some validation steps (e.g. checking duplicate names) requires conversion to `int` and will fail for this reason if these bad names are not removed. + This is necessary as some validation steps (e.g. checking duplicate names) + requires conversion to `int` and will fail for this reason if + these bad names are not removed. """ new_list = [] From 3421d94002aca0d2d81374d6bde4f99747d231a4 Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Sun, 22 Jun 2025 12:44:13 +0100 Subject: [PATCH 47/70] Update tooltips.py --- datashuttle/tui/tooltips.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/datashuttle/tui/tooltips.py b/datashuttle/tui/tooltips.py index 20ebba53f..63c7767ec 100644 --- a/datashuttle/tui/tooltips.py +++ b/datashuttle/tui/tooltips.py @@ -1,6 +1,11 @@ def get_tooltip(id: str) -> str: - """Master function to get tooltips for all widgets, - based on their widget (textual) id. + """Return tooltip for a widget based on its textual id. + + Parameters + ---------- + id + Textual widget i.d. (e.g. "#configs_local_path_input"). + """ # Main App Window # ------------------------------------------------------------------------- From a96b06128e2ebdf1da3051c4296ba06bdfdd3af8 Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Sun, 22 Jun 2025 12:56:29 +0100 Subject: [PATCH 48/70] Update app.py --- datashuttle/tui/app.py | 88 ++++++++++++++++++++++++++++++------------ 1 file changed, 64 insertions(+), 24 deletions(-) diff --git a/datashuttle/tui/app.py b/datashuttle/tui/app.py index c4c2dcf25..85fb56676 100644 --- a/datashuttle/tui/app.py +++ b/datashuttle/tui/app.py @@ -49,7 +49,7 @@ class TuiApp(App, inherit_bindings=False): # type: ignore ] def compose(self) -> ComposeResult: - """PLACEHOLDER.""" + """Set up widgets for the main window.""" yield Container( Label("datashuttle", id="mainwindow_banner_label"), Button( @@ -67,18 +67,26 @@ def compose(self) -> ComposeResult: ) def on_mount(self) -> None: - """PLACEHOLDER.""" + """Update widgets immediately after creation.""" self.set_dark_mode(self.load_global_settings()["dark_mode"]) id = "#mainwindow_validate_from_project_path" self.query_one(id).tooltip = get_tooltip(id) def set_dark_mode(self, dark_mode: bool) -> None: - """PLACEHOLDER.""" + """Set the color theme for the application.""" self.theme = "textual-dark" if dark_mode else "textual-light" def on_button_pressed(self, event: Button.Pressed) -> None: - """Raise the relevant screen after button press. `push_screen` - second argument is a callback function returned after screen closes. + """Handle a button press on the main window and raise a new screen. + + Note that`push_screen` second argument is a callback function returned + after screen closes. + + Parameters + ---------- + event + A textual event containing information about the button press. + """ if event.button.id == "mainwindow_existing_project_button": self.push_screen( @@ -106,7 +114,16 @@ def on_button_pressed(self, event: Button.Pressed) -> None: self.push_screen(validate_at_path.ValidateScreen(self)) def load_project_page(self, interface: Interface) -> None: - """PLACEHOLDER.""" + """Load the project manager page. + + This page contains a list of all existing projects. + + Parameters + ---------- + interface + Datashuttle tui Interface object. + + """ if interface: self.push_screen( project_manager.ProjectManagerScreen( @@ -115,19 +132,24 @@ def load_project_page(self, interface: Interface) -> None: ) def show_modal_error_dialog(self, message: str) -> None: - """PLACEHOLDER.""" + """Show an error in a pop-up window. + + The pop-up window has a red border and is modal (cannot click + elsewhere on the application when it is shown). + + Parameter + --------- + message + Error message to display. + """ self.push_screen(modal_dialogs.MessageBox(message, border_color="red")) def show_modal_error_dialog_from_main_thread(self, message: str) -> None: - """Used to call `show_modal_error_dialog from main thread when executing - in another thread. Throws error when called from main thread. - """ + """Call`show_modal_error_dialog from main thread when executing in another thread.""" self.call_from_thread(self.show_modal_error_dialog, message) def handle_open_filesystem_browser(self, path_: Path) -> None: - """Open the system file browser to the path with the `showinfm` - package, performing checks that the path exists prior to opening. - """ + """Open the system file browser to the path with the `showinfm` package.""" if not path_.exists(): self.show_modal_error_dialog( f"{path_.as_posix()} cannot be opened as it " @@ -154,14 +176,30 @@ def handle_open_filesystem_browser(self, path_: Path) -> None: self.show_modal_error_dialog(message) def prompt_rename_file_or_folder(self, path_): - """PLACEHOLDER.""" + """Display pop-up window to rename a file or folder in a tab DirectoryTree. + + Todo: + ---- + Can this not be moved to the relevant tab page? + + """ self.push_screen( modal_dialogs.RenameFileOrFolderScreen(self, path_), lambda new_name: self.rename_file_or_folder(path_, new_name), ) def rename_file_or_folder(self, path_, new_name): - """PLACEHOLDER.""" + """Rename a file or folder within the project. + + Parameters + ---------- + path_ + Path to the file or folder to rename. + + new_name + New name for the file or folder. + + """ if new_name is False: return try: @@ -190,10 +228,11 @@ def rename_file_or_folder(self, path_, new_name): # Global Settings --------------------------------------------------------- def load_global_settings(self) -> Dict: - """Load the 'global settings' for the TUI that determine - project-independent settings that are persistent across - sessions. These are stored in the canonical - .datashuttle folder (see `get_global_settings_path`). + """Load the 'global settings' for the TUI. + + These settings determine project-independent settings + that are persistent across sessions. These are stored + in the canonical .datashuttle folder (see `get_global_settings_path`). """ settings_path = self.get_global_settings_path() @@ -207,19 +246,19 @@ def load_global_settings(self) -> Dict: return global_settings def get_global_settings_path(self) -> Path: - """The canonical path for the TUI's global settings.""" + """Return the canonical path for the TUI global settings.""" path_ = canonical_folders.get_datashuttle_path() return path_ / "global_tui_settings.yaml" def get_default_global_settings(self) -> Dict: - """PLACEHOLDER.""" + """Return placeholder default global settings for the TUI.""" return { "dark_mode": True, "show_transfer_tree_status": False, } def save_global_settings(self, global_settings: Dict) -> None: - """PLACEHOLDER.""" + """Save the TUI global settings to disk.""" settings_path = self.get_global_settings_path() if not settings_path.parent.is_dir(): @@ -230,7 +269,8 @@ def save_global_settings(self, global_settings: Dict) -> None: def copy_to_clipboard(self, value): """Centralized function to copy to clipboard. - This may fail under some circumstances (e.g., in headless mode on an HPC). + + This may fail in headless mode on an HPC. """ try: pyperclip.copy(value) @@ -241,7 +281,7 @@ def copy_to_clipboard(self, value): def main(): - """PLACEHOLDER.""" + """Start the application.""" TuiApp().run() From 3ce7b1f0b19921c0b074983cb0065574e7206f1f Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Sun, 22 Jun 2025 13:21:14 +0100 Subject: [PATCH 49/70] Update custom widgets. --- datashuttle/tui/custom_widgets.py | 138 +++++++++++++++++++++++------- 1 file changed, 106 insertions(+), 32 deletions(-) diff --git a/datashuttle/tui/custom_widgets.py b/datashuttle/tui/custom_widgets.py index 52a36fcfd..aa4c8a493 100644 --- a/datashuttle/tui/custom_widgets.py +++ b/datashuttle/tui/custom_widgets.py @@ -39,14 +39,14 @@ # ClickableInput # -------------------------------------------------------------------------------------- class ClickableInput(Input): - """An input widget which emits a `ClickableInput.Clicked` - signal when clicked, containing the input name - `input` and mouse button index `button`. + """An input widget which emits a `ClickableInput.Clicked` event when clicked. + + The event contains the input name `input` and mouse button index `button`. """ @dataclass class Clicked(Message): - """PLACEHOLDER.""" + """The event class.""" input: ClickableInput ctrl: bool @@ -59,7 +59,26 @@ def __init__( validate_on: Optional[List[str]] = None, validators: Optional[List[Validator]] = None, ) -> None: - """PLACEHOLDER.""" + """Initialise the Clicked event class. + + Parameters + ---------- + mainwindow + The Datashuttle TUI application + + placeholder + Placeholder (i.e. example) text for the Input. + + id + Textual ID of the Input. + + validate_on + Textual keywords specifying actions to validate on (e.g. ["changed", "submitted"]) + + validators + Textual validator objects to apply. + + """ super(ClickableInput, self).__init__( placeholder=placeholder, id=id, @@ -70,14 +89,15 @@ def __init__( self.mainwindow = mainwindow def _on_click(self, event: events.Click) -> None: + """Handle when the input is clicked.""" self.post_message(self.Clicked(self, event.ctrl)) def as_names_list(self) -> List[str]: - """PLACEHOLDER.""" + """Return the contents of the input as a list split by ','.""" return self.value.replace(" ", "").split(",") def on_key(self, event: events.Key) -> None: - """PLACEHOLDER.""" + """Handle keyboard press on the Input.""" if event.key == "ctrl+q": self.mainwindow.copy_to_clipboard(self.value) @@ -91,14 +111,16 @@ def on_key(self, event: events.Key) -> None: class CustomDirectoryTree(DirectoryTree): - """Base class for directory tree with some customised additions: + """Base class for a custom directory tree. + + This has some customised additions: - filter out top-level folders that are not canonical - add additional keyboard shortcuts defined in `on_key`. """ @dataclass class DirectoryTreeSpecialKeyPress(Message): - """PLACEHOLDER.""" + """Event to handle a key press on the CustomDirectoryTree.""" key: str node_path: Optional[Path] @@ -106,21 +128,46 @@ class DirectoryTreeSpecialKeyPress(Message): def __init__( self, mainwindow: TuiApp, path: Path, id: Optional[str] = None ) -> None: - """PLACEHOLDER.""" + """Initialise the Directory Tree. + + Parameters + ---------- + mainwindow + Datashuttle TUI application. + + path + Root path for the CustomDirectoryTree. + + id + Textual ID for the CustomDirectoryTree. + + """ super(CustomDirectoryTree, self).__init__(path=path, id=id) self.mainwindow = mainwindow def filter_paths(self, paths: Iterable[Path]) -> Iterable[Path]: - """Filter out all hidden folders and files from DirectoryTree - display. + """Filter out all hidden folders and files from CustomDirectoryTree display. + + Parameters + ---------- + paths + Paths to be filtered before being added to the CustomDirectoryTree. + """ return [path for path in paths if not path.name.startswith(".")] def on_key(self, event: events.Key) -> None: - """Handle key presses on the CustomDirectoryTree. Depending on the keys pressed, - copy the path under the cursor, refresh the directorytree or - emit a DirectoryTreeSpecialKeyPress event. + """Handle key presses on the CustomDirectoryTree. + + Depending on the keys pressed, copy the path under the cursor, + refresh the CustomDirectoryTree or emit a DirectoryTreeSpecialKeyPress event. + + Parameters + ---------- + event + Textual event containing information on the key press. + """ if event.key == "ctrl+q": path_ = self.get_node_at_line(self.hover_line).data.path @@ -148,7 +195,9 @@ def on_key(self, event: events.Key) -> None: def _render_line( self, y: int, x1: int, x2: int, base_style: Style ) -> Strip: - """Overridden from textual's `Tree` class to stop + """Overridden function that renders CustomDirectoryTree lines. + + Overridden from textual's `Tree` class to stop CSS styling on hovering and clicking which was distracting / changed the default color used for transfer status, respectively. @@ -199,11 +248,14 @@ def _render_line( def get_guides(style: Style) -> tuple[str, str, str, str]: """Get the guide strings for a given style. - Args: - style: A Style object. + Parameters + ---------- + style + A Style object. - Returns: - Strings for space, vertical, terminator and cross. + Returns + ------- + Strings for space, vertical, terminator and cross. """ lines: tuple[ @@ -292,7 +344,9 @@ def get_guides(style: Style) -> tuple[str, str, str, str]: class TreeAndInputTab(TabPane): - """A parent class that defined common methods for screens with + """High level class for Custom DirectoryTrees and Inputs. + + A parent class that defined common methods for screens with a directory tree and sub / session inputs, .e. the Create tab and the Transfer tab. """ @@ -300,7 +354,9 @@ class TreeAndInputTab(TabPane): def handle_fill_input_from_directorytree( self, sub_input_key: str, ses_input_key: str, event: events.Key ) -> None: - """When a CustomDirectoryTree key is pressed, we typically + """Handle interactions between CustomDirectoryTree and Inputs. + + When a CustomDirectoryTree key is pressed, we typically want to perform an action that involves an Input. These are coordinated here. Note that the 'copy' and 'refresh' features of the tree is handled at the level of the @@ -345,11 +401,19 @@ def handle_fill_input_from_directorytree( def insert_sub_or_ses_name_to_input( self, sub_input_key: str, ses_input_key: str, name: str ) -> None: - """See `handle_directorytree_key_pressed` for `sub_input_key` and - `ses_input_key`. + """Fill an input with the CustomDirectoryTree subject or session folder name under mouse. + + Parameters + ---------- + sub_input_key + The textual widget id for the subject input (prefixed with #) + + ses_input_key + The textual widget id for the session input (prefixed with #) name The sub or ses name to append to the input. + """ if name.startswith("sub-"): self.query_one(sub_input_key).value = name @@ -384,8 +448,9 @@ def get_sub_ses_names_and_datatype( class TopLevelFolderSelect(Select): - """A Select widget for display and updating of top-level-folders. The - Create tab and transfer tabs (custom, top-level-folder) all have + """A Select widget for display and updating of top-level-folders. + + The Create tab and transfer tabs (custom, top-level-folder) all have top level folder selects that perform the same function. This widget unifies these in a single place. @@ -406,7 +471,17 @@ class TopLevelFolderSelect(Select): """ def __init__(self, interface: Interface, id: str) -> None: - """PLACEHOLDER.""" + """Initialise the TopLevelFolderSelect. + + Parameters + ---------- + interface + Datashuttle Interface object. + + id + Textual id for the Select widget. + + """ self.interface = interface top_level_folders = [ @@ -438,8 +513,9 @@ def __init__(self, interface: Interface, id: str) -> None: ) def get_top_level_folder(self, init: bool = False) -> str: - """Get the top level folder from `persistent_settings`, - performing a confidence-check that it matches the textual display. + """Get the top level folder from `persistent_settings`. + + Performs a confidence-check that it matches the textual display. """ top_level_folder = self.interface.tui_settings[ "top_level_folder_select" @@ -453,9 +529,7 @@ def get_top_level_folder(self, init: bool = False) -> str: return top_level_folder def get_displayed_top_level_folder(self) -> str: - """Get the top level folder that is currently selected - on the select widget. - """ + """Get the top level folder that is currently selected on the select widget.""" assert self.value in canonical_folders.get_top_level_folders() return self.value From 681e16ee00a50367bc78b9851a48567a30e2655c Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Sun, 22 Jun 2025 13:42:21 +0100 Subject: [PATCH 50/70] Update interface.py --- datashuttle/tui/interface.py | 68 ++++++++++++++++++++---------------- 1 file changed, 38 insertions(+), 30 deletions(-) diff --git a/datashuttle/tui/interface.py b/datashuttle/tui/interface.py index 6fad2b592..3d9cfafc8 100644 --- a/datashuttle/tui/interface.py +++ b/datashuttle/tui/interface.py @@ -15,10 +15,11 @@ class Interface: - """An interface class between the TUI and datashuttle API. Takes input - to all datashuttle functions as passed from the TUI, outputs - success status (True or False) and optional data, in the case - of False. + """An interface class between the TUI and datashuttle API. + + Takes input to all datashuttle functions as passed from the TUI, + outputs success status (True or False) and optional data, in the + case of False. `self.project` is initialised when project is loaded. @@ -30,7 +31,7 @@ class Interface: """ def __init__(self) -> None: - """PLACEHOLDER.""" + """Initialise the Interface class.""" self.project: DataShuttle self.name_templates: Dict = {} self.tui_settings: Dict = {} @@ -82,8 +83,9 @@ def setup_new_project( def set_configs_on_existing_project( self, cfg_kwargs: Dict ) -> InterfaceOutput: - """Update the settings on an existing project. Only the settings - passed in `cfg_kwargs` are updated. + """Update the settings on an existing project. + + Only the settings passed in `cfg_kwargs` are updated. Parameters ---------- @@ -139,8 +141,9 @@ def create_folders( def validate_names( self, sub_names: List[str], ses_names: Optional[List[str]] ) -> InterfaceOutput: - """Validate a list of subject / session names. This is used - to populate the Input tooltips with validation errors. + """Validate a list of subject / session names. + + This is used to populate the Input tooltips with validation errors. Uses a central `_format_and_validate_names()` that is also called during folder creation itself, to ensure these a results always match. @@ -181,8 +184,9 @@ def validate_project( include_central: bool, strict_mode: bool, ) -> tuple[bool, list[str] | str]: - """Wrap the validate project function. This returns a list of validation - errors (empty if there are none). + """Wrap the validate project function. + + This returns a list of validation errors (empty if there are none). Parameters ---------- @@ -336,11 +340,12 @@ def transfer_custom_selection( # ---------------------------------------------------------------------------------- def get_name_templates(self) -> Dict: - """Get the `name_templates` defining templates to validate - against. These are stored in a variable to avoid constantly + """Get the `name_templates` defining templates to validate against. + + These are stored in a variable to avoid constantly reading these values from disk where they are stored in `persistent_settings`. It is critical this variable - and the file contetns are in sync, so when changed + and the file contents are in sync, so when changed on the TUI side they are updated also, in `get_tui_settings`. """ if not self.name_templates: @@ -349,8 +354,9 @@ def get_name_templates(self) -> Dict: return self.name_templates def set_name_templates(self, name_templates: Dict) -> InterfaceOutput: - """Set the `name_templates` here and on disk. See `get_name_templates` - for more information. + """Set the `name_templates` here and on disk. + + See `get_name_templates` for more information. """ try: self.project.set_name_templates(name_templates) @@ -361,9 +367,10 @@ def set_name_templates(self, name_templates: Dict) -> InterfaceOutput: return False, str(e) def get_tui_settings(self) -> Dict: - """Get the "tui" field of `persistent_settings`. Similar to - `get_name_templates`, there are held on the class to avoid - constantly reading from disk. + """Get the "tui" field of `persistent_settings`. + + Similar to `get_name_templates`, there are held on the + class to avoid constantly reading from disk. """ if not self.tui_settings: self.tui_settings = self.project._load_persistent_settings()["tui"] @@ -400,18 +407,19 @@ def save_tui_settings( # ---------------------------------------------------------------------------------- def get_central_host_id(self) -> str: - """PLACEHOLDER.""" + """Get the central host id for ssh.""" return self.project.cfg["central_host_id"] def get_configs(self) -> Configs: - """PLACEHOLDER.""" + """Get Datashuttle Configs.""" return self.project.cfg def get_textual_compatible_project_configs(self) -> Configs: - """Datashuttle configs keeps paths saved as pathlib.Path - objects. In some cases textual requires str representation. - This method returns datashuttle configs with all paths that - are Path converted to str. + """Datashuttle configs keeps paths saved as pathlib.Path objects. + + In some cases textual requires str representation. This method + returns datashuttle configs with all paths that are Path + converted to str. """ cfg_to_load = copy.deepcopy(self.project.cfg) load_configs.convert_str_and_pathlib_paths(cfg_to_load, "path_to_str") @@ -420,7 +428,7 @@ def get_textual_compatible_project_configs(self) -> Configs: def get_next_sub( self, top_level_folder: TopLevelFolder, include_central: bool ) -> InterfaceOutput: - """PLACEHOLDER.""" + """Get the next subject ID in the project.""" try: next_sub = self.project.get_next_sub( top_level_folder, @@ -434,7 +442,7 @@ def get_next_sub( def get_next_ses( self, top_level_folder: TopLevelFolder, sub: str, include_central: bool ) -> InterfaceOutput: - """PLACEHOLDER.""" + """Get the next session ID for the `sub` in the project.""" try: next_ses = self.project.get_next_ses( top_level_folder, @@ -447,7 +455,7 @@ def get_next_ses( return False, str(e) def get_ssh_hostkey(self) -> InterfaceOutput: - """PLACEHOLDER.""" + """Get the SSH remote server host key.""" try: key = ssh.get_remote_server_key( self.project.cfg["central_host_id"] @@ -457,7 +465,7 @@ def get_ssh_hostkey(self) -> InterfaceOutput: return False, str(e) def save_hostkey_locally(self, key: paramiko.RSAKey) -> InterfaceOutput: - """PLACEHOLDER.""" + """Save the SSH hostkey to disk.""" try: ssh.save_hostkey_locally( key, @@ -472,7 +480,7 @@ def save_hostkey_locally(self, key: paramiko.RSAKey) -> InterfaceOutput: def setup_key_pair_and_rclone_config( self, password: str ) -> InterfaceOutput: - """PLACEHOLDER.""" + """Set up SSH key pair and associated rclone configuration.""" try: ssh.add_public_key_to_central_authorized_keys( self.project.cfg, password, log=False From 4221de76dc088198e4c5ada8c6a95fde2717c7a4 Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Sun, 22 Jun 2025 13:44:36 +0100 Subject: [PATCH 51/70] Update decorators.py --- datashuttle/tui/utils/tui_decorators.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/datashuttle/tui/utils/tui_decorators.py b/datashuttle/tui/utils/tui_decorators.py index 2de10d811..8a2e493d6 100644 --- a/datashuttle/tui/utils/tui_decorators.py +++ b/datashuttle/tui/utils/tui_decorators.py @@ -13,19 +13,21 @@ class ClickInfo: - """A class to hold click-info to checking - double clicks are within the time threshold - and match the widget id. + """A class to hold click-info. + + This stores click history to allow later checking + that double clicks occur within a time threshold + and that the same widget is clicked twice. """ def __init__(self): + """Initialise the ClickInfo.""" self.prev_click_time = 0.0 self.prev_click_widget_id = "" def require_double_click(func): - """A decorator that calls the decorated function - on a double click, otherwise will not do anything. + """Call the decorated function on a double click. Requires the first argument (`self` on the class) to have the attribute `click_info`). Any class holding a widget From e8ded8d2e8a0fa960b8bad465d0aff33a45807a6 Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Sun, 22 Jun 2025 13:49:22 +0100 Subject: [PATCH 52/70] Update tui_validators.py --- datashuttle/tui/utils/tui_validators.py | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/datashuttle/tui/utils/tui_validators.py b/datashuttle/tui/utils/tui_validators.py index 4124e4f44..916eb6241 100644 --- a/datashuttle/tui/utils/tui_validators.py +++ b/datashuttle/tui/utils/tui_validators.py @@ -12,21 +12,30 @@ class NeuroBlueprintValidator(Validator): - """PLACEHOLDER.""" + """A textual validator subclass for validating Input text. + + Takes the sub / ses prefix as input. Runs validation of + the name against the project and propagates + any error message through the Input tooltip. + """ def __init__(self, prefix: Prefix, parent: CreateFoldersTab) -> None: - """Custom Validator() class that takes - sub / ses prefix as input. Runs validation of - the name against the project and propagates - any error message through the Input tooltip. + """Initialise the NeuroBlueprintValidator. + + prefix + "sub" or "ses". + + parent + The tab on which the validated input exists. """ super(NeuroBlueprintValidator, self).__init__() self.parent = parent self.prefix = prefix def validate(self, name: str) -> ValidationResult: - """Run validation and update the tooltip with the error, - if no error then the formatted sub / ses name is displayed. + """Run validation and update the tooltip with the error. + + If no error then the formatted sub / ses name is displayed. This is set on an Input widget. """ valid, message = self.parent.run_local_validation(self.prefix) From 24918dbc348f16c7dbb92cad02fc6b9ec0575e36 Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Sun, 22 Jun 2025 14:03:11 +0100 Subject: [PATCH 53/70] Update configs_content.py --- datashuttle/tui/shared/configs_content.py | 91 ++++++++++++++++------- 1 file changed, 66 insertions(+), 25 deletions(-) diff --git a/datashuttle/tui/shared/configs_content.py b/datashuttle/tui/shared/configs_content.py index e7f7b044a..be3b3444e 100644 --- a/datashuttle/tui/shared/configs_content.py +++ b/datashuttle/tui/shared/configs_content.py @@ -31,6 +31,7 @@ class ConfigsContent(Container): """Holds widgets and logic for setting datashuttle configs. + It is used in `NewProjectPage` to instantiate a new project and initialise configs, or in `TabbedContent` to update an existing project's configs. @@ -40,11 +41,20 @@ class ConfigsContent(Container): additional information. Otherwise, widgets are filled with the existing projects configs. + + Note: + ---- + The natural design would be to have two classes (one for a new project, + one for an existing project) with a shared base class. The issue is that + for a new project, the screen becomes a set-up project once the config is saved. + Therefore, because the class mutates from a new-project config to existing-project + configs, this shared design is used. + """ @dataclass class ConfigsSaved(Message): - """PLACEHOLDER.""" + """An event signalling when the configs are saved.""" pass @@ -54,7 +64,20 @@ def __init__( interface: Optional[Interface], id: str, ) -> None: - """PLACEHOLDER.""" + """Initialise the ConfigsContent. + + Parameters + ---------- + parent_class + The Screen on which the contents container is mounted. + + interface + Datashuttle Interface object. + + id + Textual ID for the configs container. + + """ super(ConfigsContent, self).__init__(id=id) self.parent_class = parent_class @@ -62,7 +85,9 @@ def __init__( self.config_ssh_widgets: List[Any] = [] def compose(self) -> ComposeResult: - """`self.config_ssh_widgets` are SSH-setup related widgets + """Set up the Configs widgets. + + `self.config_ssh_widgets` are SSH-setup related widgets that are only required when the user selects the SSH connection method. These are displayed / hidden based on the `connection_method`. @@ -170,7 +195,9 @@ def compose(self) -> ComposeResult: yield Container(*config_screen_widgets, id="configs_container") def on_mount(self) -> None: - """When we have mounted the widgets, the following logic depends on whether + """Handle logic immediately following widget mounting. + + When we have mounted the widgets, the following logic depends on whether we are setting up a new project (`self.project is `None`) or have an instantiated project. @@ -222,8 +249,7 @@ def on_mount(self) -> None: self.query_one(id).tooltip = get_tooltip(id) def on_radio_set_changed(self, event: RadioSet.Changed) -> None: - """Update the displayed SSH widgets when the `connection_method` - radiobuttons are changed. + """Update the SSH widgets when the `connection_method` radiobuttons are changed. When SSH is set, ssh config-setters are shown. Otherwise, these are hidden. @@ -256,9 +282,7 @@ def on_radio_set_changed(self, event: RadioSet.Changed) -> None: self.set_central_path_input_tooltip(display_ssh) def set_central_path_input_tooltip(self, display_ssh: bool) -> None: - """Use a different tooltip depending on whether connection method - is ssh or local filesystem. - """ + """Set tooltip depending on whether connection method is SSH or local filesystem.""" id = "#configs_central_path_input" if display_ssh: self.query_one(id).tooltip = get_tooltip( @@ -272,7 +296,17 @@ def set_central_path_input_tooltip(self, display_ssh: bool) -> None: def get_platform_dependent_example_paths( self, local_or_central: Literal["local", "central"], ssh: bool = False ) -> str: - """PLACEHOLDER.""" + """Get example paths for the local or central Inputs depending on operating system. + + Parameters + ---------- + local_or_central + The "local" or "central" input to fill. + + ssh + If the user has selected SSH (which changes the central input). + + """ assert local_or_central in ["local", "central"] # Handle the ssh central case separately @@ -290,8 +324,10 @@ def get_platform_dependent_example_paths( return example_path def switch_ssh_widgets_display(self, display_ssh: bool) -> None: - """Show or hide SSH-related configs based on whether the current - `connection_method` widget is "ssh" or "local_filesystem". + """Show or hide SSH-related configs. + + This is based on whether the current `connection_method` + widget is "ssh" or "local_filesystem". Parameters ---------- @@ -325,8 +361,10 @@ def switch_ssh_widgets_display(self, display_ssh: bool) -> None: ).placeholder = placeholder def on_button_pressed(self, event: Button.Pressed) -> None: - """Enables the Create Folders button to read out current input values - and use these to call project.create_folders(). + """Handle a button press event. + + Enables the Create Folders button to read out current input + values and use these to call project.create_folders(). """ if event.button.id == "configs_save_configs_button": if not self.interface: @@ -362,8 +400,7 @@ def on_button_pressed(self, event: Button.Pressed) -> None: def handle_input_fill_from_select_directory( self, path_: Path, local_or_central: Literal["local", "central"] ) -> None: - """Update the `local` or `central` path inputs after - `SelectDirectoryTreeScreen` returns a path. + """Update the `local` or `central` Inputs after `SelectDirectoryTreeScreen` returns a path. Parameters ---------- @@ -403,9 +440,9 @@ def setup_ssh_connection(self) -> None: ) def widget_configs_match_saved_configs(self): - """Check that the configs currently stored in the widgets - on the screen match those stored in the app. This check - is to avoid user starting to set up SSH with unexpected + """Ensure configs as set on screen match those stored in the project object. + + This check is to avoid user starting to set up SSH with unexpected settings. It is a little fiddly as the Input for local and central path may or may not contain the project name. Therefore, need to check the stored values against @@ -425,7 +462,9 @@ def widget_configs_match_saved_configs(self): return True def setup_configs_for_a_new_project(self) -> None: - """If a project does not exist, we are in NewProjectScreen. + """Set up configs when the project does not exist. + + If a project does not exist, we are in NewProjectScreen. We need to instantiate a new project based on the project name, create configs based on the current widget settings, and display any errors to the user, along with confirmation and the @@ -484,7 +523,9 @@ def setup_configs_for_a_new_project(self) -> None: self.parent_class.mainwindow.show_modal_error_dialog(output) def setup_configs_for_an_existing_project(self) -> None: - """If the project already exists, we are on the TabbedContent + """Set up configs when the project already exists. + + If the project already exists, we are on the TabbedContent screen. We need to get the configs to set from the current widget values and display the set values (or an error if there was a problem during setup) to the user. @@ -512,7 +553,9 @@ def setup_configs_for_an_existing_project(self) -> None: self.parent_class.mainwindow.show_modal_error_dialog(output) def fill_widgets_with_project_configs(self) -> None: - """If a configured project already exists, we want to fill the + """Fill widgets on screen with content from the project config file. + + If a configured project already exists, we want to fill the widgets with the current project configs. This in some instances requires recasting to a new type of changing the value. @@ -574,9 +617,7 @@ def fill_widgets_with_project_configs(self) -> None: input.value = value def get_datashuttle_inputs_from_widgets(self) -> Dict: - """Get the configs to pass to `make_config_file()` from - the current TUI settings. - """ + """Get the configs to pass to `make_config_file()` from the current TUI settings.""" cfg_kwargs: Dict[str, Any] = {} cfg_kwargs["local_path"] = Path( From e6176ff916844533dd901179938d14757146358a Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Sun, 22 Jun 2025 14:12:39 +0100 Subject: [PATCH 54/70] Update validate_content.py --- datashuttle/tui/shared/validate_content.py | 43 +++++++++++++++++----- 1 file changed, 33 insertions(+), 10 deletions(-) diff --git a/datashuttle/tui/shared/validate_content.py b/datashuttle/tui/shared/validate_content.py index 87ae57b71..bbf53f65d 100644 --- a/datashuttle/tui/shared/validate_content.py +++ b/datashuttle/tui/shared/validate_content.py @@ -27,7 +27,19 @@ class ValidateContent(Container): - """PLACEHOLDER.""" + """A container containing widgets for project validation. + + This is shared between the Validate Project from Path + and validation tab on the project manager. It takes a similar + approach to ConfigsContent. + + Todo: + ---- + In contrast to ConfigsContent, these contents are always + split. This should probably be factored into two classes + with a shared base class. + + """ def __init__( self, @@ -37,13 +49,24 @@ def __init__( interface: Optional[Interface], id: str, ) -> None: + """Initialise the ValidateContent class. + + parent_class + The parent screen (project manager or validate from path). + + interface + Datashuttle Interface class. + + id + Textual ID for the container. + """ super(ValidateContent, self).__init__(id=id) self.parent_class = parent_class self.interface = interface def compose(self) -> ComposeResult: - """PLACEHOLDER.""" + """Set up the widgets for the container.""" if platform.system() == "Windows": example_path = r"C:\path\to\project\project_name" else: @@ -90,7 +113,7 @@ def compose(self) -> ComposeResult: yield Container(*widgets, id="validate_top_container") def on_mount(self) -> None: - """PLACEHOLDER.""" + """Handle the widgets immediately after they are mounted.""" for id in [ "validate_path_input", "validate_top_level_folder_select", @@ -113,13 +136,8 @@ def on_mount(self) -> None: else: self.query_one("#validate_include_central_checkbox").remove() - def set_select_path(self, path_): - """PLACEHOLDER.""" - if path_: - self.query_one("#validate_path_input").value = path_.as_posix() - def on_button_pressed(self, event: Button.Pressed) -> None: - """PLACEHOLDER.""" + """Handle a button press event.""" if event.button.id == "validate_select_button": self.parent_class.mainwindow.push_screen( modal_dialogs.SelectDirectoryTreeScreen( @@ -183,8 +201,13 @@ def on_button_pressed(self, event: Button.Pressed) -> None: ) self.write_results_to_richlog(output) + def set_select_path(self, path_): + """Fill the Input with the path_ if it is not None.""" + if path_: + self.query_one("#validate_path_input").value = path_.as_posix() + def write_results_to_richlog(self, results): - """PLACEHOLDER.""" + """Display the validation results on the Rich Log widget.""" text_log = self.query_one("#validate_richlog") text_log.clear() if any(results): From 1d4490bcf77185db62867f929f975e5486393259 Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Sun, 22 Jun 2025 14:46:05 +0100 Subject: [PATCH 55/70] Update create_folders.py. --- datashuttle/tui/tabs/create_folders.py | 104 +++++++++++++++---------- 1 file changed, 62 insertions(+), 42 deletions(-) diff --git a/datashuttle/tui/tabs/create_folders.py b/datashuttle/tui/tabs/create_folders.py index d36eb00d5..8c49bc424 100644 --- a/datashuttle/tui/tabs/create_folders.py +++ b/datashuttle/tui/tabs/create_folders.py @@ -47,7 +47,17 @@ class CreateFoldersTab(TreeAndInputTab): """Create new project files formatted according to the NeuroBlueprint specification.""" def __init__(self, mainwindow: TuiApp, interface: Interface) -> None: - """PLACEHOLDER.""" + """Initialise the CreateFoldersTab. + + Parameters + ---------- + mainwindow + The main TUI application. + + interface + Datashuttle Interface object. + + """ super(CreateFoldersTab, self).__init__( "Create", id="tabscreen_create_tab" ) @@ -60,7 +70,7 @@ def __init__(self, mainwindow: TuiApp, interface: Interface) -> None: self.click_info = ClickInfo() def compose(self) -> ComposeResult: - """PLACEHOLDER.""" + """Add widgets to the Create Folders tab.""" yield CustomDirectoryTree( self.mainwindow, self.interface.get_configs()["local_path"], @@ -105,7 +115,7 @@ def compose(self) -> ComposeResult: ) def on_mount(self) -> None: - """PLACEHOLDER.""" + """Handle the widgets immediately after mounting.""" if not self.interface: self.query_one("#configs_name_input").tooltip = get_tooltip( "#configs_name_input" @@ -122,11 +132,14 @@ def on_mount(self) -> None: self.query_one(id).tooltip = get_tooltip(id) def on_button_pressed(self, event: Button.Pressed) -> None: - """Enables the Create Folders button to read out current input values + """Handle a button press event. + + The Create Folders button to read out current Input values and use these to call project.create_folders(). - `unused_bool` is necessary to get dismiss to call - the callback. + `unused_bool` is necessary as dismiss() on the pushed screen + returns a bool. We use this to trigger revalidation regardless + of the bool state. """ if event.button.id == "create_folders_create_folders_button": self.create_folders() @@ -144,7 +157,7 @@ def on_button_pressed(self, event: Button.Pressed) -> None: ) async def refresh_after_datatypes_changed(self, ignore): - """PLACEHOLDER.""" + """Redisplay the datatype checkboxes.""" await self.recompose() self.on_mount() @@ -152,9 +165,9 @@ async def refresh_after_datatypes_changed(self, ignore): def on_clickable_input_clicked( self, event: ClickableInput.Clicked ) -> None: - """Handled a double click on the custom ClickableInput widget, - which indicates the input should be filled with a suggested value. + """Handle a double click on the custom ClickableInput widget. + A double click indicates the input should be filled with a suggested value. Determine if we have the subject or session input, and if it was a left or right click. Then, fill with either a generic suggestion or suggestion based on next sub / ses number. @@ -175,9 +188,10 @@ def on_clickable_input_clicked( def on_custom_directory_tree_directory_tree_special_key_press( self, event: CustomDirectoryTree.DirectoryTreeSpecialKeyPress ): - """Handle a key press on the directory tree, which can refresh the - directorytree or fill / append subject/session folder name to - the relevant input widget. + """Handle a key press on the CustomDirectoryTree. + + This can refresh the CustomDirectoryTree or fill / append + subject/session folder name to the relevant input widget. """ if event.key == "ctrl+r": self.reload_directorytree() @@ -193,9 +207,18 @@ def on_custom_directory_tree_directory_tree_special_key_press( self.mainwindow.prompt_rename_file_or_folder(event.node_path) def fill_input_with_template(self, prefix: Prefix, input_id: str) -> None: - """Given the `name_template`, fill the sub or ses - Input with the template (based on `prefix`). + """Fill the sub or ses Input with the name template. + If `self.templates` is off, then just suggest "sub-" or "ses-". + + Parameters + ---------- + prefix + "sub" or "ses" + + input_id + Textual ID of the Input to fill. + """ if self.templates_on(prefix): fill_value = self.interface.get_name_templates()[prefix] @@ -206,7 +229,7 @@ def fill_input_with_template(self, prefix: Prefix, input_id: str) -> None: input.value = fill_value def templates_on(self, prefix: Prefix) -> bool: - """PLACEHOLDER.""" + """Return `True` if the name templates are used in the project.""" return ( self.interface.get_name_templates()["on"] and self.interface.get_name_templates()[prefix] is not None @@ -216,9 +239,7 @@ def templates_on(self, prefix: Prefix) -> bool: # ---------------------------------------------------------------------------------- def revalidate_inputs(self, all_prefixes: List[str]) -> None: - """Revalidate and style both subject and session - inputs based on their value. - """ + """Revalidate and style both subject and session inputs based on their value.""" input_names = { "sub": "#create_folders_subject_input", "ses": "#create_folders_session_input", @@ -230,9 +251,7 @@ def revalidate_inputs(self, all_prefixes: List[str]) -> None: self.query_one(key).validate(value=value) def update_input_tooltip(self, message: List[str], prefix: Prefix) -> None: - """Update the value of a subject or session tooltip, which - indicates the validation status of the input value. - """ + """Update the value of a subject or session tooltip indicating the validation status.""" id = ( "#create_folders_subject_input" if prefix == "sub" @@ -249,9 +268,7 @@ def update_input_tooltip(self, message: List[str], prefix: Prefix) -> None: # ---------------------------------------------------------------------------------- def create_folders(self) -> None: - """Create project folders based on current widget input - through the datashuttle API. - """ + """Create project folders based on the information in TUI widgets.""" ses_names: Optional[List[str]] sub_names, ses_names, datatype = self.get_sub_ses_names_and_datatype( @@ -271,9 +288,13 @@ def create_folders(self) -> None: self.mainwindow.show_modal_error_dialog(output) def reload_directorytree(self) -> None: - """Reloads the directorytree and also updates validation. - Not now a good method name but done for consistency with other - tab refresh methods. + """Reload the DirectoryTree and also updates validation. + + Notes + ----- + No longer a good method name due to extended responsibilities + but done for consistency with other tab refresh methods. + """ self.revalidate_inputs(["sub", "ses"]) self.query_one("#create_folders_directorytree").reload() @@ -284,8 +305,9 @@ def reload_directorytree(self) -> None: def suggest_next_sub_ses( self, prefix: Prefix, input_id: str, include_central: bool ): - """Suggests the next sub/ses name for the project. Shows - a pop up screen in cases when searching for next sub/ses takes + """Suggests the next sub/ses name for the project. + + Shows a pop up screen in cases when searching for next sub/ses takes time such as searching central in SSH connection method. Creates an asyncio task which handles the suggestion logic and @@ -316,9 +338,10 @@ def suggest_next_sub_ses( async def fill_suggestion_and_dismiss_popup( self, prefix, input_id, include_central ): - """Runs the `fill_input_with_next_sub_or_ses_template` - worker and waiting for it to complete. If an error occurs in - `fill_input_with_next_sub_or_ses_template`, it dismisses the popup itself. + """Run the `fill_input_with_next_sub_or_ses_template` worker. + + Awaits completion. If an error occurs in `fill_input_with_next_sub_or_ses_template`, + it dismisses the popup itself. Else, if the worker successfully exits, this function handles dismissal of the popup. @@ -335,15 +358,14 @@ async def fill_suggestion_and_dismiss_popup( def fill_input_with_next_sub_or_ses_template( self, prefix: Prefix, input_id: str, include_central: bool ) -> Worker: - """Fills a sub / ses Input with a suggested name based on the - next subject / session in the project (local). + """Fill an Input the next subject / session in the project (local). If `name_templates` are set, then the sub- or ses- first key of the template name will be replaced with the suggested sub or ses key-value. Otherwise, the sub/ses key-value pair only will be suggested. - It runs in a worker thread so as to allow the TUI to show a loading + It runs in a worker thread as to allow the TUI to show a loading animation. Parameters @@ -418,10 +440,10 @@ def fill_input_with_next_sub_or_ses_template( def dismiss_popup_and_show_modal_error_dialog_from_thread( self, message: str ) -> None: - """Utility function that the `fill_input_with_next_sub_or_ses_template` - worker calls to display error dialog an if an error occurs while suggesting - the next sub/ses. Handles the TUI widget manipulation from the main thread - when called from within a worker thread. + """Handle an error while suggesting the next session or subject. + + Called by `fill_input_with_next_sub_or_ses_template` to display error + dialog an if an error occurs while suggesting the next sub/ses. """ if self.searching_central_popup_widget: self.mainwindow.call_from_thread( @@ -435,9 +457,7 @@ def dismiss_popup_and_show_modal_error_dialog_from_thread( # ---------------------------------------------------------------------------------- def run_local_validation(self, prefix: Prefix): - """Run validation of the values stored in the - sub / ses Input according to the passed prefix - using core datashuttle functions. + """Run validation of the values stored in the subject / session Inputs. First, format the subject name (and session if required) which also performs quick name format validations. Then, From 71aaa664af945eb47d97f857e505827c170d3017 Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Sun, 22 Jun 2025 14:50:18 +0100 Subject: [PATCH 56/70] Update transfer_status_tree.py --- datashuttle/tui/tabs/transfer_status_tree.py | 49 +++++++++++++------- 1 file changed, 32 insertions(+), 17 deletions(-) diff --git a/datashuttle/tui/tabs/transfer_status_tree.py b/datashuttle/tui/tabs/transfer_status_tree.py index 0c6b8b442..9993849a1 100644 --- a/datashuttle/tui/tabs/transfer_status_tree.py +++ b/datashuttle/tui/tabs/transfer_status_tree.py @@ -23,16 +23,10 @@ class TransferStatusTree(CustomDirectoryTree): - """A directorytree in which the nodes are styled depending on their - transfer status. e.g. indicates whether files are changed between - local or central, or appear in local only. - - Attributes - ---------- - Keep the local path as a string, linked to project.cfg["local_path"], - so that no conversion to string is necessary in `format_transfer_label` - which is called many times. + """A DirectoryTree in which the nodes are styled depending on their transfer status. + Foe example, indicates whether files are changed between + local or central, or appear in local only. """ def __init__( @@ -41,7 +35,26 @@ def __init__( interface: Interface, id: Optional[str] = None, ): - """PLACEHOLDER.""" + """Initialise the TransferStatusTree. + + Parameters + ---------- + mainwindow + The main TUI application. + + interface + Datashuttle Interface object. + + id + Textual ID for the TransferStatusTree. + + Attributes + ---------- + Keep the local path as a string, linked to project.cfg["local_path"], + so that no conversion to string is necessary in `format_transfer_label` + which is called many times. + + """ self.interface = interface self.local_path_str = self.interface.get_configs()[ "local_path" @@ -53,13 +66,11 @@ def __init__( ) def on_mount(self) -> None: - """PLACEHOLDER.""" + """Update the directory tree after the widget is mounted.""" self.update_transfer_tree(init=True) def update_transfer_tree(self, init: bool = False) -> None: - """Updates tree styling to reflect the current TUI state - and project transfer status. - """ + """Update tree styling to reflect the current TUI state and project transfer status.""" self.local_path_str = self.interface.get_configs()[ "local_path" ].as_posix() @@ -87,7 +98,7 @@ def update_local_transfer_paths(self) -> None: self.transfer_paths = paths_list def update_transfer_diffs(self) -> None: - """Updates the transfer diffs used to style the DirectoryTree.""" + """Update the transfer diffs used to style the DirectoryTree.""" self.transfer_diffs = get_local_and_central_file_differences( self.interface.get_configs(), top_level_folders_to_check=["rawdata", "derivatives"], @@ -99,7 +110,9 @@ def update_transfer_diffs(self) -> None: def render_label( self, node: TreeNode[DirEntry], base_style: Style, style: Style ) -> Text: - """Extends the `DirectoryTree.render_label()` method to allow + """Handle label rendering on the TransferStatusTree. + + Extends the `DirectoryTree.render_label()` method to allow custom styling of file nodes according to their transfer status. """ node_label = node._label.copy() @@ -141,7 +154,9 @@ def render_label( return text def format_transfer_label(self, node_label, node_path) -> None: - """Takes nodes being formatted using `render_label` and applies custom + """Format the folder label depending on its transfer status. + + Takes nodes being formatted using `render_label` and applies custom formatting according to the node's transfer status. """ node_relative_path = node_path.as_posix().replace( From 5728b45e9985859dd5c77ea3d4e56520071fb5a0 Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Sun, 22 Jun 2025 15:10:08 +0100 Subject: [PATCH 57/70] Update logging.py. --- datashuttle/tui/tabs/logging.py | 54 +++++++++++++++++++++++---------- 1 file changed, 38 insertions(+), 16 deletions(-) diff --git a/datashuttle/tui/tabs/logging.py b/datashuttle/tui/tabs/logging.py index fa41710b2..cfe496cdd 100644 --- a/datashuttle/tui/tabs/logging.py +++ b/datashuttle/tui/tabs/logging.py @@ -8,6 +8,9 @@ from textual import events from textual.widgets import DirectoryTree + from datashuttle import DataShuttle + from datashuttle.tui.app import TuiApp + from textual.containers import Container, Horizontal from textual.screen import ModalScreen from textual.widgets import Button, Label, RichLog, TabPane @@ -22,38 +25,56 @@ class RichLogScreen(ModalScreen): - """PLACEHOLDER.""" + """Screen to display the log output.""" def __init__(self, log_file): - """PLACEHOLDER.""" + """Initialise the RichLogScreen.""" super(RichLogScreen, self).__init__() with open(log_file) as file: self.log_contents = "".join(file.readlines()) def compose(self): - """PLACEHOLDER.""" + """Set the widgets for the screen.""" yield Container( RichLog(highlight=True, markup=True, id="richlog_screen_rich_log"), Button("Close", id="richlog_screen_close_button"), ) def on_mount(self): - """PLACEHOLDER.""" + """Update widgets immediately after mount.""" text_log = self.query_one(RichLog) text_log.write(self.log_contents) def on_button_pressed(self, event): - """PLACEHOLDER.""" + """Handle a button press on the screen.""" if event.button.id == "richlog_screen_close_button": self.dismiss() class LoggingTab(TabPane): - """PLACEHOLDER.""" + """The logging tab on the project manager screen.""" + + def __init__( + self, title: str, mainwindow: TuiApp, project: DataShuttle, id: str + ): + """Initialise the Logging Tab. + + Parameters + ---------- + title + Title for the tab. + + mainwindow + Tui main appl + + project + DataShuttle project. + + id + Textual ID for the LoggingTab. - def __init__(self, title, mainwindow, project, id): - """PLACEHOLDER.""" + """ super(LoggingTab, self).__init__(title=title, id=id) self.mainwindow = mainwindow @@ -66,7 +87,7 @@ def __init__(self, title, mainwindow, project, id): self.click_info = ClickInfo() def update_latest_log_path(self): - """PLACEHOLDER.""" + """Set the `latest_log_path` attribute that can be opened through a button.""" logs = list(self.project.get_logging_path().glob("*.log")) self.latest_log_path = ( max(logs, key=os.path.getctime) @@ -75,7 +96,7 @@ def update_latest_log_path(self): ) def compose(self): - """PLACEHOLDER.""" + """Set with widgets on the LoggingTab.""" yield Container( Label( "Double click logging file to select:", @@ -100,10 +121,11 @@ def compose(self): ) def _on_mount(self, event: events.Mount) -> None: + """Update the widgets immediately after mounting.""" self.update_most_recent_label() def update_most_recent_label(self): - """PLACEHOLDER.""" + """Update the label indicating the most recently saved log.""" self.update_latest_log_path() self.query_one("#logging_most_recent_label").update( f"or open most recent: {self.latest_log_path.stem}" @@ -111,7 +133,7 @@ def update_most_recent_label(self): self.refresh() def on_button_pressed(self, event): - """PLACEHOLDER.""" + """Handle button press on the tab.""" if event.button.id == "logging_tab_open_most_recent_button": self.push_rich_log_screen(self.latest_log_path) @@ -119,7 +141,7 @@ def on_button_pressed(self, event): def on_directory_tree_file_selected( self, event: DirectoryTree.FileSelected ): - """PLACEHOLDER.""" + """Handle a click on the DirectoryTree showing the log files.""" if not event.path.is_file(): self.mainwindow.show_modal_error_dialog( "Log file no longer exists. Refresh the directory tree" @@ -130,7 +152,7 @@ def on_directory_tree_file_selected( self.push_rich_log_screen(event.path) def push_rich_log_screen(self, log_path): - """PLACEHOLDER.""" + """Push the screen that displays the log file contents.""" self.mainwindow.push_screen( RichLogScreen( log_path, @@ -138,9 +160,9 @@ def push_rich_log_screen(self, log_path): ) def reload_directorytree(self): - """PLACEHOLDER.""" + """Refresh the DirectoryTree (e.g. if a new log file is saved).""" self.query_one("#logging_tab_custom_directory_tree").reload() def on_custom_directory_tree_directory_tree_special_key_press(self): - """PLACEHOLDER.""" + """Handle the CTRL+R refresh of the directory tree.""" self.reload_directorytree() From 99b4a348c0f836dc8b1240fb19efeb95f8d1cf8d Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Sun, 22 Jun 2025 15:31:32 +0100 Subject: [PATCH 58/70] Update transfer.py --- datashuttle/tui/tabs/transfer.py | 93 ++++++++++++++++---------------- 1 file changed, 47 insertions(+), 46 deletions(-) diff --git a/datashuttle/tui/tabs/transfer.py b/datashuttle/tui/tabs/transfer.py index 8fe58a8ed..97e6631c9 100644 --- a/datashuttle/tui/tabs/transfer.py +++ b/datashuttle/tui/tabs/transfer.py @@ -43,37 +43,12 @@ class TransferTab(TreeAndInputTab): - """Handles the upload / download of files between local + """The Project Manager's Transfer tab. + + Handles the upload / download of files between local and central folders. It contains a TransferDirectoryTree that displays the transfer status of the files in the local folder, and calls underlying datashuttle transfer functions. - - Parameters - ---------- - title - The title of the tab - - mainwindow - The main TUI app - - interface - TUI-datashuttle interface object - - id - The textual widget id. - - Attributes - ---------- - show_legend - Convenience attribute linked to a global setting exists that - turns off / on styling of directorytree nodes based on transfer status. ` - - `self.mainwindow.load_global_settings()[ - "show_transfer_tree_status" - ]` - - When on, the legend must be hidden. - """ def __init__( @@ -83,7 +58,34 @@ def __init__( interface: Interface, id: Optional[str] = None, ) -> None: - """PLACEHOLDER.""" + """Initialise the TransferTab. + + Parameters + ---------- + title + The title of the tab + + mainwindow + The main TUI app + + interface + TUI-datashuttle interface object + + id + The textual widget id. + + Attributes + ---------- + show_legend + Convenience attribute linked to a global setting exists that + turns off / on styling of DirectoryTree nodes based on transfer status. ` + + `self.mainwindow.load_global_settings()[ + "show_transfer_tree_status" + ]` + When on, the legend must be hidden. + + """ super(TransferTab, self).__init__(title, id=id) self.mainwindow = mainwindow self.interface = interface @@ -96,7 +98,7 @@ def __init__( # ---------------------------------------------------------------------------------- def compose(self) -> ComposeResult: - """PLACEHOLDER.""" + """Set the widgets on the Transfer Tab.""" self.transfer_all_widgets = [ Label( "All data from: \n\n - Rawdata \n - Derivatives \n\nwill be transferred.", @@ -210,7 +212,7 @@ def compose(self) -> ComposeResult: yield Label("â­• Legend", id="transfer_legend") def on_mount(self) -> None: - """PLACEHOLDER.""" + """Update the widgets immediately after mounting.""" for id in [ "#transfer_directorytree", "#transfer_switch_container", @@ -242,7 +244,7 @@ def on_mount(self) -> None: ) def on_select_changed(self, event: Select.Changed) -> None: - """PLACEHOLDER.""" + """Handle a Select widget changed on the tab.""" if event.select.id == "transfer_tab_overwrite_select": assert event.select.value in ["Never", "Always", "If Source Newer"] format_select = event.select.value.lower().replace(" ", "_") @@ -252,7 +254,7 @@ def on_select_changed(self, event: Select.Changed) -> None: ) def on_checkbox_changed(self, event: Checkbox.Changed) -> None: - """PLACEHOLDER.""" + """Handle a Checkbox widget changed on the tab.""" if event.checkbox.id == "transfer_tab_dry_run_checkbox": self.interface.save_tui_settings( event.checkbox.value, @@ -263,8 +265,9 @@ def on_checkbox_changed(self, event: Checkbox.Changed) -> None: # ---------------------------------------------------------------------------------- def switch_transfer_widgets_display(self) -> None: - """Show or hide transfer parameters based on whether the transfer mode - currently selected in `transfer_radioset`. + """Show or hide transfer parameters based on whether the transfer mode. + + The transfer mode is selected by the radiobutton e.g. Custom. """ for widget in self.transfer_all_widgets: widget.display = self.query_one("#transfer_all_radiobutton").value @@ -280,15 +283,15 @@ def switch_transfer_widgets_display(self) -> None: ).value def on_radio_set_changed(self, event: RadioSet.Changed) -> None: - """Update the displayed transfer parameter widgets when the - `transfer_radioset` radiobuttons are changed. - """ + """Update the transfer parameter widgets when the `transfer_radioset` are changed.""" label = str(event.pressed.label) assert label in ["All", "Top Level", "Custom"], "Unexpected label." self.switch_transfer_widgets_display() def on_button_pressed(self, event: Button.Pressed) -> None: - """If the Transfer button is clicked, opens a modal dialog + """Handle a button press on a tab. + + If the Transfer button is clicked, opens a modal dialog to confirm that the user wishes to transfer their data (in the direction selected). If "Yes" is selected, `self.transfer_data` (see below) is run. @@ -322,7 +325,7 @@ def on_button_pressed(self, event: Button.Pressed) -> None: ) async def refresh_after_datatype_changed(self, ignore): - """PLACEHOLDER.""" + """Refresh Checkboxes after the shown datatypes have changed.""" await self.recompose() self.on_mount() self.query_one("#transfer_custom_radiobutton").value = True @@ -331,7 +334,7 @@ async def refresh_after_datatype_changed(self, ignore): def on_custom_directory_tree_directory_tree_special_key_press( self, event: CustomDirectoryTree.DirectoryTreeSpecialKeyPress ) -> None: - """PLACEHOLDER.""" + """Handle a key press on the CustomDirectoryTree.""" if event.key == "ctrl+r": self.reload_directorytree() @@ -345,13 +348,11 @@ def on_custom_directory_tree_directory_tree_special_key_press( self.reload_directorytree() def reload_directorytree(self) -> None: - """PLACEHOLDER.""" + """Refresh the CustomDirectoryTree.""" self.query_one("#transfer_directorytree").update_transfer_tree() def update_directorytree_root(self, new_root_path: Path) -> None: - """Automatically refreshes the tree through the - reactive variable `path`. - """ + """Automatically refresh the tree through the reactive variable `path`.""" self.query_one("#transfer_directorytree").path = new_root_path # Transfer @@ -359,7 +360,7 @@ def update_directorytree_root(self, new_root_path: Path) -> None: @work(exclusive=True, thread=True) def transfer_data(self) -> Worker[InterfaceOutput]: - """A threaded worker to transfer data. + """Transfer data in a threaded worker. This function transfers data based on the config provided by the radio buttons such as a) the data to be transferred (all / top-level-folders / custom) b) the From 3d9945d66248b770e89d327f765e22993e795ce9 Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Sun, 22 Jun 2025 15:49:53 +0100 Subject: [PATCH 59/70] Update new_project.py --- datashuttle/tui/screens/new_project.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/datashuttle/tui/screens/new_project.py b/datashuttle/tui/screens/new_project.py index 60c3148d6..38bb51a93 100644 --- a/datashuttle/tui/screens/new_project.py +++ b/datashuttle/tui/screens/new_project.py @@ -14,9 +14,7 @@ class NewProjectScreen(Screen): - """Screen for setting up a new datashuttle project, by - inputting the desired configs. This uses the - ConfigsContent window to display and set the configs. + """Screen for setting up a new datashuttle project. If "Main Manu" button is pressed, the callback function returns None, so the project screen is not switched to. @@ -25,7 +23,6 @@ class NewProjectScreen(Screen): project is in ConfigsContent. ConfigsContent calls the dismiss method of this class to return an initialised project to mainwindow. - See ConfigsContent.on_button_pressed() for more details Parameters ---------- @@ -37,13 +34,13 @@ class NewProjectScreen(Screen): TITLE = "Make New Project" def __init__(self, mainwindow: TuiApp) -> None: - """PLACEHOLDER.""" + """Initialise the NewProjectScreen.""" super(NewProjectScreen, self).__init__() self.mainwindow = mainwindow def compose(self) -> ComposeResult: - """PLACEHOLDER.""" + """Add widgets to the NewProjectScreen.""" yield Header() yield Button("Main Menu", id="all_main_menu_buttons") yield configs_content.ConfigsContent( @@ -51,6 +48,6 @@ def compose(self) -> ComposeResult: ) def on_button_pressed(self, event: Button.Pressed) -> None: - """PLACEHOLDER.""" + """Handle a button press on the NewProjectScreen.""" if event.button.id == "all_main_menu_buttons": self.dismiss(None) From fd90782c3c50eff5f0e061a83f1ae5ac3500251a Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Sun, 22 Jun 2025 15:51:20 +0100 Subject: [PATCH 60/70] update get_help.py --- datashuttle/tui/screens/get_help.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/datashuttle/tui/screens/get_help.py b/datashuttle/tui/screens/get_help.py index 76a7ff1c6..38e813a66 100644 --- a/datashuttle/tui/screens/get_help.py +++ b/datashuttle/tui/screens/get_help.py @@ -18,10 +18,10 @@ class GetHelpScreen(ModalScreen): - """PLACEHOLDER.""" + """A screen with helpful information.""" def __init__(self) -> None: - """PLACEHOLDER.""" + """Initialise the GetHelpScreen.""" super(GetHelpScreen, self).__init__() self.text = """ @@ -36,23 +36,23 @@ def __init__(self) -> None: """ def action_link_docs(self) -> None: - """PLACEHOLDER.""" + """Link to datashuttle documentation.""" webbrowser.open(links.get_docs_link()) def action_link_github(self) -> None: - """PLACEHOLDER.""" + """Link to datashuttle github.""" webbrowser.open(links.get_github_link()) def action_link_github_issues(self) -> None: - """PLACEHOLDER.""" + """Link to datashuttle github issues.""" webbrowser.open(links.get_link_github_issues()) def action_link_zulip(self): - """PLACEHOLDER.""" + """Link to datashuttle zulip.""" webbrowser.open(links.get_link_zulip()) def compose(self) -> ComposeResult: - """PLACEHOLDER.""" + """Add widgets to the GetHelpScreen.""" yield Container( Static(self.text, id="get_help_label"), Button("Main Menu", id="all_main_menu_buttons"), @@ -60,6 +60,6 @@ def compose(self) -> ComposeResult: ) def on_button_pressed(self, event: Button.Pressed) -> None: - """PLACEHOLDER.""" + """Handle a button press on the GetHelpScreen.""" if event.button.id == "all_main_menu_buttons": self.dismiss() From 8c59e912394ef26ffe00cb0fcddfc17e800b0a3e Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Sun, 22 Jun 2025 15:54:06 +0100 Subject: [PATCH 61/70] Update validate_at_path.py --- datashuttle/tui/screens/validate_at_path.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/datashuttle/tui/screens/validate_at_path.py b/datashuttle/tui/screens/validate_at_path.py index 18b9fcdea..2b88e087e 100644 --- a/datashuttle/tui/screens/validate_at_path.py +++ b/datashuttle/tui/screens/validate_at_path.py @@ -14,8 +14,8 @@ class ValidateScreen(Screen): - """Screen to hold the validation window for - validating an existing project at a given path. + """Screen to the validate project from path window. + All widgets are stored in `ValidateContent`, which is shared between here and the validation tab on the project manager. """ @@ -23,12 +23,13 @@ class ValidateScreen(Screen): TITLE = "Validate Project" def __init__(self, mainwindow: TuiApp) -> None: + """Initialise the ValidateScreen.""" super(ValidateScreen, self).__init__() self.mainwindow = mainwindow def compose(self) -> ComposeResult: - """PLACEHOLDER.""" + """Add widgets to the ValidateScreen.""" yield Header() yield Button("Main Menu", id="all_main_menu_buttons") yield validate_content.ValidateContent( @@ -36,6 +37,6 @@ def compose(self) -> ComposeResult: ) def on_button_pressed(self, event: Button.Pressed) -> None: - """PLACEHOLDER.""" + """Handle button press on the ValidateScreen.""" if event.button.id == "all_main_menu_buttons": self.dismiss(None) From 93a54085ef3db2342b376669927b22a13f572149 Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Sun, 22 Jun 2025 15:55:32 +0100 Subject: [PATCH 62/70] Update settings.py --- datashuttle/tui/screens/settings.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/datashuttle/tui/screens/settings.py b/datashuttle/tui/screens/settings.py index 2a60d13f2..c8b018bde 100644 --- a/datashuttle/tui/screens/settings.py +++ b/datashuttle/tui/screens/settings.py @@ -20,21 +20,21 @@ class SettingsScreen(ModalScreen): - """Screen accessible from the main window that contains - 'global' settings for the TUI. 'Global' settings are non-project - specific settings (e.g. dark mode) and are handled independently - of the main datashuttle API. + """Screen accessible that contains 'global' settings for the TUI. + + 'Global' settings are non-project specific settings (e.g. dark mode) + and are handled independently of the main datashuttle API. """ def __init__(self, mainwindow: TuiApp) -> None: - """PLACEHOLDER.""" + """Initialise the SettingsScreen.""" super(SettingsScreen, self).__init__() self.mainwindow = mainwindow self.global_settings = self.mainwindow.load_global_settings() def compose(self) -> ComposeResult: - """PLACEHOLDER.""" + """Add widgets to the SettingsScreen.""" dark_mode = self.global_settings["dark_mode"] yield Container( RadioSet( @@ -60,12 +60,12 @@ def compose(self) -> ComposeResult: ) def on_mount(self) -> None: - """PLACEHOLDER.""" + """Update widgets immediately after they have been mounted.""" id = "#show_transfer_tree_status_checkbox" self.query_one(id).tooltip = get_tooltip(id) def on_radio_set_changed(self, event: RadioSet.Changed) -> None: - """PLACEHOLDER.""" + """Handle a radio set widget changed on SettingsScreen.""" label = str(event.pressed.label) assert label in ["Light Mode", "Dark Mode"] @@ -76,11 +76,11 @@ def on_radio_set_changed(self, event: RadioSet.Changed) -> None: self.mainwindow.save_global_settings(self.global_settings) def on_checkbox_changed(self, event: Checkbox.Changed) -> None: - """PLACEHOLDER.""" + """Handle checkbox changed on SettingsScreen.""" self.global_settings["show_transfer_tree_status"] = event.value self.mainwindow.save_global_settings(self.global_settings) def on_button_pressed(self, event: Button.Pressed) -> None: - """PLACEHOLDER.""" + """Handle button pressed on SettingsScreen.""" if event.button.id == "all_main_menu_buttons": self.dismiss() From fd831eb91a9b197d78aa193864b8dfafcec003b6 Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Sun, 22 Jun 2025 15:57:04 +0100 Subject: [PATCH 63/70] Update project_selector.py --- datashuttle/tui/screens/project_selector.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/datashuttle/tui/screens/project_selector.py b/datashuttle/tui/screens/project_selector.py index 42f9126c6..a3fb00afc 100644 --- a/datashuttle/tui/screens/project_selector.py +++ b/datashuttle/tui/screens/project_selector.py @@ -18,8 +18,9 @@ class ProjectSelectorScreen(Screen): - """The project selection screen. Finds and displays DataShuttle - projects present on the local system. + """The project selection screen. + + Finds and displays DataShuttle projects present on the local system. `self.dismiss()` returns an initialised project if initialisation was successful. Otherwise, in case `Main Menu` button is pressed, @@ -36,7 +37,7 @@ class ProjectSelectorScreen(Screen): TITLE = "Select Project" def __init__(self, mainwindow: TuiApp) -> None: - """PLACEHOLDER.""" + """Initialise the ProjectSelectorScreen.""" super(ProjectSelectorScreen, self).__init__() self.project_names = [ @@ -45,7 +46,7 @@ def __init__(self, mainwindow: TuiApp) -> None: self.mainwindow = mainwindow def compose(self) -> ComposeResult: - """PLACEHOLDER.""" + """Add widgets to the ProjectSelectorScreen.""" yield Header(id="project_select_header") yield Button("Main Menu", id="all_main_menu_buttons") yield Container( @@ -54,7 +55,7 @@ def compose(self) -> ComposeResult: ) def on_button_pressed(self, event: Button.Pressed) -> None: - """PLACEHOLDER.""" + """Handle a button press on ProjectSelectorScreen.""" if event.button.id in self.project_names: project_name = event.button.id From 069e33ad7eb5f763463c128f2ad7a75f2d0557da Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Sun, 22 Jun 2025 16:00:15 +0100 Subject: [PATCH 64/70] Update project_manager.py --- datashuttle/tui/screens/project_manager.py | 35 ++++++++++++---------- 1 file changed, 20 insertions(+), 15 deletions(-) diff --git a/datashuttle/tui/screens/project_manager.py b/datashuttle/tui/screens/project_manager.py index dc4b0ef92..c5dfd23ae 100644 --- a/datashuttle/tui/screens/project_manager.py +++ b/datashuttle/tui/screens/project_manager.py @@ -23,9 +23,10 @@ class ProjectManagerScreen(Screen): - """Screen containing the Create, Transfer and Configs tabs. This is - the primary screen within which the user interacts with - a pre-configured project. + """Screen containing the Create, Transfer and Configs tabs. + + This is the primary screen within which the user interacts + with a pre-configured project. The 'Create' tab interacts with Datashuttle's `create_folders()` method to create new project folders. @@ -42,7 +43,7 @@ class ProjectManagerScreen(Screen): """ def __init__(self, mainwindow: TuiApp, interface: Interface, id) -> None: - """PLACEHOLDER.""" + """Initialise the ProjectManagerScreen.""" super(ProjectManagerScreen, self).__init__(id=id) self.mainwindow = mainwindow @@ -53,7 +54,7 @@ def __init__(self, mainwindow: TuiApp, interface: Interface, id) -> None: self.tabbed_content_mount_signal = True def compose(self) -> ComposeResult: - """PLACEHOLDER.""" + """Add widgets to the ProjectManagerScreen.""" yield Header() yield Button("Main Menu", id="all_main_menu_buttons") with TabbedContent( @@ -91,21 +92,21 @@ def compose(self) -> ComposeResult: ) def on_button_pressed(self, event: Button.Pressed) -> None: - """Dismisses the TabScreen (and returns to the main menu) once - the 'Main Menu' button is pressed. - """ + """Dismiss the TabScreen and return to the main menu.""" if event.button.id == "all_main_menu_buttons": self.dismiss() def on_tabbed_content_tab_activated( self, event: TabbedContent.TabActivated ) -> None: - """Refresh the directorytree for create or transfer tabs whenever - the tabbedcontent is switched to one of these tabs. + """Handle a tab switch. + + Refresh the DirectoryTree for create or transfer tabs whenever + the TabbedContent is switched to one of these tabs. This is also triggered on mount, leading to it being reloaded twice, leading to a strange flicker. Ideally no trigger - would be sent on mount. Therefore the ugly `tabbed_content_mount_signal` + would be sent on mount. Therefore, the ugly `tabbed_content_mount_signal` variable is introduced to track this. """ if self.tabbed_content_mount_signal: @@ -125,12 +126,14 @@ def on_tabbed_content_tab_activated( ).update_most_recent_label() def update_active_tab_tree(self): - """PLACEHOLDER.""" + """Reload the CustomDirectoryTree on the now-active tab.""" active_tab_id = self.query_one("#tabscreen_tabbed_content").active self.query_one(f"#{active_tab_id}").reload_directorytree() def on_configs_content_configs_saved(self) -> None: - """When configs are saved, we may switch between a 'full' project + """Handle saving of the configs tab. + + When configs are saved, we may switch between a 'full' project and a 'local only' project (no `central_path` or `connection_method` set). In such a case we need to refresh the ProjectManager screen to add / remove the transfer tab. @@ -170,7 +173,9 @@ def on_configs_content_configs_saved(self) -> None: ) def wrap_dismiss(self, _): - """Need to wrap dismiss as cannot include it directly - in push_screen callback, or even wrapped in lambda. + """Wrap the dismiss function for push screen callbacks. + + Need to wrap dismiss as cannot include it directly in + push_screen callback, or even wrapped in lambda. """ self.dismiss() From 10c331eda3bbf983764d86423cac3035d1fb537e Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Sun, 22 Jun 2025 16:06:16 +0100 Subject: [PATCH 65/70] Update setup_ssh.py --- datashuttle/tui/screens/setup_ssh.py | 35 ++++++++++++++++------------ 1 file changed, 20 insertions(+), 15 deletions(-) diff --git a/datashuttle/tui/screens/setup_ssh.py b/datashuttle/tui/screens/setup_ssh.py index 54c428ec1..e02915699 100644 --- a/datashuttle/tui/screens/setup_ssh.py +++ b/datashuttle/tui/screens/setup_ssh.py @@ -18,10 +18,10 @@ class SetupSshScreen(ModalScreen): - """Dialog window that handles the TUI equivalent of API's - setup_ssh_connection(). This asks to - confirm the central hostkey, and takes password to setup - SSH key pair. + """Dialog window that sets up an SSH connection. + + This asks to confirm the central hostkey, and takes password to setup + SSH key pair. Under the hood uses `project.setup_ssh_connection()`. This is the one instance in which it is not possible for the TUI to nearly wrap the API, because the logic flow is @@ -29,7 +29,7 @@ class SetupSshScreen(ModalScreen): """ def __init__(self, interface: Interface) -> None: - """PLACEHOLDER.""" + """Initialise the SetupSshScreen.""" super(SetupSshScreen, self).__init__() self.interface = interface @@ -39,7 +39,7 @@ def __init__(self, interface: Interface) -> None: self.key: paramiko.RSAKey def compose(self) -> ComposeResult: - """PLACEHOLDER.""" + """Add widgets to the SetupSshScreen.""" yield Container( Horizontal( Static( @@ -58,11 +58,13 @@ def compose(self) -> ComposeResult: ) def on_mount(self) -> None: - """PLACEHOLDER.""" + """Update widgets immediately after they are mounted.""" self.query_one("#setup_ssh_password_input").visible = False def on_button_pressed(self, event: Button.pressed) -> None: - """When each stage is successfully progressed by clicking the "ok" button, + """Handle button press on the SetupSshScreen. + + When each stage is successfully progressed by clicking the "ok" button, `self.stage` is iterated by 1. For saving and excepting hostkey, if there is a problem (error or user declines) the 'OK' button is frozen so it is not possible to proceed. For accepting password @@ -85,7 +87,8 @@ def on_button_pressed(self, event: Button.pressed) -> None: self.dismiss() def ask_user_to_accept_hostkeys(self) -> None: - """The central server is identified by a hostkey. + """Ask the user to accept the hostkey that identifies the central server. + Get this hostkey and present it to user, clicking 'OK' is they are happy. If there is an error, block process (because it most likely is necessary to edit the central host id) and @@ -116,9 +119,10 @@ def ask_user_to_accept_hostkeys(self) -> None: self.stage += 1 def save_hostkeys_and_prompt_password_input(self) -> None: - """Once the hostkey is accepted, get the user password - for the central server. When 'OK' is pressed we go - straight to 'use_password_to_setup_ssh_key_pairs'. + """Get the user password for the central server. + + When 'OK' is pressed we go straight to + 'use_password_to_setup_ssh_key_pairs'. """ success, output = self.interface.save_hostkey_locally(self.key) @@ -140,9 +144,10 @@ def save_hostkeys_and_prompt_password_input(self) -> None: self.stage += 1 def use_password_to_setup_ssh_key_pairs(self) -> None: - """Get the user password for the central server. If correct, - SSH key pair is setup and 'OK' button changed to 'Finish'. - Otherwise, continue allowing failed password attempts. + """Get the user password for the central server. + + If correct, SSH key pair is set up and 'OK' button changed + to 'Finish'. Otherwise, continue allowing failed password attempts. """ password = self.query_one("#setup_ssh_password_input").value From 07ce802521f742881c4d30605a66cd33888d32b3 Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Sun, 22 Jun 2025 16:28:04 +0100 Subject: [PATCH 66/70] Update modal_dialogs.py --- datashuttle/tui/screens/modal_dialogs.py | 105 ++++++++++++++++------- 1 file changed, 76 insertions(+), 29 deletions(-) diff --git a/datashuttle/tui/screens/modal_dialogs.py b/datashuttle/tui/screens/modal_dialogs.py index 176526719..f0ca7bf6d 100644 --- a/datashuttle/tui/screens/modal_dialogs.py +++ b/datashuttle/tui/screens/modal_dialogs.py @@ -47,14 +47,24 @@ class MessageBox(ModalScreen): """ def __init__(self, message: str, border_color: str) -> None: - """PLACEHOLDER.""" + """Initialise the MessageBox. + + Parameters + ---------- + message + Message to display on the MessageBox. + + border_color + Color of the MessageBox border (e.g. green if the message is positive). + + """ super(MessageBox, self).__init__() self.message = message self.border_color = border_color def compose(self) -> ComposeResult: - """PLACEHOLDER.""" + """Add widgets to the MessageBox.""" yield Container( Container( Static(self.message, id="messagebox_message_label"), @@ -65,7 +75,7 @@ def compose(self) -> ComposeResult: ) def on_mount(self) -> None: - """PLACEHOLDER.""" + """Update widgets immediately after mounting.""" if self.border_color == "red": color = "rgb(140, 12, 0)" elif self.border_color == "green": @@ -81,7 +91,7 @@ def on_mount(self) -> None: ) def on_button_pressed(self) -> None: - """PLACEHOLDER.""" + """Handle button press.""" self.dismiss(True) @@ -98,14 +108,24 @@ def __init__( message: str, transfer_func: Callable[[], Worker[InterfaceOutput]], ) -> None: - """PLACEHOLDER.""" + """Initialise the ConfirmAndAwaitTransferPopup. + + Parameters + ---------- + message + Message to display while running the transfer. + + transfer_func + Function to run in a worker that performs the transfer. + + """ super().__init__() self.transfer_func = transfer_func self.message = message def compose(self) -> ComposeResult: - """PLACEHOLDER.""" + """Add widgets to the ConfirmAndAwaitTransferPopup.""" yield Container( Label(self.message, id="confirm_message_label"), Horizontal( @@ -117,7 +137,7 @@ def compose(self) -> ComposeResult: ) def on_button_pressed(self, event: Button.Pressed) -> None: - """PLACEHOLDER.""" + """Handle button press on the ConfirmAndAwaitTransferPopup.""" if event.button.id == "confirm_ok_button": self.query_one("#confirm_button_container").remove() @@ -134,7 +154,7 @@ def on_button_pressed(self, event: Button.Pressed) -> None: self.dismiss() async def handle_transfer_and_update_ui_when_complete(self) -> None: - """Runs the data transfer worker and updates the UI on completion.""" + """Run the data transfer worker and updates the UI on completion.""" data_transfer_worker = self.transfer_func() await data_transfer_worker.wait() success, output = data_transfer_worker.result @@ -155,19 +175,23 @@ async def handle_transfer_and_update_ui_when_complete(self) -> None: class SearchingCentralForNextSubSesPopup(ModalScreen): - """A popup to show message and a loading indicator when awaiting search next sub/ses across - the folders present in both local and central machines. This search happens in a separate - thread so as to allow TUI to display the loading indicate without freezing. + """Show message and a loading indicator for suggesting the next subject or session. - Only displayed when the `include_central` flag is checked and the connection method is "ssh". + Used to await searching next sub/ses across including folders + present on the central machine. This search happens in a separate + thread to allow TUI to display the loading indicate without freezing. + + Only displayed when the `include_central` flag is checked and the + connection method is "ssh". """ def __init__(self, sub_or_ses: Prefix) -> None: + """Initialise SearchingCentralForNextSubSesPopup.""" super().__init__() self.message = f"Searching central for next {sub_or_ses}" def compose(self) -> ComposeResult: - """PLACEHOLDER.""" + """Add widgets to SearchingCentralForNextSubSesPopup.""" yield Container( Label(self.message, id="searching_message_label"), LoadingIndicator(id="searching_animated_indicator"), @@ -176,10 +200,10 @@ def compose(self) -> ComposeResult: class SelectDirectoryTreeScreen(ModalScreen): - """A modal screen that includes a DirectoryTree to browse - and select folders. If a folder is double-clicked, - the path to the folder is returned through 'dismiss' - callback mechanism. + """Screen that includes a DirectoryTree to browse and select folders. + + If a folder is double-clicked, the path to the folder is + returned through 'dismiss' callback mechanism. Parameters ---------- @@ -195,7 +219,17 @@ class SelectDirectoryTreeScreen(ModalScreen): def __init__( self, mainwindow: TuiApp, path_: Optional[Path] = None ) -> None: - """PLACEHOLDER.""" + """Initialise SelectDirectoryTreeScreen. + + Parameters + ---------- + mainwindow + The main TUI app. + + path_ + Root path for the DirectoryTree. + + """ super(SelectDirectoryTreeScreen, self).__init__() self.mainwindow = mainwindow @@ -206,7 +240,7 @@ def __init__( self.click_info = ClickInfo() def compose(self) -> ComposeResult: - """PLACEHOLDER.""" + """Add widgets to the SelectDirectoryTreeScreen.""" label_message = ( "Select (double click) a folder with the same name as the project.\n" "If the project folder does not exist, select the parent folder and it will be created." @@ -232,6 +266,7 @@ def compose(self) -> ComposeResult: @staticmethod def get_drives(): """Get drives available on the machine to switch between. + For Windows, use `psutil` to get the list of drives. Otherwise, assume root is "/" and take all folders from that level. """ @@ -252,9 +287,11 @@ def get_drives(): ] def get_selected_drive(self): - """Get the default drive which the select starts on. For windows, - use the .drive attribute but for macOS and Linux this is blank. - On these Os use the first folder (e.g. /Users) as the default drive. + """Get the default drive which the select starts on. + + For windows, use the .drive attribute but for macOS and Linux + this is blank. On these Os use the first folder (e.g. /Users) + as the default drive. """ if platform.system() == "Windows": selected_drive = f"{self.path_.drive}\\" @@ -263,7 +300,7 @@ def get_selected_drive(self): return selected_drive def on_select_changed(self, event: Select.Changed) -> None: - """Updates the directory tree when the drive is changed.""" + """Update the directory tree when the drive is changed.""" self.path_ = Path(event.value) self.query_one( "#select_directory_tree_directory_tree" @@ -273,30 +310,40 @@ def on_select_changed(self, event: Select.Changed) -> None: def on_directory_tree_directory_selected( self, event: DirectoryTree.DirectorySelected ) -> None: - """PLACEHOLDER.""" + """Handle a node on the DirectoryTree selected.""" if event.path.is_file(): return else: self.dismiss(event.path) def on_button_pressed(self, event: Button.Pressed) -> None: - """PLACEHOLDER.""" + """Handle cancel button pressed.""" if event.button.id == "cancel_button": self.dismiss(False) class RenameFileOrFolderScreen(ModalScreen): - """PLACEHOLDER.""" + """A screen to handle the renaming of a file or folder selected through the DirectoryTree.""" def __init__(self, mainwindow: TuiApp, path_: Path) -> None: - """PLACEHOLDER.""" + """Initialise RenameFileOrFolderScreen. + + Parameters + ---------- + mainwindow + The main TUI app. + + path_ + Path of the file or folder to rename. + + """ super(RenameFileOrFolderScreen, self).__init__() self.mainwindow = mainwindow self.path_ = path_ def compose(self) -> ComposeResult: - """PLACEHOLDER.""" + """Add widgets to the RenameFileOrFolderScreen.""" yield Container( Label("Input the new name:", id="rename_screen_label"), Input(value=self.path_.stem, id="rename_screen_input"), @@ -309,7 +356,7 @@ def compose(self) -> ComposeResult: ) def on_button_pressed(self, event: Button.Pressed) -> None: - """PLACEHOLDER.""" + """Handle button pressed on the RenameFileOrFolderScreen.""" if event.button.id == "rename_screen_okay_button": self.dismiss(self.query_one("#rename_screen_input").value) From ffa4ef85b4dcbb182a70a32cb2befad841dfb681 Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Sun, 22 Jun 2025 16:33:01 +0100 Subject: [PATCH 67/70] Update datatypes.py --- datashuttle/tui/screens/datatypes.py | 67 +++++++++++++++++----------- 1 file changed, 40 insertions(+), 27 deletions(-) diff --git a/datashuttle/tui/screens/datatypes.py b/datashuttle/tui/screens/datatypes.py index b9f97f8a6..e8fb2d5d6 100644 --- a/datashuttle/tui/screens/datatypes.py +++ b/datashuttle/tui/screens/datatypes.py @@ -91,7 +91,17 @@ def __init__( create_or_transfer: Literal["create", "transfer"], interface: Interface, ) -> None: - """PLACEHOLDER.""" + """Initialise the DisplayedDatatypesScreen. + + Parameters + ---------- + create_or_transfer + Whether we are on the "create" or "transfer" tab. + + interface + Datashuttle Interface object. + + """ super(DisplayedDatatypesScreen, self).__init__() self.interface = interface @@ -104,9 +114,7 @@ def __init__( ) def compose(self) -> ComposeResult: - """Collect the datatypes names and status from - the persistent settings and display. - """ + """Collect the datatypes names and status from the persistent settings and display.""" selections = [] for idx, (datatype, setting) in enumerate( self.datatype_config.items() @@ -140,15 +148,10 @@ def compose(self) -> ComposeResult: id="display_datatypes_screen_container", ) - def on_mount(self): - """PLACEHOLDER.""" - pass - - # self.query_one("#display_datatypes_screen_container").action_scroll_up() - # assert False, f"{dir(self.query_one('#displayed_datatypes_selection_list'))}" - def on_button_pressed(self, event): - """When 'Save' is pressed, the configs copied on this class + """Handle button press on the DisplayedDatatypesScreen. + + When 'Save' is pressed, the configs copied on this class are updated back onto the interface configs, and written to disk. Otherwise, close the screen without saving. """ @@ -164,9 +167,7 @@ def on_button_pressed(self, event): def on_selection_list_selection_toggled( self, event: SelectionList.SelectionMessage.SelectionToggled ): - """When a selection is toggled, update the configs with - the 'displayed' status and save to disk. - """ + """Update the configs with the 'displayed' status and save to disk when Select is changed.""" datatype_name = event.selection.prompt.plain datatype_name = datatype_name.split(" ")[0] is_checked = not event.selection.initial_state @@ -182,8 +183,7 @@ def on_selection_list_selection_toggled( class DatatypeCheckboxes(Static): - """Dynamically-populated checkbox widget for convenient datatype - selection during folder creation. + """Dynamically-populated checkbox widget for convenient datatype selection. Parameters ---------- @@ -195,7 +195,7 @@ class DatatypeCheckboxes(Static): Attributes ---------- datatype_config - a Dictionary containing datatype as key (e.g. "ephys", "behav") + A Dictionary containing datatype as key (e.g. "ephys", "behav") and values are `bool` indicating whether the checkbox is on / off. If 'transfer', then transfer datatype arguments (e.g. "all") are also included. This structure mirrors @@ -216,7 +216,20 @@ def __init__( create_or_transfer: Literal["create", "transfer"] = "create", id: Optional[str] = None, ) -> None: - """PLACEHOLDER.""" + """Initialise the DatatypeCheckboxes. + + Parameters + ---------- + interface + Datashuttle Interface object. + + create_or_transfer + Whether we are on the "create" or "transfer" tab. + + id + Textual ID for the DatatypeCheckboxes widget. + + """ super(DatatypeCheckboxes, self).__init__(id=id) self.interface = interface @@ -231,7 +244,7 @@ def __init__( ] def compose(self) -> ComposeResult: - """PLACEHOLDER.""" + """Add widgets to the DatatypeCheckboxes.""" for datatype, setting in self.datatype_config.items(): if setting["displayed"]: yield Checkbox( @@ -242,7 +255,9 @@ def compose(self) -> ComposeResult: @on(Checkbox.Changed) def on_checkbox_changed(self) -> None: - """When a checkbox is changed, update the `self.datatype_config` + """Handle a datatype checkbox change. + + When a checkbox is changed, update the `self.datatype_config` to contain new boolean values for each datatype. Also update the stored `persistent_settings`. """ @@ -257,7 +272,7 @@ def on_checkbox_changed(self) -> None: ) def on_mount(self) -> None: - """PLACEHOLDER.""" + """Add widgets to the DatatypeCheckboxes.""" for datatype in self.datatype_config.keys(): if self.datatype_config[datatype]["displayed"]: self.query_one( @@ -265,9 +280,7 @@ def on_mount(self) -> None: ).tooltip = tooltips[datatype] def selected_datatypes(self) -> List[str]: - """Get the names of the datatype options for which the - checkboxes are switched on. - """ + """Get the names of the datatype options for which the checkboxes are switched on.""" selected_datatypes = [ datatype for datatype, settings in self.datatype_config.items() @@ -283,14 +296,14 @@ def selected_datatypes(self) -> List[str]: def get_checkbox_name( create_or_transfer: Literal["create", "transfer"], datatype ): - """PLACEHOLDER.""" + """Return a canonical formatted checkbox name.""" return f"{create_or_transfer}_{datatype}_checkbox" def get_tui_settings_key_name( create_or_transfer: Literal["create", "transfer"], ) -> str: - """PLACEHOLDER.""" + """Return the canonical tui settings key.""" if create_or_transfer == "create": settings_key = "create_checkboxes_on" else: From 520c1ceea69f1f7ee6b974d618098b16b1980f68 Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Sun, 22 Jun 2025 17:26:49 +0100 Subject: [PATCH 68/70] Update create_folder_settings.py --- .../tui/screens/create_folder_settings.py | 42 ++++++++++++------- 1 file changed, 28 insertions(+), 14 deletions(-) diff --git a/datashuttle/tui/screens/create_folder_settings.py b/datashuttle/tui/screens/create_folder_settings.py index ca9cec099..93d083a15 100644 --- a/datashuttle/tui/screens/create_folder_settings.py +++ b/datashuttle/tui/screens/create_folder_settings.py @@ -27,7 +27,9 @@ class CreateFoldersSettingsScreen(ModalScreen): - """Handles setting datashuttle's `name_template`'s, as well + """Settings for the Create Folders tab. + + Handles setting datashuttle's `name_template`, as well as the top-level-folder select and option to bypass all validation. Name Templates @@ -53,7 +55,14 @@ class CreateFoldersSettingsScreen(ModalScreen): TITLE = "Create Folders Settings" def __init__(self, mainwindow: TuiApp, interface: Interface) -> None: - """PLACEHOLDER.""" + """Initialise the CreateFoldersSettingsScreen. + + mainwindow + The main TUI app. + + interface + Datashuttle Interface object. + """ super(CreateFoldersSettingsScreen, self).__init__() self.mainwindow = mainwindow @@ -68,11 +77,11 @@ def __init__(self, mainwindow: TuiApp, interface: Interface) -> None: } def action_link_docs(self) -> None: - """PLACEHOLDER.""" + """Link to datashuttle documentation.""" webbrowser.open(links.get_docs_link()) def compose(self) -> ComposeResult: - """PLACEHOLDER.""" + """Add widgets to the CreateFoldersSettingsScreen.""" sub_on = True if self.input_mode == "sub" else False ses_on = not sub_on @@ -154,7 +163,7 @@ def compose(self) -> ComposeResult: ) def on_mount(self) -> None: - """PLACEHOLDER.""" + """Update widgets immediately after mounting.""" for id in [ "#create_folders_settings_toplevel_select", "#create_folders_settings_bypass_validation_checkbox", @@ -168,22 +177,23 @@ def on_mount(self) -> None: self.switch_template_container_disabled() def init_input_values_holding_variable(self) -> None: - """PLACEHOLDER.""" + """Add the project Name Templates to the relevant Inputs.""" name_templates = self.interface.get_name_templates() self.input_values["sub"] = name_templates["sub"] self.input_values["ses"] = name_templates["ses"] def switch_template_container_disabled(self) -> None: - """PLACEHOLDER.""" + """Switch the name template widgets disabled / enabled.""" is_on = self.query_one( "#template_settings_validation_on_checkbox" ).value self.query_one("#template_inner_container").disabled = not is_on def fill_input_from_template(self) -> None: - """Fill the `name_templates` Input, that is shared - between subject and session, depending on the - current radioset value. + """Fill the `name_templates` Input. + + The Input is shared between subject and session, + depending on the current RadioSet value. """ input = self.query_one("#template_settings_input") value = self.input_values[self.input_mode] @@ -195,7 +205,9 @@ def fill_input_from_template(self) -> None: input.value = value def on_button_pressed(self, event: Button.Pressed) -> None: - """On close, update the `name_templates` stored in + """Handle button press on the screen. + + On close, update the `name_templates` stored in `persistent_settings` with those set on the TUI. Setting may error if templates are turned on but @@ -214,7 +226,7 @@ def on_button_pressed(self, event: Button.Pressed) -> None: self.interface.save_tui_settings(False, "bypass_validation") def make_name_templates_from_widgets(self) -> Dict: - """PLACEHOLDER.""" + """Create a canonical `name_templates` entry based on the current widget settings.""" return { "on": self.query_one( "#template_settings_validation_on_checkbox" @@ -256,7 +268,9 @@ def on_checkbox_changed(self, event: Checkbox.Changed) -> None: ) def on_radio_set_changed(self, event: RadioSet.Changed) -> None: - """Update the displayed SSH widgets when the `connection_method` + """Update the displayed SSH widgets. + + These are updated when the `connection_method` radiobuttons are changed. """ label = str(event.pressed.label) @@ -266,7 +280,7 @@ def on_radio_set_changed(self, event: RadioSet.Changed) -> None: self.fill_input_from_template() def on_input_changed(self, message: Input.Changed) -> None: - """PLACEHOLDER.""" + """Store the current Input value in the attribute to be saved later.""" if message.input.id == "template_settings_input": val = None if message.value == "" else message.value self.input_values[self.input_mode] = val From 16c9509de69c4801643bbf9e19f19268e84c6bcc Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Sun, 22 Jun 2025 22:25:58 +0100 Subject: [PATCH 69/70] Add Returns sections. --- datashuttle/configs/canonical_configs.py | 8 +- datashuttle/configs/config_class.py | 18 +- datashuttle/configs/load_configs.py | 4 + datashuttle/datashuttle_class.py | 45 ++-- datashuttle/datashuttle_functions.py | 9 + datashuttle/tui/app.py | 21 +- datashuttle/tui/custom_widgets.py | 8 +- datashuttle/tui/interface.py | 24 +- .../tui/screens/create_folder_settings.py | 5 +- datashuttle/tui/screens/datatypes.py | 5 +- datashuttle/tui/screens/get_help.py | 2 +- datashuttle/tui/screens/modal_dialogs.py | 12 +- datashuttle/tui/screens/project_manager.py | 4 +- datashuttle/tui/tabs/create_folders.py | 46 +++- datashuttle/tui/tabs/logging.py | 19 +- datashuttle/tui/tabs/transfer.py | 6 + datashuttle/tui/tooltips.py | 9 +- datashuttle/utils/data_transfer.py | 11 +- datashuttle/utils/ds_logger.py | 15 +- datashuttle/utils/folders.py | 71 +++++- datashuttle/utils/formatting.py | 41 +++- datashuttle/utils/getters.py | 21 +- datashuttle/utils/rclone.py | 45 +++- datashuttle/utils/ssh.py | 43 +++- datashuttle/utils/utils.py | 27 ++- datashuttle/utils/validation.py | 228 +++++++++++++++++- pyproject.toml | 25 +- 27 files changed, 631 insertions(+), 141 deletions(-) diff --git a/datashuttle/configs/canonical_configs.py b/datashuttle/configs/canonical_configs.py index bdd0ebeb9..381d04f73 100644 --- a/datashuttle/configs/canonical_configs.py +++ b/datashuttle/configs/canonical_configs.py @@ -29,7 +29,7 @@ def get_canonical_configs() -> dict: - """Define the only permitted types for DataShuttle config values.""" + """Return the only permitted types for DataShuttle config values.""" canonical_configs = { "local_path": Union[str, Path], "central_path": Optional[Union[str, Path]], @@ -42,7 +42,7 @@ def get_canonical_configs() -> dict: def keys_str_on_file_but_path_in_class() -> list[str]: - """List all config keys that are paths but stored as str in the file. + """Return a list of all config keys that are paths but stored as str in the file. These are converted to pathlib.Path objects when loaded. """ @@ -256,7 +256,7 @@ def get_tui_config_defaults() -> Dict: def get_name_templates_defaults() -> Dict: - """Get the default values for name_templates.""" + """Return the default values for name_templates.""" return {"name_templates": {"on": False, "sub": None, "ses": None}} @@ -275,7 +275,7 @@ def get_persistent_settings_defaults() -> Dict: def get_datatypes() -> List[str]: - """Canonical list of datatype flags based on NeuroBlueprint. + """Return canonical list of datatype flags based on NeuroBlueprint. This must be kept up to date with the datatypes in the NeuroBlueprint specification. """ diff --git a/datashuttle/configs/config_class.py b/datashuttle/configs/config_class.py index 2f05c2e7d..4d216576b 100644 --- a/datashuttle/configs/config_class.py +++ b/datashuttle/configs/config_class.py @@ -71,7 +71,7 @@ def setup_after_load(self) -> None: self.ensure_local_and_central_path_end_in_project_name() self.check_dict_values_raise_on_fail() - def ensure_local_and_central_path_end_in_project_name(self): + def ensure_local_and_central_path_end_in_project_name(self) -> None: """Ensure that the local and central path end in the name of the project.""" for path_type in ["local_path", "central_path"]: if path_type == "central_path" and self[path_type] is None: @@ -94,15 +94,15 @@ def check_dict_values_raise_on_fail(self) -> None: canonical_configs.check_dict_values_raise_on_fail(self) def keys(self) -> KeysView: - """D.keys() -> a set-like object providing a view on D's keys.""" + """Return D.keys(), a set-like object providing a view on D's keys.""" return self.data.keys() def items(self) -> ItemsView: - """D.items() -> a set-like object providing a view on D's items.""" + """Return D.items(), a set-like object providing a view on D's items.""" return self.data.items() def values(self) -> ValuesView: - """D.values() -> a set-like object providing a view on D's values.""" + """Return D.values(), a set-like object providing a view on D's values.""" return self.data.values() # ------------------------------------------------------------------------- @@ -158,6 +158,10 @@ def build_project_path( top_level_folder either "rawdata" or "derivatives" + Returns + ------- + The full path to the `sub_folders` in the project. + """ if isinstance(sub_folders, list): sub_folders_str = "/".join(sub_folders) @@ -190,6 +194,10 @@ def get_base_folder( top_level_folder Either "rawdata" or "derivatives". + Returns + ------- + Full path to the local or central project top level folder. + """ if base == "local": base_folder = self["local_path"] / top_level_folder @@ -286,7 +294,7 @@ def get_datatype_as_dict_items( return items def is_local_project(self): - """Check if project is a local-only project. + """Return bool indicating if project is a local-only project. A project is 'local-only' if it has no `central_path` and `connection_method`. It can be used to make folders and validate, but not for transfer. diff --git a/datashuttle/configs/load_configs.py b/datashuttle/configs/load_configs.py index 3b1c00d1c..65b46ba00 100644 --- a/datashuttle/configs/load_configs.py +++ b/datashuttle/configs/load_configs.py @@ -33,6 +33,10 @@ def attempt_load_configs( verbose If True, warnings and error messages will be printed. + Returns + ------- + The loaded config, or `None` if it could not be loaded. + """ exists = config_path.is_file() diff --git a/datashuttle/datashuttle_class.py b/datashuttle/datashuttle_class.py index 23e7b42aa..1b7f4e268 100644 --- a/datashuttle/datashuttle_class.py +++ b/datashuttle/datashuttle_class.py @@ -435,7 +435,7 @@ def upload_rawdata( self, overwrite_existing_files: OverwriteExistingFiles = "never", dry_run: bool = False, - ): + ) -> None: """Upload all files in the `rawdata` top level folder. Parameters @@ -465,7 +465,7 @@ def upload_derivatives( self, overwrite_existing_files: OverwriteExistingFiles = "never", dry_run: bool = False, - ): + ) -> None: """Upload all files in the `derivatives` top level folder. Parameters @@ -495,7 +495,7 @@ def download_rawdata( self, overwrite_existing_files: OverwriteExistingFiles = "never", dry_run: bool = False, - ): + ) -> None: """Download all files in the `rawdata` top level folder. Parameters @@ -525,7 +525,7 @@ def download_derivatives( self, overwrite_existing_files: OverwriteExistingFiles = "never", dry_run: bool = False, - ): + ) -> None: """Download all files in the `derivatives` top level folder. Parameters @@ -699,7 +699,7 @@ def _transfer_top_level_folder( overwrite_existing_files: OverwriteExistingFiles = "never", dry_run: bool = False, init_log: bool = True, - ): + ) -> None: """Upload or download files within a particular top-level-folder. A centralised function to upload or download data within @@ -731,7 +731,7 @@ def _transfer_top_level_folder( def _transfer_specific_file_or_folder( self, upload_or_download, filepath, overwrite_existing_files, dry_run - ): + ) -> None: """Core function for upload/download_specific_folder_or_file().""" if isinstance(filepath, str): filepath = Path(filepath) @@ -952,13 +952,13 @@ def update_config_file(self, **kwargs) -> None: @check_configs_set def get_local_path(self) -> Path: - """Get the projects local path.""" + """Return the projects local path.""" return self.cfg["local_path"] @check_configs_set @check_is_not_local_project def get_central_path(self) -> Path: - """Get the project central path.""" + """Return the project central path.""" return self.cfg["central_path"] def get_datashuttle_path(self) -> Path: @@ -970,22 +970,22 @@ def get_datashuttle_path(self) -> Path: @check_configs_set def get_config_path(self) -> Path: - """Get the full path to the DataShuttle config file.""" + """Return the full path to the DataShuttle config file.""" return self._config_path @check_configs_set def get_configs(self) -> Configs: - """Get the datashuttle configs.""" + """Return the datashuttle configs.""" return self.cfg @check_configs_set def get_logging_path(self) -> Path: - """Get the path where datashuttle logs are written.""" + """Return the path where datashuttle logs are written.""" return self.cfg.logging_path @staticmethod def get_existing_projects() -> List[Path]: - """Get a list of existing project names found on the local machine. + """Return a list of existing project names found on the local machine. This is based on project folders in the "home / .datashuttle" folder that contain valid config.yaml files. @@ -1014,6 +1014,10 @@ def get_next_sub( `local_path` and `central_path`. If in local-project mode, this flag is ignored. + Returns + ------- + The next subject ID. + """ name_template = self.get_name_templates() name_template_regexp = ( @@ -1059,6 +1063,10 @@ def get_next_ses( ``local_path`` and ``central_path``. If in local-project mode, this flag is ignored. + Returns + ------- + The next session ID. + """ name_template = self.get_name_templates() name_template_regexp = ( @@ -1170,6 +1178,11 @@ def validate_project( any folder not prefixed with sub-, ses- or a valid datatype will raise a validation issue. + Returns + ------- + error_messages + A list of validation errors found in the project. + """ if include_central and strict_mode: raise ValueError( @@ -1366,7 +1379,7 @@ def _log_successful_config_change(self, message: bool = False) -> None: ) def _get_json_dumps_config(self) -> str: - """Get the config dictionary formatted as json.dumps() which allows well formatted printing.""" + """Return the config dictionary formatted as json.dumps() which allows well formatted printing.""" copy_dict = copy.deepcopy(self.cfg.data) load_configs.convert_str_and_pathlib_paths(copy_dict, "path_to_str") return json.dumps(copy_dict, indent=4) @@ -1434,7 +1447,7 @@ def _save_persistent_settings(self, settings: Dict) -> None: yaml.dump(settings, settings_file, sort_keys=False) def _load_persistent_settings(self) -> Dict: - """Load settings that are stored persistently across datashuttle sessions.""" + """Return settings that are stored persistently across datashuttle sessions.""" if not self._persistent_settings_path.is_file(): self._init_persistent_settings() @@ -1445,7 +1458,7 @@ def _load_persistent_settings(self) -> Dict: return settings - def _update_settings_with_new_canonical_keys(self, settings: Dict): + def _update_settings_with_new_canonical_keys(self, settings: Dict) -> None: """Check and update keys within persistent settings if missing. Perform a check on the keys within persistent settings. @@ -1478,7 +1491,7 @@ def _update_settings_with_new_canonical_keys(self, settings: Dict): settings ) - def _check_top_level_folder(self, top_level_folder): + def _check_top_level_folder(self, top_level_folder) -> None: """Raise an error if ``top_level_folder`` not correct.""" canonical_top_level_folders = canonical_folders.get_top_level_folders() diff --git a/datashuttle/datashuttle_functions.py b/datashuttle/datashuttle_functions.py index eb3522a77..5f24b9301 100644 --- a/datashuttle/datashuttle_functions.py +++ b/datashuttle/datashuttle_functions.py @@ -58,6 +58,11 @@ def quick_validate_project( to validate against. See ``DataShuttle.set_name_templates()`` for details. + Returns + ------- + error_messages + A list of validation errors found in the project. + """ project_path = Path(project_path) @@ -109,6 +114,10 @@ def _format_top_level_folder( top_level_folder The top-level folder to format. Can be "rawdata", "derivatives", or None. + Returns + ------- + A list of top-level folder names (e.g. ["rawdata"]). + """ rawdata_and_derivatives: List[TopLevelFolder] = ["rawdata", "derivatives"] diff --git a/datashuttle/tui/app.py b/datashuttle/tui/app.py index 85fb56676..4507b719f 100644 --- a/datashuttle/tui/app.py +++ b/datashuttle/tui/app.py @@ -175,10 +175,15 @@ def handle_open_filesystem_browser(self, path_: Path) -> None: self.show_modal_error_dialog(message) - def prompt_rename_file_or_folder(self, path_): + def prompt_rename_file_or_folder(self, path_) -> None: """Display pop-up window to rename a file or folder in a tab DirectoryTree. - Todo: + Parameters + ---------- + path_ + Path to the file or folder to rename. + + TODO ---- Can this not be moved to the relevant tab page? @@ -188,7 +193,7 @@ def prompt_rename_file_or_folder(self, path_): lambda new_name: self.rename_file_or_folder(path_, new_name), ) - def rename_file_or_folder(self, path_, new_name): + def rename_file_or_folder(self, path_, new_name) -> None: """Rename a file or folder within the project. Parameters @@ -228,7 +233,7 @@ def rename_file_or_folder(self, path_, new_name): # Global Settings --------------------------------------------------------- def load_global_settings(self) -> Dict: - """Load the 'global settings' for the TUI. + """Return the loaded 'global settings' for the TUI. These settings determine project-independent settings that are persistent across sessions. These are stored @@ -267,10 +272,16 @@ def save_global_settings(self, global_settings: Dict) -> None: with open(settings_path, "w") as file: yaml.dump(global_settings, file, sort_keys=False) - def copy_to_clipboard(self, value): + def copy_to_clipboard(self, value) -> None: """Centralized function to copy to clipboard. This may fail in headless mode on an HPC. + + Parameters + ---------- + value + Value to copy to clipboard. + """ try: pyperclip.copy(value) diff --git a/datashuttle/tui/custom_widgets.py b/datashuttle/tui/custom_widgets.py index aa4c8a493..5ab548fca 100644 --- a/datashuttle/tui/custom_widgets.py +++ b/datashuttle/tui/custom_widgets.py @@ -154,6 +154,10 @@ def filter_paths(self, paths: Iterable[Path]) -> Iterable[Path]: paths Paths to be filtered before being added to the CustomDirectoryTree. + Returns + ------- + A list of filtered paths + """ return [path for path in paths if not path.name.startswith(".")] @@ -513,7 +517,7 @@ def __init__(self, interface: Interface, id: str) -> None: ) def get_top_level_folder(self, init: bool = False) -> str: - """Get the top level folder from `persistent_settings`. + """Return the top level folder from `persistent_settings`. Performs a confidence-check that it matches the textual display. """ @@ -529,7 +533,7 @@ def get_top_level_folder(self, init: bool = False) -> str: return top_level_folder def get_displayed_top_level_folder(self) -> str: - """Get the top level folder that is currently selected on the select widget.""" + """Return the top level folder that is currently selected on the select widget.""" assert self.value in canonical_folders.get_top_level_folders() return self.value diff --git a/datashuttle/tui/interface.py b/datashuttle/tui/interface.py index 3d9cfafc8..cc7c91ef9 100644 --- a/datashuttle/tui/interface.py +++ b/datashuttle/tui/interface.py @@ -198,6 +198,14 @@ def validate_project( strict_mode If `True`, validation will be run in strict mode. + Returns + ------- + success + A bool inicating whether the validation was run successfully + + issues + A message or list of discovered validation issues. + """ try: results = self.project.validate_project( @@ -340,7 +348,7 @@ def transfer_custom_selection( # ---------------------------------------------------------------------------------- def get_name_templates(self) -> Dict: - """Get the `name_templates` defining templates to validate against. + """Return the `name_templates` defining templates to validate against. These are stored in a variable to avoid constantly reading these values from disk where they are stored in @@ -367,7 +375,7 @@ def set_name_templates(self, name_templates: Dict) -> InterfaceOutput: return False, str(e) def get_tui_settings(self) -> Dict: - """Get the "tui" field of `persistent_settings`. + """Return the "tui" field of `persistent_settings`. Similar to `get_name_templates`, there are held on the class to avoid constantly reading from disk. @@ -407,15 +415,15 @@ def save_tui_settings( # ---------------------------------------------------------------------------------- def get_central_host_id(self) -> str: - """Get the central host id for ssh.""" + """Return the central host id for ssh.""" return self.project.cfg["central_host_id"] def get_configs(self) -> Configs: - """Get Datashuttle Configs.""" + """Return Datashuttle Configs.""" return self.project.cfg def get_textual_compatible_project_configs(self) -> Configs: - """Datashuttle configs keeps paths saved as pathlib.Path objects. + """Return Datashuttle configs with paths stored as str. In some cases textual requires str representation. This method returns datashuttle configs with all paths that are Path @@ -428,7 +436,7 @@ def get_textual_compatible_project_configs(self) -> Configs: def get_next_sub( self, top_level_folder: TopLevelFolder, include_central: bool ) -> InterfaceOutput: - """Get the next subject ID in the project.""" + """Return the next subject ID in the project.""" try: next_sub = self.project.get_next_sub( top_level_folder, @@ -442,7 +450,7 @@ def get_next_sub( def get_next_ses( self, top_level_folder: TopLevelFolder, sub: str, include_central: bool ) -> InterfaceOutput: - """Get the next session ID for the `sub` in the project.""" + """Return the next session ID for the `sub` in the project.""" try: next_ses = self.project.get_next_ses( top_level_folder, @@ -455,7 +463,7 @@ def get_next_ses( return False, str(e) def get_ssh_hostkey(self) -> InterfaceOutput: - """Get the SSH remote server host key.""" + """Return the SSH remote server host key.""" try: key = ssh.get_remote_server_key( self.project.cfg["central_host_id"] diff --git a/datashuttle/tui/screens/create_folder_settings.py b/datashuttle/tui/screens/create_folder_settings.py index 93d083a15..5448d7ea5 100644 --- a/datashuttle/tui/screens/create_folder_settings.py +++ b/datashuttle/tui/screens/create_folder_settings.py @@ -57,11 +57,14 @@ class CreateFoldersSettingsScreen(ModalScreen): def __init__(self, mainwindow: TuiApp, interface: Interface) -> None: """Initialise the CreateFoldersSettingsScreen. + Parameters + ---------- mainwindow The main TUI app. interface Datashuttle Interface object. + """ super(CreateFoldersSettingsScreen, self).__init__() @@ -226,7 +229,7 @@ def on_button_pressed(self, event: Button.Pressed) -> None: self.interface.save_tui_settings(False, "bypass_validation") def make_name_templates_from_widgets(self) -> Dict: - """Create a canonical `name_templates` entry based on the current widget settings.""" + """Return a canonical `name_templates` entry based on the current widget settings.""" return { "on": self.query_one( "#template_settings_validation_on_checkbox" diff --git a/datashuttle/tui/screens/datatypes.py b/datashuttle/tui/screens/datatypes.py index e8fb2d5d6..7c21f84df 100644 --- a/datashuttle/tui/screens/datatypes.py +++ b/datashuttle/tui/screens/datatypes.py @@ -280,7 +280,7 @@ def on_mount(self) -> None: ).tooltip = tooltips[datatype] def selected_datatypes(self) -> List[str]: - """Get the names of the datatype options for which the checkboxes are switched on.""" + """Return the names of the datatype options for which the checkboxes are switched on.""" selected_datatypes = [ datatype for datatype, settings in self.datatype_config.items() @@ -295,7 +295,7 @@ def selected_datatypes(self) -> List[str]: def get_checkbox_name( create_or_transfer: Literal["create", "transfer"], datatype -): +) -> str: """Return a canonical formatted checkbox name.""" return f"{create_or_transfer}_{datatype}_checkbox" @@ -308,4 +308,5 @@ def get_tui_settings_key_name( settings_key = "create_checkboxes_on" else: settings_key = "transfer_checkboxes_on" + return settings_key diff --git a/datashuttle/tui/screens/get_help.py b/datashuttle/tui/screens/get_help.py index 38e813a66..511f59958 100644 --- a/datashuttle/tui/screens/get_help.py +++ b/datashuttle/tui/screens/get_help.py @@ -47,7 +47,7 @@ def action_link_github_issues(self) -> None: """Link to datashuttle github issues.""" webbrowser.open(links.get_link_github_issues()) - def action_link_zulip(self): + def action_link_zulip(self) -> None: """Link to datashuttle zulip.""" webbrowser.open(links.get_link_zulip()) diff --git a/datashuttle/tui/screens/modal_dialogs.py b/datashuttle/tui/screens/modal_dialogs.py index f0ca7bf6d..c7ca57c3e 100644 --- a/datashuttle/tui/screens/modal_dialogs.py +++ b/datashuttle/tui/screens/modal_dialogs.py @@ -38,12 +38,15 @@ class MessageBox(ModalScreen): """A screen for rendering error messages. + Parameters + ---------- message The message to display in the message box border_color The color to pass to the `border` style on the widget. Note that the keywords 'red' 'grey' 'green' are overridden for custom style. + """ def __init__(self, message: str, border_color: str) -> None: @@ -264,8 +267,8 @@ def compose(self) -> ComposeResult: ) @staticmethod - def get_drives(): - """Get drives available on the machine to switch between. + def get_drives() -> list[str]: + """Return drives available on the machine to switch between. For Windows, use `psutil` to get the list of drives. Otherwise, assume root is "/" and take all folders from that level. @@ -286,8 +289,8 @@ def get_drives(): f"/{dir.name}" for dir in Path("/").iterdir() if dir.is_dir() ] - def get_selected_drive(self): - """Get the default drive which the select starts on. + def get_selected_drive(self) -> str: + """Return the default drive which the select starts on. For windows, use the .drive attribute but for macOS and Linux this is blank. On these Os use the first folder (e.g. /Users) @@ -297,6 +300,7 @@ def get_selected_drive(self): selected_drive = f"{self.path_.drive}\\" else: selected_drive = f"/{self.path_.parts[1]}" + return selected_drive def on_select_changed(self, event: Select.Changed) -> None: diff --git a/datashuttle/tui/screens/project_manager.py b/datashuttle/tui/screens/project_manager.py index c5dfd23ae..8de148106 100644 --- a/datashuttle/tui/screens/project_manager.py +++ b/datashuttle/tui/screens/project_manager.py @@ -125,7 +125,7 @@ def on_tabbed_content_tab_activated( "#tabscreen_logging_tab" ).update_most_recent_label() - def update_active_tab_tree(self): + def update_active_tab_tree(self) -> None: """Reload the CustomDirectoryTree on the now-active tab.""" active_tab_id = self.query_one("#tabscreen_tabbed_content").active self.query_one(f"#{active_tab_id}").reload_directorytree() @@ -172,7 +172,7 @@ def on_configs_content_configs_saved(self) -> None: self.wrap_dismiss, ) - def wrap_dismiss(self, _): + def wrap_dismiss(self, _) -> None: """Wrap the dismiss function for push screen callbacks. Need to wrap dismiss as cannot include it directly in diff --git a/datashuttle/tui/tabs/create_folders.py b/datashuttle/tui/tabs/create_folders.py index 8c49bc424..2ec76e728 100644 --- a/datashuttle/tui/tabs/create_folders.py +++ b/datashuttle/tui/tabs/create_folders.py @@ -156,7 +156,7 @@ def on_button_pressed(self, event: Button.Pressed) -> None: lambda unused_bool: self.revalidate_inputs(["sub", "ses"]), ) - async def refresh_after_datatypes_changed(self, ignore): + async def refresh_after_datatypes_changed(self, ignore) -> None: """Redisplay the datatype checkboxes.""" await self.recompose() self.on_mount() @@ -187,7 +187,7 @@ def on_clickable_input_clicked( def on_custom_directory_tree_directory_tree_special_key_press( self, event: CustomDirectoryTree.DirectoryTreeSpecialKeyPress - ): + ) -> None: """Handle a key press on the CustomDirectoryTree. This can refresh the CustomDirectoryTree or fill / append @@ -250,7 +250,7 @@ def revalidate_inputs(self, all_prefixes: List[str]) -> None: value = self.query_one(key).value self.query_one(key).validate(value=value) - def update_input_tooltip(self, message: List[str], prefix: Prefix) -> None: + def update_input_tooltip(self, message: str, prefix: Prefix) -> None: """Update the value of a subject or session tooltip indicating the validation status.""" id = ( "#create_folders_subject_input" @@ -258,7 +258,7 @@ def update_input_tooltip(self, message: List[str], prefix: Prefix) -> None: else "#create_folders_session_input" ) input = self.query_one(id) - input.tooltip = message if any(message) else None + input.tooltip = message # ---------------------------------------------------------------------------------- # Datashuttle Callers @@ -304,7 +304,7 @@ def reload_directorytree(self) -> None: def suggest_next_sub_ses( self, prefix: Prefix, input_id: str, include_central: bool - ): + ) -> None: """Suggests the next sub/ses name for the project. Shows a pop up screen in cases when searching for next sub/ses takes @@ -312,6 +312,18 @@ def suggest_next_sub_ses( Creates an asyncio task which handles the suggestion logic and dismissing the pop up. + + Parameters + ---------- + prefix + Suggest the next "sub" or "ses". + + input_id + Textual ID of the widget in which to fill the suggested sub or ses. + + include_central + If `True`, search central project as well to generate the suggestion. + """ assert self.interface.project.cfg["connection_method"] in [ None, @@ -337,7 +349,7 @@ def suggest_next_sub_ses( async def fill_suggestion_and_dismiss_popup( self, prefix, input_id, include_central - ): + ) -> None: """Run the `fill_input_with_next_sub_or_ses_template` worker. Awaits completion. If an error occurs in `fill_input_with_next_sub_or_ses_template`, @@ -345,6 +357,8 @@ async def fill_suggestion_and_dismiss_popup( Else, if the worker successfully exits, this function handles dismissal of the popup. + + see `suggest_next_sub_ses()` for parameters. """ worker = self.fill_input_with_next_sub_or_ses_template( prefix, input_id, include_central @@ -357,7 +371,7 @@ async def fill_suggestion_and_dismiss_popup( @work(exclusive=True, thread=True) def fill_input_with_next_sub_or_ses_template( self, prefix: Prefix, input_id: str, include_central: bool - ) -> Worker: + ) -> Worker[None]: """Fill an Input the next subject / session in the project (local). If `name_templates` are set, then the sub- or ses- first key @@ -379,6 +393,10 @@ def fill_input_with_next_sub_or_ses_template( include_central If `True`, the central project is also validated. + Returns + ------- + A textual Worker object for the thread in which the function is run. + """ top_level_folder = self.interface.tui_settings[ "top_level_folder_select" @@ -456,7 +474,7 @@ def dismiss_popup_and_show_modal_error_dialog_from_thread( # Validation # ---------------------------------------------------------------------------------- - def run_local_validation(self, prefix: Prefix): + def run_local_validation(self, prefix: Prefix) -> tuple[bool, str]: """Run validation of the values stored in the subject / session Inputs. First, format the subject name (and session if required) @@ -477,7 +495,15 @@ def run_local_validation(self, prefix: Prefix): Parameters ---------- prefix - Whether to run validation on the subject or session Input + Whether to run validation on the "sub" or "ses". + + Returns + ------- + bool + Indicate whether validation passed successfully. + If `False`, there was a validation error. + output + Str containing the validation error, or successfully formatted name. """ sub_names = self.query_one( @@ -506,5 +532,5 @@ def run_local_validation(self, prefix: Prefix): return True, f"Formatted names: {names}" def update_directorytree_root(self, new_root_path: Path) -> None: - """Will automatically refresh the tree through the reactive attribute `path`.""" + """Refresh the tree through the reactive attribute `path`.""" self.query_one("#create_folders_directorytree").path = new_root_path diff --git a/datashuttle/tui/tabs/logging.py b/datashuttle/tui/tabs/logging.py index cfe496cdd..1e3e3e475 100644 --- a/datashuttle/tui/tabs/logging.py +++ b/datashuttle/tui/tabs/logging.py @@ -6,6 +6,7 @@ if TYPE_CHECKING: from textual import events + from textual.app import ComposeResult from textual.widgets import DirectoryTree from datashuttle import DataShuttle @@ -34,19 +35,19 @@ def __init__(self, log_file): with open(log_file) as file: self.log_contents = "".join(file.readlines()) - def compose(self): + def compose(self) -> ComposeResult: """Set the widgets for the screen.""" yield Container( RichLog(highlight=True, markup=True, id="richlog_screen_rich_log"), Button("Close", id="richlog_screen_close_button"), ) - def on_mount(self): + def on_mount(self) -> None: """Update widgets immediately after mount.""" text_log = self.query_one(RichLog) text_log.write(self.log_contents) - def on_button_pressed(self, event): + def on_button_pressed(self, event: Button.Pressed) -> None: """Handle a button press on the screen.""" if event.button.id == "richlog_screen_close_button": self.dismiss() @@ -95,7 +96,7 @@ def update_latest_log_path(self): else Path("None found.") ) - def compose(self): + def compose(self) -> ComposeResult: """Set with widgets on the LoggingTab.""" yield Container( Label( @@ -132,7 +133,7 @@ def update_most_recent_label(self): ) self.refresh() - def on_button_pressed(self, event): + def on_button_pressed(self, event: Button.Pressed) -> None: """Handle button press on the tab.""" if event.button.id == "logging_tab_open_most_recent_button": self.push_rich_log_screen(self.latest_log_path) @@ -140,7 +141,7 @@ def on_button_pressed(self, event): @require_double_click def on_directory_tree_file_selected( self, event: DirectoryTree.FileSelected - ): + ) -> None: """Handle a click on the DirectoryTree showing the log files.""" if not event.path.is_file(): self.mainwindow.show_modal_error_dialog( @@ -159,10 +160,12 @@ def push_rich_log_screen(self, log_path): ) ) - def reload_directorytree(self): + def reload_directorytree(self) -> None: """Refresh the DirectoryTree (e.g. if a new log file is saved).""" self.query_one("#logging_tab_custom_directory_tree").reload() - def on_custom_directory_tree_directory_tree_special_key_press(self): + def on_custom_directory_tree_directory_tree_special_key_press( + self, + ) -> None: """Handle the CTRL+R refresh of the directory tree.""" self.reload_directorytree() diff --git a/datashuttle/tui/tabs/transfer.py b/datashuttle/tui/tabs/transfer.py index 97e6631c9..63dd78112 100644 --- a/datashuttle/tui/tabs/transfer.py +++ b/datashuttle/tui/tabs/transfer.py @@ -369,6 +369,12 @@ def transfer_data(self) -> Worker[InterfaceOutput]: This function is passed to `ConfirmAndAwaitTransferPopup` which calls it to handle data transfer in a worker thread. The UI updates during and after transfer are handled by `ConfirmAndAwaitTransferPopup`. + + Returns + ------- + An InterfaceOutput object that indicates whether the transfer was a success. + Wrapped in an awaitable Textual Worker for the thread in the function is run. + """ upload = not self.query_one("#transfer_switch").value diff --git a/datashuttle/tui/tooltips.py b/datashuttle/tui/tooltips.py index 63c7767ec..f39bb9a4b 100644 --- a/datashuttle/tui/tooltips.py +++ b/datashuttle/tui/tooltips.py @@ -1,12 +1,5 @@ def get_tooltip(id: str) -> str: - """Return tooltip for a widget based on its textual id. - - Parameters - ---------- - id - Textual widget i.d. (e.g. "#configs_local_path_input"). - - """ + """Return tooltip for a widget based on its textual id.""" # Main App Window # ------------------------------------------------------------------------- diff --git a/datashuttle/utils/data_transfer.py b/datashuttle/utils/data_transfer.py index 5a855a50c..1db712859 100644 --- a/datashuttle/utils/data_transfer.py +++ b/datashuttle/utils/data_transfer.py @@ -192,7 +192,7 @@ def build_a_list_of_all_files_and_folders_to_transfer(self) -> List[str]: def make_include_arg( self, list_of_paths: List[str], recursive: bool = True ) -> List[str]: - """Format the list of paths to rclone's required `--include` flag format.""" + """Return the list of paths formatted to rclone's required `--include` flag format.""" if not any(list_of_paths): return [] @@ -351,7 +351,7 @@ def update_list_with_dtype_paths( # ------------------------------------------------------------------------- def to_list(self, names: Union[str, List[str]]) -> List[str]: - """Convert a name or list of names to a list.""" + """Return a name or list of names as a list.""" if isinstance(names, str): names = [names] return names @@ -436,6 +436,11 @@ def get_processed_names( see transfer_sub_ses_data() for list of parameters. + Returns + ------- + A list of folder names generated from the original + names list that included search wildcards, "all" keys etc. + """ prefix: Prefix if sub is None: @@ -475,7 +480,7 @@ def get_processed_names( return processed_names def transfer_non_datatype(self, datatype_checked: List[str]) -> bool: - """Return bool if all non-datatype folders are to be transferred.""" + """Return bool indicating whether all non-datatype folders are to be transferred.""" return any( [name in ["all_non_datatype", "all"] for name in datatype_checked] ) diff --git a/datashuttle/utils/ds_logger.py b/datashuttle/utils/ds_logger.py index f5dd08633..d1591038d 100644 --- a/datashuttle/utils/ds_logger.py +++ b/datashuttle/utils/ds_logger.py @@ -3,6 +3,7 @@ from typing import TYPE_CHECKING, Any, List, Optional if TYPE_CHECKING: + from logging import Logger from pathlib import Path from datashuttle.configs.configs import Configs @@ -17,18 +18,18 @@ from datashuttle.utils import utils -def get_logger_name(): +def get_logger_name() -> str: """Return the name of the logger.""" return "datashuttle" -def get_logger(): +def get_logger() -> Logger: """Return the instance of the logger object.""" return logging.getLogger(get_logger_name()) -def logging_is_active(): - """Check if the logger is active.""" +def logging_is_active() -> bool: + """Return a bool indicating if the logger is active.""" logger_exists = get_logger_name() in logging.root.manager.loggerDict if logger_exists and get_logger().handlers != []: return True @@ -120,6 +121,12 @@ def wrap_variables_for_fancylog(local_vars: dict, cfg: Configs) -> List: Delete the self attribute (which is DataShuttle class) to keep the logs neat, as it adds no information. + + Returns + ------- + A list holding a wrapper class that holds all variable + state for fancylog to log. + """ class VariablesState: diff --git a/datashuttle/utils/folders.py b/datashuttle/utils/folders.py index 74fe2575e..a8ff8827b 100644 --- a/datashuttle/utils/folders.py +++ b/datashuttle/utils/folders.py @@ -230,6 +230,33 @@ def search_project_for_sub_or_ses_names( as session folders for local subjects that are not yet on central will be searched for on central, showing a confusing 'folder not found' message. + + Parameters + ---------- + cfg + Datashuttle Configs object. + + top_level_folder + "rawdata" or "derivatives". + + sub + Subject name (if provided, search for a session within that sub) + + search_str + Glob-style search to perform e.g. "sub-*" + + include_central + If `True`, central project is also searched. + + return_full_path + If True, the full path to the discovered folders is provided. + Otherwise, just the name. + + Returns + ------- + A dictionary with "local" and "central" keys, where values + are the discovered folders. "central" is `None` if include_central is `False`. + """ # Search local and central for folders that begin with "sub-*" local_foldernames, _ = search_sub_or_ses_level( @@ -272,12 +299,19 @@ def items_from_datatype_input( sub: str, ses: Optional[str] = None, ) -> Union[ItemsView, zip]: - """Get the list of datatypes to transfer. + """Return the list of datatypes to transfer. Take these directly from user input, or by searching what is available if "all" is passed. see _transfer_datatype() for full parameters list. + + Returns + ------- + Datatypes as a dictionary items() or zip that mimics that structure. + The dictionary is in the form datatype name: Folder() struct. + See `canonical_folders.py`. + """ base_folder = cfg.get_base_folder(local_or_central, top_level_folder) @@ -386,10 +420,6 @@ def search_for_wildcards( if sub is None it is assumed all_names are sub names and the level above is searched. - Outputs a new list of names including all original names - but where @*@-containing names have been replaced with - search results. - Parameters ---------- cfg @@ -415,6 +445,13 @@ def search_for_wildcards( optional subject to search for sessions in. If not provided, will search for subjects rather than sessions. + Returns + ------- + new_all_names + A new list of names including all original names + but where @*@-containing names have been replaced with + search results. + """ new_all_names: List[str] = [] for name in all_names: @@ -496,6 +533,10 @@ def search_sub_or_ses_level( return_full_path include the search_path in the returned paths + Returns + ------- + Discovered folders (`all_folder_names`) and files (`all_filenames`). + """ if ses and not sub: utils.log_and_raise_error( @@ -552,6 +593,10 @@ def search_for_folders( return_full_path include the search_path in the returned paths + Returns + ------- + Discovered folders (`all_folder_names`) and files (`all_filenames`). + """ if local_or_central == "central" and cfg["connection_method"] == "ssh": all_folder_names, all_filenames = ssh.search_ssh_central_for_folders( @@ -579,10 +624,24 @@ def search_for_folders( def search_filesystem_path_for_folders( search_path_with_prefix: Path, return_full_path: bool = False ) -> Tuple[List[Path | str], List[Path | str]]: - """Search a folder through the local filesystem. + r"""Search a folder through the local filesystem. Use glob to search the full search path (including prefix) with glob. Files are filtered out of results, returning folders only. + + Parameters + ---------- + search_path_with_prefix + Path to search along with search prefix e.g. "C:\drive\project\sub-*" + + return_full_path + If `True` returns the path to the discovered folder or file, + otherwise just the name. + + Returns + ------- + Discovered folders (`all_folder_names`) and files (`all_filenames`). + """ all_folder_names = [] all_filenames = [] diff --git a/datashuttle/utils/formatting.py b/datashuttle/utils/formatting.py index f3504c26e..625174218 100644 --- a/datashuttle/utils/formatting.py +++ b/datashuttle/utils/formatting.py @@ -53,6 +53,10 @@ def check_and_format_names( If `True`, NeuroBlueprint validation will be performed on the passed names. + Returns + ------- + A list of formatted names. + """ if isinstance(names, str): names = [names] @@ -94,6 +98,10 @@ def format_names(names: List, prefix: Prefix) -> List[str]: prefix "sub" or "ses" - this defines the prefix checks. + Returns + ------- + A list of formatted names. + """ assert prefix in ["sub", "ses"], "`prefix` must be 'sub' or 'ses'." @@ -212,6 +220,12 @@ def make_list_of_zero_padded_names_across_range( rest of the name after the flag, i.e. all other key-value pairs. + Returns + ------- + A list of subject or session names expanded across a range. + e.g. sub-001@TO@002_date-20220101 becomes + ["sub-001_date-20220101", "sub-002_date-20220101"]. + """ max_leading_zeros = max( utils.num_leading_zeros(left_number), @@ -238,6 +252,7 @@ def make_list_of_zero_padded_names_across_range( def update_names_with_datetime(names: List[str]) -> None: """Replace @DATE@ and @DATETIME@ flag with date and datetime respectively. + `names` is a list of subject or session names. Format using key-value pair for bids, i.e. date-20221223_time- """ date = str(datetime.datetime.now().date().strftime("%Y%m%d")) @@ -258,8 +273,24 @@ def replace_date_time_tags_in_name( datetime_with_key: str, date_with_key: str, time_with_key: str, -): - """Replace tags with their final value for every name in a list.""" +) -> None: + """Replace tags with their final value for every name in a list. + + Parameters + ---------- + names + A list of subject or session names. + + datetime_with_key + Formatted datetime key-value pair .e.g datetime-20220101T010101. + + date_with_key + Formatted date key-value pair .e.g date-20220101. + + time_with_key + Formatted time key-value pair .e.g time-010101. + + """ for i, name in enumerate(names): # datetime conditional must come first. if tags("datetime") in name: @@ -278,17 +309,17 @@ def replace_date_time_tags_in_name( def format_date(date: str) -> str: - """Format the `date` as `date-`.""" + """Return the date formatted as `date-`.""" return f"date-{date}" def format_time(time_: str) -> str: - """Format the `time_` as `time-`.""" + """Return the time `time_` formatted as `time-`.""" return f"time-{time_}" def format_datetime(date: str, time_: str) -> str: - """Format the `date` and `time_` as `datetime-T`.""" + """Return the `date` and `time_` formatted as `datetime-T`.""" return f"datetime-{date}T{time_}" diff --git a/datashuttle/utils/getters.py b/datashuttle/utils/getters.py index 3af513b51..33ee68f86 100644 --- a/datashuttle/utils/getters.py +++ b/datashuttle/utils/getters.py @@ -216,7 +216,7 @@ def get_max_sub_or_ses_num_and_value_length( def get_num_value_digits_from_project( all_values_str: List[str], prefix: Prefix ) -> int: - """Find the number of digits for the sub or ses key within the project. + """Return the number of digits for the sub or ses key within the project. Parameters ---------- @@ -249,6 +249,20 @@ def get_num_value_digits_from_regexp( If there is length-unspecific wildcard (.*) in the sub key, then skip. In practice, there should never really be a .* in the sub or ses key of a name template, but handle it just in case. + + Parameters + ---------- + prefix + "sub" or "ses". + + name_template_regexp + Regexp for the name template to validate against. + + Returns + ------- + num_digits + Number of digits in the sub- or ses- value, or `False` if wildcard searching. + """ all_values_str = utils.get_values_from_bids_formatted_name( [name_template_regexp], prefix, return_as_int=False @@ -329,6 +343,11 @@ def get_all_sub_and_ses_paths( If `False, only get names from `local_path`, otherwise from `local_path` and `central_path`. + Returns + ------- + A dictionary with "sub" key (path to all subject folders) + and "ses" key (path to all session folders). + """ sub_folder_paths = folders.search_project_for_sub_or_ses_names( cfg, diff --git a/datashuttle/utils/rclone.py b/datashuttle/utils/rclone.py index 8958c4824..2d66a8cc1 100644 --- a/datashuttle/utils/rclone.py +++ b/datashuttle/utils/rclone.py @@ -22,6 +22,10 @@ def call_rclone(command: str, pipe_std: bool = False) -> CompletedProcess: pipe_std if True, do not output anything to stdout. + Returns + ------- + subprocess.CompletedProcess with `stdout` and `stderr` attributes. + """ command = "rclone " + command if pipe_std: @@ -39,6 +43,16 @@ def call_rclone_through_script(command: str) -> CompletedProcess: This is to avoid limits on command-line calls (in particular on Windows). Used for transfers due to generation of large call strings. + + Parameters + ---------- + command + Full command to run with RClone. + + Returns + ------- + subprocess.CompletedProcess with `stdout` and `stderr` attributes. + """ system = platform.system() @@ -81,7 +95,7 @@ def call_rclone_through_script(command: str) -> CompletedProcess: def setup_rclone_config_for_local_filesystem( rclone_config_name: str, log: bool = True, -): +) -> None: """Set the RClone remote config for local filesystem. RClone sets remote targets in a config file that are @@ -117,7 +131,7 @@ def setup_rclone_config_for_ssh( rclone_config_name: str, ssh_key_path: Path, log: bool = True, -): +) -> None: """Set the RClone remote config for ssh. RClone sets remote targets in a config file that are @@ -156,7 +170,7 @@ def setup_rclone_config_for_ssh( log_rclone_config_output() -def log_rclone_config_output(): +def log_rclone_config_output() -> None: """Log the output from creating Rclone config.""" output = call_rclone("config file", pipe_std=True) utils.log( @@ -165,7 +179,7 @@ def log_rclone_config_output(): def check_rclone_with_default_call() -> bool: - """Check to see whether rclone is installed.""" + """Return a bool indicating whether rclone is installed.""" try: output = call_rclone("-h", pipe_std=True) except FileNotFoundError: @@ -216,6 +230,10 @@ def transfer_data( A list of options to pass to Rclone's copy function. see `cfg.make_rclone_transfer_options()`. + Returns + ------- + subprocess.CompletedProcess with `stdout` and `stderr` attributes. + """ assert upload_or_download in [ "upload", @@ -356,7 +374,22 @@ def perform_rclone_check( def handle_rclone_arguments( rclone_options: Dict, include_list: List[str] ) -> str: - """Construct the extra arguments to pass to RClone.""" + """Construct the extra arguments to pass to RClone. + + Parameters + ---------- + rclone_options + A list of option keywords to be passed to + + include_list + The (already formatted) list of filepaths for the + rclone `--include` option. + + Returns + ------- + A full list of arguments to pass to rclone. + + """ extra_arguments_list = [] extra_arguments_list += ["-" + rclone_options["transfer_verbosity"]] @@ -386,7 +419,7 @@ def handle_rclone_arguments( def rclone_args(name: str) -> str: - """Central function to hold rclone commands.""" + """Return list of Rclone commands.""" valid_names = [ "dry_run", "copy", diff --git a/datashuttle/utils/ssh.py b/datashuttle/utils/ssh.py index bd418a62d..587e9416a 100644 --- a/datashuttle/utils/ssh.py +++ b/datashuttle/utils/ssh.py @@ -28,7 +28,7 @@ def connect_client_core( client: paramiko.SSHClient, cfg: Configs, password: Optional[str] = None, -): +) -> None: """Connect to the client. A centralised function to connect to a paramiko client. @@ -64,7 +64,20 @@ def connect_client_core( def add_public_key_to_central_authorized_keys( cfg: Configs, password: str, log=True ) -> None: - """Append the public part of key to central server ~/.ssh/authorized_keys.""" + """Append the public part of key to central server ~/.ssh/authorized_keys. + + Parameters + ---------- + cfg + Datashuttle Configs object. + + password + Password to the central server. + + log + If `True`, log the client connection process. + + """ generate_and_write_ssh_key(cfg.ssh_key_path) key = paramiko.RSAKey.from_private_key_file(cfg.ssh_key_path.as_posix()) @@ -312,18 +325,16 @@ def search_ssh_central_for_folders( ) -> Tuple[List[Any], List[Any]]: """Search for the search prefix in the search path over SSH. - Returns the list of matching folders, files are filtered out. - Parameters ---------- search_path - path to search for folders in + Path to search for folders in. search_prefix - search prefix for folder names e.g. "sub-*" + Search prefix for folder names e.g. "sub-*". cfg - see connect_client_with_logging() + See connect_client_with_logging(). verbose If `True`, if a search folder cannot be found, a message @@ -332,6 +343,10 @@ def search_ssh_central_for_folders( return_full_path include the search_path in the returned paths + Returns + ------- + Discovered folders (`all_folder_names`) and files (`all_filenames`). + """ client: paramiko.SSHClient with paramiko.SSHClient() as client: @@ -366,14 +381,14 @@ def get_list_of_folder_names_over_sftp( Parameters ---------- sftp - connected paramiko stfp object - (see search_ssh_central_for_folders()) + Connected paramiko stfp object + (see search_ssh_central_for_folders()). search_path - path to search for folders in + Path to search for folders in. search_prefix - prefix (can include wildcards) + Prefix (can include wildcards) to search folder names. verbose @@ -381,7 +396,11 @@ def get_list_of_folder_names_over_sftp( will be printed with the un-found path. return_full_path - include the search_path in the returned paths + include the search_path in the returned paths. + + Returns + ------- + Discovered folders (`all_folder_names`) and files (`all_filenames`). """ all_folder_names = [] diff --git a/datashuttle/utils/utils.py b/datashuttle/utils/utils.py index 6c102279b..7cc9df541 100644 --- a/datashuttle/utils/utils.py +++ b/datashuttle/utils/utils.py @@ -110,7 +110,7 @@ def get_user_input(message: str) -> str: def path_starts_with_base_folder(base_folder: Path, path_: Path) -> bool: - """Check whether the path starts with the base folder path.""" + """Return a bool indicating whether the path starts with the base folder path.""" return path_.as_posix().startswith(base_folder.as_posix()) @@ -159,6 +159,11 @@ def get_values_from_bids_formatted_name( sort If True, results are sorted before being returned. + Returns + ------- + all_values + The values of the corresponding `key` extracted from the name. + Notes ----- This function does not raise through datashuttle because we @@ -196,7 +201,7 @@ def get_values_from_bids_formatted_name( def sub_or_ses_value_to_int(value: str) -> int: - """Convert a subject or session value to an integer.""" + """Return a subject or session value converted to an integer.""" try: int_value = int(value) except ValueError: @@ -207,7 +212,7 @@ def sub_or_ses_value_to_int(value: str) -> int: def get_value_from_key_regexp(name: str, key: str) -> List[str]: - """Find the value related to the key in a BIDS-style key-value pair name. + """Return the value related to the key in a BIDS-style key-value pair name. e.g. sub-001_ses-312 would find 312 for key "ses". """ @@ -220,36 +225,36 @@ def get_value_from_key_regexp(name: str, key: str) -> List[str]: def integers_are_consecutive(list_of_ints: List[int]) -> bool: - """Check if a list of integers is consecutive.""" + """Return a bool indicating whether a list of integers is consecutive.""" diff_between_ints = diff(list_of_ints) return all([diff == 1 for diff in diff_between_ints]) def diff(x: List) -> List: - """Differentiate list of numbers. + """Return differentiated list of numbers. Slow, only to avoid adding numpy as a dependency. """ return [x[i + 1] - x[i] for i in range(len(x) - 1)] -def num_leading_zeros(string: str) -> int: +def num_leading_zeros(name: str) -> int: """Return the number of leading zeros in a sub- or ses- id. e.g. sub-001 has 2 leading zeros. int() strips leading zeros. """ - if string[:4] in ["sub-", "ses-"]: - string = string[4:] + if name[:4] in ["sub-", "ses-"]: + name = name[4:] - return len(string) - len(str(int(string))) + return len(name) - len(str(int(name))) def all_unique(list_: List) -> bool: - """Check that all values in a list are different.""" + """Return bool indicating whether all values in a list are different.""" return len(list_) == len(set(list_)) def all_identical(list_: List) -> bool: - """Check that all values in a list are identical.""" + """Return bool indicating whether all values in a list are identical.""" return len(set(list_)) == 1 diff --git a/datashuttle/utils/validation.py b/datashuttle/utils/validation.py index 390d82171..59ad5fbe2 100644 --- a/datashuttle/utils/validation.py +++ b/datashuttle/utils/validation.py @@ -166,6 +166,11 @@ def validate_list_of_names( If `True`, check that the prefix- value lengths are consistent across the passed list. + Returns + ------- + error_messages + A list of found validation errors. + """ if len(path_or_name_list) == 0: return [] @@ -220,6 +225,22 @@ def prefix_is_duplicate_or_has_bad_values( Ensure it is found only once in the name and that its value can be converted to integer. + + Parameters + ---------- + name + Name to check against template. + + prefix + "sub" or "ses" + + path_ + Path to the folder that is being checked. + + Returns + ------- + A list of validation errors. + """ value = re.findall(f"{prefix}(.*?)(?=_|$)", name) @@ -248,6 +269,22 @@ def new_name_duplicates_existing( (for example, when making sessions). However, if "sub-001_another-tag" exists, we should throw an error, because this shares the same subject id but refers to a different subject. + + Parameters + ---------- + new_name + The name to check against all existing names. + + existing_path_or_name_list + A list of existing names to check against. + + prefix + "sub" or "ses" + + Returns + ------- + A list of validation errors. + """ # Make a list of matches between `new_name` and any in `existing_names` new_name_id = utils.get_values_from_bids_formatted_name( @@ -278,7 +315,27 @@ def names_dont_match_templates( prefix: Prefix, name_templates: Optional[Dict] = None, ) -> List[str]: - """Validate a list of sub/ses names against the respective regexp `name_templates`.""" + """Validate a list of sub/ses names against the respective regexp `name_templates`. + + Parameters + ---------- + name + Name to check against template. + + path_ + Path to the folder that is being checked. + + prefix + "sub" or "ses" + + name_templates + Datashuttle's Name Templates dictionary defining the templates used. + + Returns + ------- + A list of validation errors. + + """ if name_templates is None: return [] @@ -315,6 +372,16 @@ def replace_tags_in_regexp(regexp: str) -> str: in the regexp with their actual regexp equivalent before comparison. Note `replace_date_time_tags_in_name()` operates in place on a list. + + Parameters + ---------- + regexp + The name template regexp in which to replace tags. + + Returns + ------- + The regexp with tags (e.g. @DATE@) formatted properly. + """ regexp_list = [regexp] date_regexp = r"\d\d\d\d\d\d\d\d" @@ -332,7 +399,24 @@ def replace_tags_in_regexp(regexp: str) -> str: def name_begins_with_bad_key( name: str, prefix: Prefix, path_: Path | None ) -> List[str]: - """Check that a list of NeuroBlueprint names begin with the required prefix (sub- or ses-).""" + """Check that a list of NeuroBlueprint names begin with the required prefix (sub- or ses-). + + Parameters + ---------- + name + Name of the folder to validate. + + prefix + "sub" or "ses". + + path_ + Path to the folder that is being validated. + + Returns + ------- + A list of validation errors. + + """ if name[:4] != f"{prefix}-": return [get_name_error(name, prefix, path_)] else: @@ -345,6 +429,19 @@ def names_include_special_characters( """Check that a list of NeuroBlueprint formatted names do not contain special characters. Special characters are characters that are not integers, letters, dash or underscore. + + Parameters + ---------- + name + Name of the folder to validate + + path_ + Path of the folder that is being validated. + + Returns + ------- + A list of validation errors. + """ if name_has_special_character(name): return [get_special_char_error(name, path_)] @@ -353,17 +450,30 @@ def names_include_special_characters( def name_has_special_character(name: str) -> bool: - """Check if the name contains special characters.""" + """Return a bool indicating if the name contains special characters.""" return not re.match("^[A-Za-z0-9_-]*$", name) def dashes_and_underscore_alternate_incorrectly( name: str, path_: Path | None ) -> List[str]: - """Check a list of NeuroBlueprint formatted names. + """Check a list of names for expected Neuroblueprint underscore-dash order. Names should have the "-" and "-" ordered correctly. Names should be key-value pairs separated by underscores e.g. sub-001_ses-001. + + Parameters + ---------- + name + Name of the folder to validate + + path_ + Path of the folder that is being validated. + + Returns + ------- + A list of validation errors. + """ discrim = {"-": 1, "_": -1} @@ -392,6 +502,19 @@ def value_lengths_are_inconsistent( """Determine if there are inconsistent value lengths for the sub or ses key in a list of names. For example, ["sub-01", "sub-001"] is an error. + + Parameters + ---------- + path_or_names_list + A path of names of folders to validate, or path to folders to validate. + + prefix + "sub" or "ses" + + Returns + ------- + A list of validation errors. + """ names_list = [ path_or_name if isinstance(path_or_name, str) else path_or_name.name @@ -420,7 +543,22 @@ def datetime_are_iso_format( name: str, path_: Path | None, ) -> List[str]: - """Check formatting for date-, time-, or datetime- tags.""" + """Check formatting for date-, time-, or datetime- tags. + + Parameters + ---------- + name + Name of the folder to validate + + path_ + Path of the folder that is being validated. + + Returns + ------- + error_message + A list of validation errors. + + """ formats = { "datetime": "%Y%m%dT%H%M%S", "time": "%H%M%S", @@ -458,6 +596,18 @@ def raise_display_mode( """Show a message by raising an error, displaying warning, or printing. Optionally log with the current datashuttle logger. + + Parameters + ---------- + message + Message to display. + + display_mode + Mode to display, "error", "warn" or "print". + + log + If `True`, log the message. + """ if display_mode == "error": utils.log_and_raise_error(message, NeuroBlueprintError) @@ -522,6 +672,11 @@ def validate_project( any folder not prefixed with sub-, ses- or a valid datatype will raise a validation issue. + Returns + ------- + error_messages + A list of validation errors. + """ error_messages = [] @@ -731,6 +886,20 @@ def check_high_level_project_structure( This includes checking that the project folder name is valid, and that the project folder contains either a "rawdata" or "derivatives" folder. + + Parameters + ---------- + cfg + Datashuttle Configs class. + + include_central + If `True`, the central project is also checked. + + Returns + ------- + error_messages + A list of validation errors. + """ # To avoid circular imports from datashuttle.utils.folders import search_for_folders @@ -801,6 +970,23 @@ def check_strict_mode( prefix typos and other issues (e.g. if subs-extra is a folder, or "rat1", how do we know if it is intended to be a subject folder and validate it, or just an auxiliary folder. + + Parameters + ---------- + cfg + Datashuttle Configs class. + + top_level_folder + "rawdata" or "derivatives". + + include_central + If `True`, the central project is also checked. + + Returns + ------- + error_messages + A list of validation errors. + """ # For circular imports from datashuttle.utils import folders @@ -899,6 +1085,20 @@ def strip_uncheckable_names( This is necessary as some validation steps (e.g. checking duplicate names) requires conversion to `int` and will fail for this reason if these bad names are not removed. + + Parameters + ---------- + path_or_names_list + A path of names of folders to validate, or path to folders to validate. + + prefix + "sub" or "ses". + + Returns + ------- + List of path or names which the uncheckable ones (e.g. that are too broken + to be validated against) removed. + """ new_list = [] @@ -928,7 +1128,23 @@ def strip_uncheckable_names( def check_datatypes_are_valid( datatype: Union[List[str], str], allow_all: bool = False ) -> str | None: - """Check a datatype of list of datatypes is a valid NeuroBlueprint datatype.""" + """Check a datatype of list of datatypes is a valid NeuroBlueprint datatype. + + Parameters + ---------- + datatype + A str or a list of str where str is a NeuroBlueprint datatype. + + allow_all + At this stage, "all" may still be included in the list of datatypes. + This is valid in some cases as it indicates all datatypes. + + Returns + ------- + message + An error message or None if no errors found. + + """ datatype_folders = canonical_folders.get_datatype_folders() if isinstance(datatype, str): diff --git a/pyproject.toml b/pyproject.toml index 0a9166652..87af8ef45 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -96,8 +96,6 @@ line-length = 79 exclude = ["__init__.py","build",".eggs"] fix = true - - # Ruff config is not exactly the same as in the movement repo, as # currently we are only adding linting enforcement to docstrings. # Other ruff rules that are also present in movement repo can be @@ -105,10 +103,19 @@ fix = true [tool.ruff.lint] # See https://docs.astral.sh/ruff/rules/ -ignore = ["E203","E501","E731","C901","W291","W293","E402","E722", - "D100", # missing docstring in public module (not enforced FOR NOW) - "D203", # one blank line before class - "D213", # multi-line-summary second line + +ignore = [ + "E203", # whitespace before ':' (conflicts with Black) + "E501", # line too long (handled by Black) + "E731", # do not assign a lambda expression, use a def + "C901", # function is too complex + "W291", # trailing whitespace + "W293", # blank line contains whitespace + "E402", # module level import not at top of file + "E722", # do not use bare 'except' + "D100", # missing docstring in public module + "D203", # 1 blank line required before class docstring (conflicts with D211) + "D213", # multi-line docstring summary should start at the second line (conflicts with D212) ] select = [ "I", # isort @@ -124,11 +131,7 @@ per-file-ignores = { "tests/*" = [ "D400", # first line should end with a period. "D415", # first line should end with a period, question mark... "D205", # missing blank line between summary and description -], "__init__.py" = [ - # This was part of the old config - # Is this needed? __init__.py is already part of tool.ruff.exclude - "F401", # auto remove unused imports -] } +]} [tool.ruff.format] docstring-code-format = true # Also format code in docstrings From 05a682623f048f57be0df9e05820b5b3b70ece97 Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Sun, 22 Jun 2025 22:59:42 +0100 Subject: [PATCH 70/70] Fix tests and remove PLACEHOLDER in tests. --- tests/test_utils.py | 7 ------- tests/tests_integration/base.py | 2 -- tests/tests_integration/test_configs.py | 2 -- tests/tests_integration/test_create_folders.py | 4 ---- tests/tests_integration/test_filesystem_transfer.py | 7 ------- tests/tests_integration/test_formatting.py | 3 --- tests/tests_integration/test_local_only_mode.py | 2 -- tests/tests_integration/test_logging.py | 6 ------ tests/tests_integration/test_settings.py | 2 -- tests/tests_integration/test_ssh_file_transfer.py | 3 --- tests/tests_integration/test_ssh_setup.py | 2 -- tests/tests_integration/test_transfer_checks.py | 3 --- tests/tests_integration/test_validation.py | 7 ------- tests/tests_tui/test_local_only_project.py | 2 -- tests/tests_tui/test_tui_configs.py | 3 --- tests/tests_tui/test_tui_create_folders.py | 7 ++----- tests/tests_tui/test_tui_datatypes.py | 2 -- tests/tests_tui/test_tui_directorytree.py | 2 -- tests/tests_tui/test_tui_get_help.py | 1 - tests/tests_tui/test_tui_logging.py | 2 -- tests/tests_tui/test_tui_transfer.py | 8 -------- tests/tests_tui/test_tui_widgets_and_defaults.py | 6 ------ tests/tests_tui/tui_base.py | 4 ---- tests/tests_unit/test_unit.py | 2 -- tests/tests_unit/test_validation_unit.py | 7 +------ 25 files changed, 3 insertions(+), 93 deletions(-) diff --git a/tests/test_utils.py b/tests/test_utils.py index ade43bf76..c70928fe5 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -95,7 +95,6 @@ def delete_all_folders_in_local_path(project): def delete_all_folders_in_project_path(project, local_or_central): - """PLACEHOLDER.""" folder = f"{local_or_central}_path" if project.cfg is None or ( @@ -112,7 +111,6 @@ def delete_all_folders_in_project_path(project, local_or_central): def delete_project_if_it_exists(project_name): - """PLACEHOLDER.""" config_path, _ = canonical_folders.get_project_datashuttle_path( project_name ) @@ -149,7 +147,6 @@ def make_test_path(base_path, local_or_central, test_project_name): def create_all_pathtable_files(pathtable): - """PLACEHOLDER.""" for i in range(pathtable.shape[0]): filepath = pathtable["base_folder"][i] / pathtable["path"][i] filepath.parents[0].mkdir(parents=True, exist_ok=True) @@ -389,7 +386,6 @@ def make_local_folders_with_files_in( def check_configs(project, kwargs, config_path=None): - """PLACEHOLDER.""" if config_path is None: config_path = project._config_path @@ -424,7 +420,6 @@ def check_project_configs( def check_config_file(config_path, *kwargs): - """PLACEHOLDER.""" with open(config_path) as config_file: config_yaml = yaml.full_load(config_file) @@ -440,7 +435,6 @@ def check_config_file(config_path, *kwargs): def get_top_level_folder_path( project, local_or_central="local", folder_name="rawdata" ): - """PLACEHOLDER.""" assert folder_name in canonical_folders.get_top_level_folders(), ( "folder_name must be canonical e.g. rawdata" ) @@ -486,7 +480,6 @@ def handle_upload_or_download( def get_transfer_func( project, upload_or_download, transfer_method, top_level_folder=None ): - """PLACEHOLDER.""" if transfer_method == "top_level_folder": assert top_level_folder is not None, "must pass top-level-folder" assert top_level_folder in [None, "rawdata", "derivatives"] diff --git a/tests/tests_integration/base.py b/tests/tests_integration/base.py index a9c9174ec..5ce5359c1 100644 --- a/tests/tests_integration/base.py +++ b/tests/tests_integration/base.py @@ -7,8 +7,6 @@ class BaseTest: - """PLACEHOLDER.""" - @pytest.fixture(scope="function") def no_cfg_project(test): """Fixture that creates an empty project. Ignore the warning diff --git a/tests/tests_integration/test_configs.py b/tests/tests_integration/test_configs.py index 168b3df74..5156f84b7 100644 --- a/tests/tests_integration/test_configs.py +++ b/tests/tests_integration/test_configs.py @@ -10,8 +10,6 @@ class TestConfigs(BaseTest): - """PLACEHOLDER.""" - # Test Errors # ------------------------------------------------------------- diff --git a/tests/tests_integration/test_create_folders.py b/tests/tests_integration/test_create_folders.py index e82815c9f..cb1de20a3 100644 --- a/tests/tests_integration/test_create_folders.py +++ b/tests/tests_integration/test_create_folders.py @@ -13,8 +13,6 @@ class TestCreateFolders(BaseTest): - """PLACEHOLDER.""" - @pytest.mark.parametrize("project", ["local", "full"], indirect=True) def test_generate_folders_default_ses(self, project): """Make a subject folders with full tree. Don't specify @@ -485,12 +483,10 @@ def test_get_next_sub_and_ses_name_template(self, project): # ---------------------------------------------------------------------------------- def get_formatted_date_and_time(self): - """PLACEHOLDER.""" date = str(datetime.datetime.now().date()) date = date.replace("-", "") time_ = datetime.datetime.now().time().strftime("%Hh%Mm") return date, time_ def broad_datatypes(self): - """PLACEHOLDER.""" return canonical_configs.get_broad_datatypes() diff --git a/tests/tests_integration/test_filesystem_transfer.py b/tests/tests_integration/test_filesystem_transfer.py index 82b244459..7dbfab4d5 100644 --- a/tests/tests_integration/test_filesystem_transfer.py +++ b/tests/tests_integration/test_filesystem_transfer.py @@ -13,8 +13,6 @@ class TestFileTransfer(BaseTest): - """PLACEHOLDER.""" - @pytest.mark.parametrize( "top_level_folder", canonical_folders.get_top_level_folders() ) @@ -59,7 +57,6 @@ def test_transfer_empty_folder_structure( ) def test_empty_folder_is_not_transferred(self, project): - """PLACEHOLDER.""" project.create_folders("rawdata", "sub-001") project.upload_rawdata() assert not ( @@ -130,7 +127,6 @@ def test_transfer_across_top_level_folders( @pytest.mark.parametrize("upload_or_download", ["upload", "download"]) def test_transfer_all_top_level_folders(self, project, upload_or_download): - """PLACEHOLDER.""" subs, sessions = test_utils.get_default_sub_sessions_to_test() for top_level_folder in canonical_folders.get_top_level_folders(): @@ -612,7 +608,6 @@ def test_overwrite_different_size_different_times( assert test_utils.read_file(central_file_path) == ["file earlier"] def get_paths_to_a_local_and_central_file(self, project, top_level_folder): - """PLACEHOLDER.""" path_to_test_file = ( Path(top_level_folder) / "sub-001" @@ -629,7 +624,6 @@ def get_paths_to_a_local_and_central_file(self, project, top_level_folder): def setup_overwrite_file_tests( self, upload_or_download, top_level_folder, project ): - """PLACEHOLDER.""" local_file_path, central_file_path = ( self.get_paths_to_a_local_and_central_file( project, top_level_folder @@ -737,7 +731,6 @@ def test_specific_file_or_folder( assert transferred_files == to_test_against def setup_specific_file_or_folder_files(self, project, top_level_folder): - """PLACEHOLDER.""" project.create_folders( top_level_folder, ["sub-001", "sub-002"], diff --git a/tests/tests_integration/test_formatting.py b/tests/tests_integration/test_formatting.py index cf5dffafe..0b5db8338 100644 --- a/tests/tests_integration/test_formatting.py +++ b/tests/tests_integration/test_formatting.py @@ -6,8 +6,6 @@ class TestFormatting(BaseTest): - """PLACEHOLDER.""" - @pytest.mark.parametrize("prefix", ["sub", "ses"]) @pytest.mark.parametrize( "input", [1, {"test": "one"}, 1.0, ["1", "2", ["three"]]] @@ -65,7 +63,6 @@ def test_format_names_prefix(self): @pytest.mark.parametrize("top_level_folder", ["rawdata", "derivatives"]) @pytest.mark.parametrize("project", ["local", "full"], indirect=True) def test_warning_non_consecutive_numbers(self, project, top_level_folder): - """PLACEHOLDER.""" project.create_folders( top_level_folder, ["sub-01", "sub-02", "sub-04"], diff --git a/tests/tests_integration/test_local_only_mode.py b/tests/tests_integration/test_local_only_mode.py index 73c098ba5..b2853cbad 100644 --- a/tests/tests_integration/test_local_only_mode.py +++ b/tests/tests_integration/test_local_only_mode.py @@ -12,8 +12,6 @@ class TestLocalOnlyProject(BaseTest): - """PLACEHOLDER.""" - def test_bad_setup(self, tmp_path): """Test setup without providing both central_path and connection method (distinguishing a full vs local-only project). diff --git a/tests/tests_integration/test_logging.py b/tests/tests_integration/test_logging.py index fe1b11468..9a3ec8855 100644 --- a/tests/tests_integration/test_logging.py +++ b/tests/tests_integration/test_logging.py @@ -17,8 +17,6 @@ class TestLogging: - """PLACEHOLDER.""" - @pytest.fixture(scope="function") def teardown_logger(self): """Ensure the logger is deleted at the end of each test.""" @@ -161,7 +159,6 @@ def test_logs_make_config_file(self, clean_project_name, tmp_path): assert "Update successful. New config file:" in log def test_logs_update_config_file(self, project): - """PLACEHOLDER.""" project.update_config_file(central_host_id="test_id") log = test_utils.read_log_file(project.cfg.logging_path) @@ -176,7 +173,6 @@ def test_logs_update_config_file(self, project): @pytest.mark.parametrize("project", ["local", "full"], indirect=True) def test_create_folders(self, project): - """PLACEHOLDER.""" subs = ["sub-111", f"sub-002{tags('to')}004"] ses = ["ses-123", "ses-101"] @@ -404,7 +400,6 @@ def test_clear_logging_path(self, clean_project_name, tmp_path): # ---------------------------------------------------------------------------------- def test_logs_check_update_config_error(self, project): - """PLACEHOLDER.""" with pytest.raises(ConfigError): project.update_config_file( connection_method="ssh", central_host_username=None @@ -423,7 +418,6 @@ def test_logs_check_update_config_error(self, project): @pytest.mark.parametrize("project", ["local", "full"], indirect=True) def test_logs_bad_create_folders_error(self, project): - """PLACEHOLDER.""" project.create_folders("rawdata", "sub-001", datatype="all") test_utils.delete_log_files(project.cfg.logging_path) diff --git a/tests/tests_integration/test_settings.py b/tests/tests_integration/test_settings.py index a20006413..bcb9c73ec 100644 --- a/tests/tests_integration/test_settings.py +++ b/tests/tests_integration/test_settings.py @@ -11,8 +11,6 @@ class TestPersistentSettings(BaseTest): - """PLACEHOLDER.""" - @pytest.mark.parametrize("project", ["local", "full"], indirect=True) def test_persistent_settings_name_templates(self, project): """Test the 'name_templates' option that is stored in persistent diff --git a/tests/tests_integration/test_ssh_file_transfer.py b/tests/tests_integration/test_ssh_file_transfer.py index 50e06acb8..4e98894c6 100644 --- a/tests/tests_integration/test_ssh_file_transfer.py +++ b/tests/tests_integration/test_ssh_file_transfer.py @@ -13,8 +13,6 @@ class TestFileTransfer: - """PLACEHOLDER.""" - @pytest.fixture( scope="class", params=[ # Set running SSH or local filesystem (see docstring). @@ -115,7 +113,6 @@ def pathtable_and_project(self, request, tmpdir_factory): # ------------------------------------------------------------------------- def central_from_local(self, path_): - """PLACEHOLDER.""" return Path(str(copy.copy(path_)).replace("local", "central")) # ------------------------------------------------------------------------- diff --git a/tests/tests_integration/test_ssh_setup.py b/tests/tests_integration/test_ssh_setup.py index 5c7bcd702..31c1fa72e 100644 --- a/tests/tests_integration/test_ssh_setup.py +++ b/tests/tests_integration/test_ssh_setup.py @@ -13,8 +13,6 @@ @pytest.mark.skipif(ssh_config.TEST_SSH is False, reason="TEST_SSH is false") class TestSSH: - """PLACEHOLDER.""" - @pytest.fixture(scope="function") def project(test, tmp_path): """Make a project as per usual, but now add diff --git a/tests/tests_integration/test_transfer_checks.py b/tests/tests_integration/test_transfer_checks.py index 145ddc0f6..7a6f631f3 100644 --- a/tests/tests_integration/test_transfer_checks.py +++ b/tests/tests_integration/test_transfer_checks.py @@ -10,8 +10,6 @@ class TestTransferChecks(BaseTest): - """PLACEHOLDER.""" - @pytest.mark.parametrize( "top_level_folders", [["rawdata", "derivatives"], ["rawdata"], ["derivatives"]], @@ -92,7 +90,6 @@ def test_rclone_check(self, project, top_level_folders): assert path_ not in results_paths def get_folder_structure(self, top_level_folder): - """PLACEHOLDER.""" # fmt: off folder_structure = [ [f"{top_level_folder}/sub-001/ses-001/ephys/local_only_1.txt", "local_only"], diff --git a/tests/tests_integration/test_validation.py b/tests/tests_integration/test_validation.py index b35510680..c8eaa0d3a 100644 --- a/tests/tests_integration/test_validation.py +++ b/tests/tests_integration/test_validation.py @@ -15,8 +15,6 @@ class TestValidation(BaseTest): - """PLACEHOLDER.""" - @pytest.mark.parametrize( "sub_name", ["sub-001", "sub-999_@DATE@", "sub-001_random-tag_another-tag"], @@ -158,7 +156,6 @@ def test_warn_on_inconsistent_sub_and_ses_value_lengths(self, project): def check_inconsistent_sub_or_ses_value_length_warning( self, project, warn_idx=0, include_central=True ): - """PLACEHOLDER.""" with pytest.warns(UserWarning) as w: project.validate_project( "rawdata", display_mode="warn", include_central=include_central @@ -383,7 +380,6 @@ def test_validate_project(self, project): @pytest.mark.parametrize("prefix", ["sub", "ses"]) def test_validate_project_returned_list(self, project, prefix): - """PLACEHOLDER.""" bad_names = [ f"{prefix}-001", f"{prefix}-001_@DATE@", @@ -415,7 +411,6 @@ def test_validate_project_returned_list(self, project, prefix): assert "VALUE_LENGTH" in concat_error def test_output_paths_are_valid(self, project): - """PLACEHOLDER.""" sub_name = "sub-001x" ses_name = "ses-001x" project.create_folders( @@ -792,7 +787,6 @@ def test_name_templates_validate_project(self, project): # ---------------------------------------------------------------------------------- def test_quick_validation(self, mocker, project): - """PLACEHOLDER.""" project.create_folders("rawdata", "sub-1") os.makedirs(project.cfg["local_path"] / "rawdata" / "sub-02") project.create_folders("derivatives", "sub-1") @@ -849,7 +843,6 @@ def test_quick_validation_top_level_folder(self, project): @pytest.mark.parametrize("top_level_folder", ["rawdata", "derivatives"]) def test_strict_mode_validation(self, project, top_level_folder): - """PLACEHOLDER.""" project.create_folders( top_level_folder, ["sub-001", "sub-002"], diff --git a/tests/tests_tui/test_local_only_project.py b/tests/tests_tui/test_local_only_project.py index 0ac50e587..8a06ccfa7 100644 --- a/tests/tests_tui/test_local_only_project.py +++ b/tests/tests_tui/test_local_only_project.py @@ -5,8 +5,6 @@ class TestTuiLocalOnlyProject(TuiBase): - """PLACEHOLDER.""" - @pytest.mark.asyncio async def test_local_only_make_project( self, diff --git a/tests/tests_tui/test_tui_configs.py b/tests/tests_tui/test_tui_configs.py index 437b8deef..c2e110b65 100644 --- a/tests/tests_tui/test_tui_configs.py +++ b/tests/tests_tui/test_tui_configs.py @@ -15,8 +15,6 @@ class TestTuiConfigs(TuiBase): - """PLACEHOLDER.""" - # ------------------------------------------------------------------------- # Test New Project Configs # ------------------------------------------------------------------------- @@ -340,7 +338,6 @@ async def test_configs_select_path(self, monkeypatch): @pytest.mark.asyncio async def test_bad_configs_screen_input(self, empty_project_paths): - """PLACEHOLDER.""" app = TuiApp() async with app.run_test(size=self.tui_size()) as pilot: # Select a new project, check NewProjectScreen is displayed correctly. diff --git a/tests/tests_tui/test_tui_create_folders.py b/tests/tests_tui/test_tui_create_folders.py index e86aa0885..b7d3822ee 100644 --- a/tests/tests_tui/test_tui_create_folders.py +++ b/tests/tests_tui/test_tui_create_folders.py @@ -13,8 +13,6 @@ class TestTuiCreateFolders(TuiBase): - """PLACEHOLDER.""" - # ------------------------------------------------------------------------- # General test Create Folders # ------------------------------------------------------------------------- @@ -222,11 +220,12 @@ async def test_create_folders_bad_validation_tooltips( ).tooltip == "Formatted names: ['sub-001']" ) + assert ( pilot.app.screen.query_one( "#create_folders_session_input" ).tooltip - == "DUPLICATE_PREFIX: The name: ses-001_ses-001 of contains more than one instance of the prefix ses." + == "DUPLICATE_PREFIX: The name: ses-001_ses-001 contains more than one instance of the prefix ses." ) await self.fill_input( @@ -708,7 +707,6 @@ async def test_create_folders_settings_top_level_folder( async def iterate_and_check_all_datatype_folders( self, pilot, subs, sessions ): - """PLACEHOLDER.""" project = pilot.app.screen.interface.project folder_used = test_utils.get_all_broad_folders_used(value=False) @@ -726,7 +724,6 @@ async def iterate_and_check_all_datatype_folders( async def create_folders_and_check_output( self, pilot, project, subs, sessions, folder_used ): - """PLACEHOLDER.""" await self.scroll_to_click_pause( pilot, "#create_folders_create_folders_button", diff --git a/tests/tests_tui/test_tui_datatypes.py b/tests/tests_tui/test_tui_datatypes.py index cc2df3bdd..76a438f35 100644 --- a/tests/tests_tui/test_tui_datatypes.py +++ b/tests/tests_tui/test_tui_datatypes.py @@ -13,7 +13,6 @@ class TestDatatypesTUI(TuiBase): async def test_select_displayed_datatypes_create( self, setup_project_paths ): - """PLACEHOLDER.""" tmp_config_path, tmp_path, project_name = setup_project_paths.values() app = TuiApp() @@ -134,7 +133,6 @@ async def test_select_displayed_datatypes_create( async def test_select_displayed_datatypes_transfer( self, setup_project_paths, mocker ): - """PLACEHOLDER.""" tmp_config_path, tmp_path, project_name = setup_project_paths.values() app = TuiApp() diff --git a/tests/tests_tui/test_tui_directorytree.py b/tests/tests_tui/test_tui_directorytree.py index 73f340d24..2bcd33ec8 100644 --- a/tests/tests_tui/test_tui_directorytree.py +++ b/tests/tests_tui/test_tui_directorytree.py @@ -151,7 +151,6 @@ async def test_create_folders_directorytree_clipboard( async def test_failed_pyperclip_copy( self, setup_project_paths, monkeypatch ): - """PLACEHOLDER.""" tmp_config_path, tmp_path, project_name = setup_project_paths.values() app = TuiApp() @@ -244,7 +243,6 @@ def set_signal_to_path(path_): async def test_create_folders_directorytree_rename( self, setup_project_paths ): - """PLACEHOLDER.""" tmp_config_path, tmp_path, project_name = setup_project_paths.values() rawdata_path = tmp_path / "local" / project_name / "rawdata" diff --git a/tests/tests_tui/test_tui_get_help.py b/tests/tests_tui/test_tui_get_help.py index 192487557..aa8986b8c 100644 --- a/tests/tests_tui/test_tui_get_help.py +++ b/tests/tests_tui/test_tui_get_help.py @@ -11,7 +11,6 @@ class TestTuiSettings(TuiBase): @pytest.mark.asyncio async def test_get_help(self, empty_project_paths): - """PLACEHOLDER.""" app = TuiApp() async with app.run_test(size=self.tui_size()) as pilot: await self.scroll_to_click_pause( diff --git a/tests/tests_tui/test_tui_logging.py b/tests/tests_tui/test_tui_logging.py index 9712ac1cd..33aa07978 100644 --- a/tests/tests_tui/test_tui_logging.py +++ b/tests/tests_tui/test_tui_logging.py @@ -7,8 +7,6 @@ class TestTuiLogging(TuiBase): - """PLACEHOLDER.""" - @pytest.mark.asyncio async def test_logging(self, setup_project_paths): """Test logging by running some commands, checking they diff --git a/tests/tests_tui/test_tui_transfer.py b/tests/tests_tui/test_tui_transfer.py index c273884b9..af26264f6 100644 --- a/tests/tests_tui/test_tui_transfer.py +++ b/tests/tests_tui/test_tui_transfer.py @@ -17,7 +17,6 @@ class TestTuiTransfer(TuiBase): async def test_transfer_entire_project( self, setup_project_paths, upload_or_download ): - """PLACEHOLDER.""" tmp_config_path, tmp_path, project_name = setup_project_paths.values() subs, sessions = test_utils.get_default_sub_sessions_to_test() @@ -68,7 +67,6 @@ async def check_persistent_settings(self, pilot): ) async def set_overwrite_checkbox(self, pilot, overwrite_setting): - """PLACEHOLDER.""" all_positions = {"never": None, "always": 5, "if_source_newer": 6} position = all_positions[overwrite_setting] @@ -78,7 +76,6 @@ async def set_overwrite_checkbox(self, pilot, overwrite_setting): ) async def set_transfer_tab_dry_run_checkbox(self, pilot, dry_run_setting): - """PLACEHOLDER.""" if ( pilot.app.screen.query_one("#transfer_tab_dry_run_checkbox") is not dry_run_setting @@ -109,7 +106,6 @@ async def set_and_check_persistent_settings( async def test_transfer_top_level_folder( self, setup_project_paths, top_level_folder, upload_or_download ): - """PLACEHOLDER.""" tmp_config_path, tmp_path, project_name = setup_project_paths.values() subs, sessions = test_utils.get_default_sub_sessions_to_test() @@ -155,7 +151,6 @@ async def test_transfer_top_level_folder( async def test_transfer_custom( self, setup_project_paths, top_level_folder, upload_or_download ): - """PLACEHOLDER.""" tmp_config_path, tmp_path, project_name = setup_project_paths.values() subs, sessions = test_utils.get_default_sub_sessions_to_test() @@ -225,7 +220,6 @@ async def test_transfer_custom( async def switch_top_level_folder_select( self, pilot, id, top_level_folder ): - """PLACEHOLDER.""" if top_level_folder == "rawdata": assert pilot.app.screen.query_one(id).value == "rawdata" else: @@ -233,7 +227,6 @@ async def switch_top_level_folder_select( assert pilot.app.screen.query_one(id).value == "derivatives" async def run_transfer(self, pilot, upload_or_download): - """PLACEHOLDER.""" # Check assumed default is correct on the transfer switch assert pilot.app.screen.query_one("#transfer_switch").value is False @@ -250,7 +243,6 @@ def setup_project_for_data_transfer( top_level_folder_list, upload_or_download, ): - """PLACEHOLDER.""" for top_level_folder in top_level_folder_list: test_utils.make_and_check_local_project_folders( project, diff --git a/tests/tests_tui/test_tui_widgets_and_defaults.py b/tests/tests_tui/test_tui_widgets_and_defaults.py index 12c81031e..e37117f31 100644 --- a/tests/tests_tui/test_tui_widgets_and_defaults.py +++ b/tests/tests_tui/test_tui_widgets_and_defaults.py @@ -208,7 +208,6 @@ async def test_new_project_configs(self, empty_project_paths): async def check_new_project_ssh_widgets( self, configs_content, ssh_on, save_pressed=False ): - """PLACEHOLDER.""" assert configs_content.query_one( "#configs_setup_ssh_connection_button" ).visible is ( @@ -1085,7 +1084,6 @@ async def test_all_checkboxes(self, setup_project_paths): await pilot.pause() def check_datatype_checkboxes(self, pilot, tab, expected_on): - """PLACEHOLDER.""" assert tab in ["create", "transfer"] if tab == "create": id = "#create_folders_datatype_checkboxes" @@ -1109,7 +1107,6 @@ def check_datatype_checkboxes(self, pilot, tab, expected_on): @pytest.mark.asyncio async def test_all_transfer_widgets(self, setup_project_paths): - """PLACEHOLDER.""" tmp_config_path, tmp_path, project_name = setup_project_paths.values() app = TuiApp() @@ -1295,7 +1292,6 @@ async def test_all_transfer_widgets(self, setup_project_paths): @pytest.mark.asyncio async def test_overwrite_existing_files(self, setup_project_paths): - """PLACEHOLDER.""" tmp_config_path, tmp_path, project_name = setup_project_paths.values() app = TuiApp() @@ -1387,7 +1383,6 @@ async def test_dry_run(self, setup_project_paths): # combine. def check_dry_run(self, pilot, project_name, value): - """PLACEHOLDER.""" assert ( pilot.app.screen.query_one("#transfer_tab_dry_run_checkbox").value == value @@ -1402,7 +1397,6 @@ def check_dry_run(self, pilot, project_name, value): def check_overwrite_existing_files_configs( self, pilot, project_name, value ): - """PLACEHOLDER.""" assert ( pilot.app.screen.query_one("#transfer_tab_overwrite_select").value == value diff --git a/tests/tests_tui/tui_base.py b/tests/tests_tui/tui_base.py index bd134da0a..b5b7e88d8 100644 --- a/tests/tests_tui/tui_base.py +++ b/tests/tests_tui/tui_base.py @@ -177,13 +177,10 @@ async def check_and_click_onto_existing_project(self, pilot, project_name): ) async def change_checkbox(self, pilot, id): - """PLACEHOLDER.""" pilot.app.screen.query_one(id).toggle() await pilot.pause() async def switch_tab(self, pilot, tab): - """PLACEHOLDER.""" - assert tab in ["create", "transfer", "configs", "logging"] assert tab in ["create", "transfer", "configs", "logging", "validate"] content_tab = ContentTab.add_prefix(f"tabscreen_{tab}_tab") @@ -237,7 +234,6 @@ async def move_select_to_position(self, pilot, id, position): await pilot.pause() async def click_and_await_transfer(self, pilot): - """PLACEHOLDER.""" await self.scroll_to_click_pause(pilot, "#transfer_transfer_button") await self.scroll_to_click_pause(pilot, "#confirm_ok_button") diff --git a/tests/tests_unit/test_unit.py b/tests/tests_unit/test_unit.py index c63b045a8..ec2cb875f 100644 --- a/tests/tests_unit/test_unit.py +++ b/tests/tests_unit/test_unit.py @@ -45,7 +45,6 @@ def test_datetime_string_replacement(self, key, underscore_position): @pytest.mark.parametrize("prefix", ["sub", "ses"]) def test_process_to_keyword_in_sub_input(self, prefix): - """PLACEHOLDER.""" results = formatting.update_names_with_range_to_flag( [f"{prefix}-001", f"{prefix}-01{tags('to')}123"], prefix ) @@ -94,7 +93,6 @@ def test_process_to_keyword_in_sub_input(self, prefix): def test_process_to_keyword_bad_input_raises_error( self, prefix, bad_input ): - """PLACEHOLDER.""" bad_input = bad_input.replace("prefix", prefix) with pytest.raises(ValueError) as e: diff --git a/tests/tests_unit/test_validation_unit.py b/tests/tests_unit/test_validation_unit.py index 21c6437c0..262a4741d 100644 --- a/tests/tests_unit/test_validation_unit.py +++ b/tests/tests_unit/test_validation_unit.py @@ -4,8 +4,6 @@ class TestValidationUnit: - """PLACEHOLDER.""" - @pytest.mark.parametrize("prefix", ["sub", "ses"]) def test_more_than_one_instance(self, prefix): """Check that any duplicate sub or ses values are caught @@ -17,7 +15,7 @@ def test_more_than_one_instance(self, prefix): assert len(error_message) == 1 assert ( - f"DUPLICATE_PREFIX: The name: {prefix}-99_date-20231214_{prefix}-98 of " + f"DUPLICATE_PREFIX: The name: {prefix}-99_date-20231214_{prefix}-98 " f"contains more than one instance of the prefix {prefix}." == error_message[0] ) @@ -71,7 +69,6 @@ def test_special_characters_in_format_names(self, prefix): ], ) def test_prefix_is_not_an_integer(self, prefix_and_names): - """PLACEHOLDER.""" prefix, names = prefix_and_names error_messages = validation.validate_list_of_names(names, prefix) @@ -251,7 +248,6 @@ def test_tags_autoreplace_in_regexp(self): ) def test_handle_path(self): - """PLACEHOLDER.""" output = validation.handle_path("message", None) assert output == "message" @@ -263,7 +259,6 @@ def test_handle_path(self): @pytest.mark.parametrize("prefix", ["sub", "ses"]) def test_datetime_iso_format(self, prefix): - """PLACEHOLDER.""" # Test dates error_messages = validation.validate_list_of_names( [