diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index dabc97e78..5e272caaf 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -8,23 +8,33 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks rev: v5.0.0 hooks: + - id: check-added-large-files - id: check-docstring-first - id: check-executables-have-shebangs + - id: check-case-conflict - id: check-merge-conflict + - id: check-symlinks + - id: check-yaml - id: check-toml + - id: debug-statements - id: end-of-file-fixer - id: mixed-line-ending args: [--fix=lf] + #- id: name-tests-test + # args: ["--pytest-test-first"] - id: requirements-txt-fixer - id: trailing-whitespace + - repo: https://github.com/pre-commit/pygrep-hooks + rev: v1.10.0 + hooks: + - id: rst-backticks + - id: rst-directive-colons + - id: rst-inline-touching-normal - repo: https://github.com/astral-sh/ruff-pre-commit rev: v0.11.12 hooks: - id: ruff - - repo: https://github.com/psf/black - rev: 25.1.0 - hooks: - - id: black + - id: ruff-format - repo: https://github.com/pre-commit/mirrors-mypy rev: v1.16.0 hooks: diff --git a/datashuttle/configs/canonical_configs.py b/datashuttle/configs/canonical_configs.py index ac41d6e4a..381d04f73 100644 --- a/datashuttle/configs/canonical_configs.py +++ b/datashuttle/configs/canonical_configs.py @@ -1,12 +1,10 @@ -""" -This module contains all information for the required -format of the configs class. This is clearly defined -as configs can be provided from file or input dynamically -and so careful checks must be done. - -If adding a new config, first add the key to -get_canonical_configs() and type to -get_canonical_configs() +"""Contains all information defining the required format of the Configs class. + +This format is clearly specified because configs can be supplied +either from a file or dynamically, so careful validation is required. + +If adding a new config key: +- First add the key to `get_canonical_configs()` and define its type in the same function """ from __future__ import annotations @@ -31,10 +29,7 @@ def get_canonical_configs() -> dict: - """ - The only permitted types for DataShuttle - config values. - """ + """Return the only permitted types for DataShuttle config values.""" canonical_configs = { "local_path": Union[str, Path], "central_path": Optional[Union[str, Path]], @@ -47,10 +42,9 @@ def get_canonical_configs() -> dict: def keys_str_on_file_but_path_in_class() -> list[str]: - """ - All configs which are paths are converted to pathlib.Path - objects on load. This list indicates which config entries - are to be converted to Path. + """Return a list of all config keys that are paths but stored as str in the file. + + These are converted to pathlib.Path objects when loaded. """ return [ "local_path", @@ -64,18 +58,18 @@ def keys_str_on_file_but_path_in_class() -> list[str]: def check_dict_values_raise_on_fail(config_dict: Configs) -> None: - """ - Central function for performing checks on a - DataShuttle Configs UserDict class. This should - be run after any change to the configs (e.g. - make_config_file, update_config_file, supply_config_file). + """Perform checks on a DataShuttle Configs UserDict class. - This will raise assert if condition is not met. + This should be run after any change to the configs + (e.g. make_config_file, update_config_file, supply_config_file). + + This will raise an error if a condition is not met. Parameters ---------- + config_dict + datashuttle config UserDict - config_dict : datashuttle config UserDict """ canonical_dict = get_canonical_configs() @@ -142,10 +136,10 @@ def check_dict_values_raise_on_fail(config_dict: Configs) -> None: def raise_on_bad_local_only_project_configs(config_dict: Configs) -> None: - """ - There is no circumstance where one of `central_path` and `connection_method` - should be set and not the other. Either both are set ('full' project) or - neither are ('local only' project). Check this assumption here. + """Check that both or neither of `central_path` and `connection_method` are set. + + There is no circumstance where one is set and not the other. Either both are set + ('full' project) or both are `None` ('local only' project). """ params_are_none = local_only_configs_are_none(config_dict) @@ -159,6 +153,7 @@ def raise_on_bad_local_only_project_configs(config_dict: Configs) -> None: def local_only_configs_are_none(config_dict: Configs) -> list[bool]: + """Check whether `central_path` and `connection_method` are both set to None.""" return [ config_dict[key] is None for key in ["central_path", "connection_method"] @@ -169,14 +164,10 @@ def raise_on_bad_path_syntax( path_name: str, path_type: str, ) -> None: - """ - Error if some common, unsupported patterns are observed - (e.g. ~, .) for path. - """ + """Raise error if path contains unsupported patterns (e.g. ~, .).""" if path_name[0] == "~": utils.log_and_raise_error( - f"{path_type} must contain the full folder path " - "with no ~ syntax.", + f"{path_type} must contain the full folder path with no ~ syntax.", ConfigError, ) @@ -191,13 +182,10 @@ def raise_on_bad_path_syntax( def check_config_types(config_dict: Configs) -> None: - """ - Check the type of passed configs matches the canonical types. - """ + """Check the type of passed configs matches the canonical types.""" required_types = get_canonical_configs() for key in config_dict.keys(): - expected_type = required_types[key] try: typeguard.check_type(config_dict[key], expected_type) @@ -216,14 +204,12 @@ def check_config_types(config_dict: Configs) -> None: def get_tui_config_defaults() -> Dict: - """ - Get the default settings for the datatype checkboxes - in the TUI. + """Return the default settings for the datatype checkboxes in the TUI. - Two sets are maintained (one for creating, - one for transfer) which have different defaults. + Two sets are maintained (one for checkboxes on the create tab, + the other for transfer tab) which have different defaults. By default, all broad datatype checkboxes are displayed, - and narrow are turned off. + and narrow datatypes are hidden and turned off. """ settings = { "tui": { @@ -247,7 +233,6 @@ def get_tui_config_defaults() -> Dict: # Fill all datatype options for broad_key in get_broad_datatypes(): - settings["tui"]["create_checkboxes_on"][broad_key] = { # type: ignore "on": True, "displayed": True, @@ -271,18 +256,16 @@ def get_tui_config_defaults() -> Dict: def get_name_templates_defaults() -> Dict: + """Return the default values for name_templates.""" return {"name_templates": {"on": False, "sub": None, "ses": None}} def get_persistent_settings_defaults() -> Dict: - """ - Persistent settings are settings that are maintained - across sessions. Currently, persistent settings for - both the API and TUI are stored in the same place. + """Return the default persistent settings maintained across sessions. - Currently, settings for the working top level folder, - TUI checkboxes and name templates (i.e. regexp - validation for sub and ses names) are stored. + Currently, these include settings for both the API and TUI, such as the + working top level folder, TUI checkboxes, and name templates + (i.e. regexp validation for sub and ses names). """ settings = {} settings.update(get_tui_config_defaults()) @@ -292,8 +275,7 @@ def get_persistent_settings_defaults() -> Dict: def get_datatypes() -> List[str]: - """ - Canonical list of datatype flags based on NeuroBlueprint. + """Return canonical list of datatype flags based on NeuroBlueprint. This must be kept up to date with the datatypes in the NeuroBlueprint specification. """ @@ -301,12 +283,13 @@ def get_datatypes() -> List[str]: def get_broad_datatypes(): + """Return a list of broad datatypes.""" return ["ephys", "behav", "funcimg", "anat"] def get_narrow_datatypes(): - """ - Return the narrow datatype associated with each broad datatype. + """Return the narrow datatype associated with each broad datatype. + The mapping between broad and narrow datatypes is required for validation. """ return { @@ -337,9 +320,9 @@ def get_narrow_datatypes(): def quick_get_narrow_datatypes(): - """ - A convenience wrapper around `get_narrow_datatypes()` - to quickly get a list of all narrow datatypes. + """Return a flat list of all narrow datatypes. + + This is a convenience wrapper around `get_narrow_datatypes()`. """ all_narrow_datatypes = get_narrow_datatypes() top_level_keys = list(all_narrow_datatypes.keys()) @@ -352,7 +335,8 @@ def quick_get_narrow_datatypes(): def in_place_update_narrow_datatypes_if_required(user_settings: dict): - """ + """Update legacy settings with the new version format. + In versions < v0.6.0, only 'broad' datatypes were implemented and available in the TUI. Since, 'narrow' datatypes are introduced and datatype tui can be set to be both on / off but also @@ -389,7 +373,9 @@ def in_place_update_narrow_datatypes_if_required(user_settings: dict): dtype in user_settings["tui"]["transfer_checkboxes_on"] for dtype in all_narrow_datatypes ] - ), "Somehow there are datatypes missing in `transfer_checkboxes_on` but not `create_checkboxes_on`" + ), ( + "Somehow there are datatypes missing in `transfer_checkboxes_on` but not `create_checkboxes_on`" + ) if has_narrow_datatypes and is_not_missing_any_narrow_datatypes: return @@ -421,15 +407,12 @@ def in_place_update_narrow_datatypes_if_required(user_settings: dict): # Copy any datatype information that exists. Broad datatypes will all be there # but some narrow datatypes might be missing. for checkbox_type in ["create_checkboxes_on", "transfer_checkboxes_on"]: - datatypes_that_user_has = list( user_settings["tui"][checkbox_type].keys() ) for dtype in get_datatypes(): - if dtype in datatypes_that_user_has: - if has_narrow_datatypes: new_checkbox_configs[checkbox_type][dtype] = user_settings[ "tui" diff --git a/datashuttle/configs/canonical_folders.py b/datashuttle/configs/canonical_folders.py index a6b77128c..da4c92a65 100644 --- a/datashuttle/configs/canonical_folders.py +++ b/datashuttle/configs/canonical_folders.py @@ -11,9 +11,7 @@ def get_datatype_folders() -> dict: - """ - This function holds the canonical folders - managed by datashuttle. + """Return the canonical folders managed by datashuttle. Notes ----- @@ -24,14 +22,19 @@ def get_datatype_folders() -> dict: kept in case this changes. The value is a Folder() class instance with - the required fields + the required fields. - name : The display name for the datatype, that will + Parameters + ---------- + name + The display name for the datatype, that will be used for making and transferring files in practice. This should always match the canonical name, but left as an option for rare cases in which advanced users want to change it. - level : "sub" or "ses", level to make the folder at. + level + "sub" or "ses", level to make the folder at. + """ return { datatype: Folder(name=datatype, level="ses") @@ -40,9 +43,9 @@ def get_datatype_folders() -> dict: def get_non_sub_names() -> List[str]: - """ - Get all arguments that are not allowed at the - subject level for data transfer, i.e. as sub_names + """Return all arguments that are not allowed at the subject level. + + These are invalid as `sub_names` for data transfer. """ return [ "all_ses", @@ -53,10 +56,7 @@ def get_non_sub_names() -> List[str]: def get_non_ses_names() -> List[str]: - """ - Get all arguments that are not allowed at the - session level for data transfer, i.e. as ses_names - """ + """Return all arguments that are not allowed at the session level.""" return [ "all_sub", "all_non_sub", @@ -66,34 +66,26 @@ def get_non_ses_names() -> List[str]: def canonical_reserved_keywords() -> List[str]: - """ - Key keyword arguments that are passed to `sub_names` or - `ses_names` but that we - """ + """Return key keyword arguments passed to `sub_names` or `ses_names`.""" return get_non_sub_names() + get_non_ses_names() def get_top_level_folders() -> List[TopLevelFolder]: + """Return a list of canonical top level folder names.""" return ["rawdata", "derivatives"] def get_datashuttle_path() -> Path: - """ - Get the datashuttle path where all project - configs are stored. - """ + """Return the datashuttle path where all project configs are stored.""" return Path.home() / ".datashuttle" def get_project_datashuttle_path(project_name: str) -> Tuple[Path, Path]: - """ - Get the datashuttle path for the project, - where configuration files are stored. - Also, return a temporary path in this for logging in - some cases where local_path location is not clear. + """Return the datashuttle config path for the project. - The datashuttle configuration path is stored in the user home - folder. + Also returns a temporary logging path used in cases when + the `local_path` is not yet defined. The base configuration + path is the user home directory. """ base_path = get_datashuttle_path() / project_name temp_logs_path = base_path / "temp_logs" diff --git a/datashuttle/configs/canonical_tags.py b/datashuttle/configs/canonical_tags.py index 233350bc6..ee7295665 100644 --- a/datashuttle/configs/canonical_tags.py +++ b/datashuttle/configs/canonical_tags.py @@ -1,9 +1,7 @@ def tags(tag_name: str) -> str: - """ - Centralised function to get the tags used - in subject / session name processing. If changing - the formatting of these tags, it is only required - to change the dict values here. + """Return the formatting tag used for subject/session name parsing. + + If changing the formatting of these tags, update the dict values here. """ tags = { "date": "@DATE@", diff --git a/datashuttle/configs/config_class.py b/datashuttle/configs/config_class.py index 562c3310b..4d216576b 100644 --- a/datashuttle/configs/config_class.py +++ b/datashuttle/configs/config_class.py @@ -25,32 +25,36 @@ class Configs(UserDict): - """ - Class to hold the datashuttle configs. + """Store and manage datashuttle configuration settings. The configs must match exactly the standard set in canonical_configs.py. If updating these configs, this should be done through changing canonical_configs.py + """ - The input dict is checked that it conforms to the - canonical standard by calling check_dict_values_raise_on_fail() + def __init__( + self, project_name: str, file_path: Path, input_dict: Union[dict, None] + ) -> None: + """Initialize the Configs class with project name, file path, and config dictionary. - project_name and all paths are set at runtime but not stored. + Parameters + ---------- + project_name + Name of the datashuttle project. - Parameters - ---------- + file_path + full filepath to save the config .yaml file to. - file_path : - full filepath to save the config .yaml file to. + input_dict + a dict of config key-value pairs to input dict. + This must contain all canonical_config keys - input_dict : - a dict of config key-value pairs to input dict. - This must contain all canonical_config keys - """ + The input dict is checked that it conforms to the + canonical standard by calling check_dict_values_raise_on_fail() - def __init__( - self, project_name: str, file_path: Path, input_dict: Union[dict, None] - ) -> None: + project_name and all paths are set at runtime but not stored. + + """ super(Configs, self).__init__(input_dict) self.project_name = project_name @@ -62,12 +66,13 @@ def __init__( self.project_metadata_path: Path def setup_after_load(self) -> None: + """Set up the config after loading it.""" load_configs.convert_str_and_pathlib_paths(self, "str_to_path") self.ensure_local_and_central_path_end_in_project_name() self.check_dict_values_raise_on_fail() - def ensure_local_and_central_path_end_in_project_name(self): - """""" + def ensure_local_and_central_path_end_in_project_name(self) -> None: + """Ensure that the local and central path end in the name of the project.""" for path_type in ["local_path", "central_path"]: if path_type == "central_path" and self[path_type] is None: continue @@ -81,9 +86,7 @@ def ensure_local_and_central_path_end_in_project_name(self): self[path_type] = self[path_type] / self.project_name def check_dict_values_raise_on_fail(self) -> None: - """ - Check the values of the current dictionary are set - correctly and will not cause downstream errors. + """Validate dictionary values against canonical config requirements. This will raise an error if the dictionary does not match the canonical keys and value types. @@ -91,12 +94,15 @@ def check_dict_values_raise_on_fail(self) -> None: canonical_configs.check_dict_values_raise_on_fail(self) def keys(self) -> KeysView: + """Return D.keys(), a set-like object providing a view on D's keys.""" return self.data.keys() def items(self) -> ItemsView: + """Return D.items(), a set-like object providing a view on D's items.""" return self.data.items() def values(self) -> ValuesView: + """Return D.values(), a set-like object providing a view on D's values.""" return self.data.values() # ------------------------------------------------------------------------- @@ -104,9 +110,7 @@ def values(self) -> ValuesView: # ------------------------------------------------------------------------- def dump_to_file(self) -> None: - """ - Save the dictionary to .yaml file stored in self.file_path. - """ + """Save the dictionary to .yaml file stored in self.file_path.""" cfg_to_save = copy.deepcopy(self.data) load_configs.convert_str_and_pathlib_paths(cfg_to_save, "path_to_str") @@ -114,12 +118,12 @@ def dump_to_file(self) -> None: yaml.dump(cfg_to_save, config_file, sort_keys=False) def load_from_file(self) -> None: + """Load a config dict saved at .yaml file. + + Note this will not automatically check the configs are valid, + this requires calling self.check_dict_values_raise_on_fail(). """ - Load a config dict saved at .yaml file. Note this will - not automatically check the configs are valid, this - requires calling self.check_dict_values_raise_on_fail() - """ - with open(self.file_path, "r") as config_file: + with open(self.file_path) as config_file: config_dict = yaml.full_load(config_file) load_configs.convert_str_and_pathlib_paths(config_dict, "str_to_path") @@ -136,19 +140,28 @@ def build_project_path( sub_folders: Union[str, list], top_level_folder: TopLevelFolder, ) -> Path: - """ - Function for joining relative path to base dir. - If path already starts with base dir, the base - dir will not be joined. + """Build a path by joining a base directory with subfolders. + + If the path already starts with the base directory, + the base will not be joined again. Parameters ---------- + base + "local", "central" or "datashuttle" - base: "local", "central" or "datashuttle" - - sub_folders: a list (or string for 1) of + sub_folders + a list (or string for 1) of folder names to be joined into a path. If file included, must be last entry (with ext). + + top_level_folder + either "rawdata" or "derivatives" + + Returns + ------- + The full path to the `sub_folders` in the project. + """ if isinstance(sub_folders, list): sub_folders_str = "/".join(sub_folders) @@ -171,13 +184,19 @@ def get_base_folder( base: str, top_level_folder: TopLevelFolder, ) -> Path: - """ - Convenience function to return the full base path. + """Return the full base path for the given top-level folder. Parameters ---------- + base + Base path, "local", "central" or "datashuttle". - base : base path, "local", "central" or "datashuttle" + top_level_folder + Either "rawdata" or "derivatives". + + Returns + ------- + Full path to the local or central project top level folder. """ if base == "local": @@ -190,10 +209,9 @@ def get_base_folder( def get_rclone_config_name( self, connection_method: Optional[str] = None ) -> str: - """ - Convenience function to get the rclone config - name (these configs are created by datashuttle - but managed and stored by rclone). + """Generate the rclone configuration name for the project. + + These configs are created by datashuttle but managed and stored by rclone. """ if connection_method is None: connection_method = self["connection_method"] @@ -203,11 +221,11 @@ def get_rclone_config_name( def make_rclone_transfer_options( self, overwrite_existing_files: OverwriteExistingFiles, dry_run: bool ) -> Dict: - """ - This function originally collected the relevant arguments - from configs. Now, all are passed via function arguments - However, now we fix the previously configurable arguments - `show_transfer_progress` and `dry_run` here. + """Create a dictionary of rclone transfer options. + + Originally these arguments were collected from configs, but now + they are passed via function arguments. The `show_transfer_progress` + and `dry_run` options are fixed here. """ allowed_overwrite = ["never", "always", "if_source_newer"] @@ -227,7 +245,7 @@ def make_rclone_transfer_options( } def init_paths(self) -> None: - """""" + """Initialize paths used by datashuttle.""" self.project_metadata_path = self["local_path"] / ".datashuttle" datashuttle_path, _ = canonical_folders.get_project_datashuttle_path( @@ -243,9 +261,9 @@ def init_paths(self) -> None: def make_and_get_logging_path( self, ) -> Path: - """ - Build (and create if does not exist) the path where - logs are stored. + """Build and return the path where logs are stored. + + Create the directory if it does not already exist. """ logging_path = self.project_metadata_path / "logs" folders.create_folders(logging_path) @@ -254,9 +272,9 @@ def make_and_get_logging_path( def get_datatype_as_dict_items( self, datatype: Union[str, list] ) -> Union[ItemsView, zip]: - """ - Get the .items() structure of the datatype, either all of - the canonical datatypes or as a single item. + """Return canonical datatypes as dictionary items. + + Returns all datatype items or a subset if specified. """ if isinstance(datatype, str): datatype = [datatype] @@ -276,7 +294,8 @@ def get_datatype_as_dict_items( return items def is_local_project(self): - """ + """Return bool indicating if project is a local-only project. + A project is 'local-only' if it has no `central_path` and `connection_method`. It can be used to make folders and validate, but not for transfer. """ diff --git a/datashuttle/configs/links.py b/datashuttle/configs/links.py index 0c354bc1a..2daa03013 100644 --- a/datashuttle/configs/links.py +++ b/datashuttle/configs/links.py @@ -1,14 +1,18 @@ def get_docs_link(): + """Return the link to the datashuttle page.""" return "https://datashuttle.neuroinformatics.dev/" def get_github_link(): + """Return the link to the datashuttle repository.""" return "https://github.com/neuroinformatics-unit/datashuttle" def get_link_github_issues(): + """Return the link to the datashuttle repository issues page.""" return "https://github.com/neuroinformatics-unit/datashuttle/issues" def get_link_zulip(): + """Return the link to the datashuttle Zulip chatroom.""" return "https://neuroinformatics.zulipchat.com/#narrow/stream/405999-DataShuttle" diff --git a/datashuttle/configs/load_configs.py b/datashuttle/configs/load_configs.py index aac1bc93d..65b46ba00 100644 --- a/datashuttle/configs/load_configs.py +++ b/datashuttle/configs/load_configs.py @@ -17,19 +17,26 @@ def attempt_load_configs( config_path: Path, verbose: bool = True, ) -> Optional[Configs]: - """ - Try to load an existing config file, that was previously - saved by Datashuttle. This should always work, unless - not already initialised (prompt) or these have been - changed manually. + """Try to load an existing config file previously saved by Datashuttle. + + This should always work unless the config is not initialized or has been + manually changed. Parameters ---------- - project_name : name of project + project_name + Name of the project. + + config_path + Path to the datashuttle config .yaml file. + + verbose + If True, warnings and error messages will be printed. - config_path : path to datashuttle config .yaml file + Returns + ------- + The loaded config, or `None` if it could not be loaded. - verbose : warnings and error messages will be printed. """ exists = config_path.is_file() @@ -65,15 +72,19 @@ def attempt_load_configs( def convert_str_and_pathlib_paths( config_dict: Union["Configs", dict], direction: str ) -> None: - """ - Config paths are stored as str in the .yaml but used as Path - in the module, so make the conversion here. + """Convert config paths between strings and pathlib.Path objects. + + Paths are stored as strings in the .yaml file but used as Path objects in + the module. This function performs the conversion. Parameters ---------- + config_dict + DataShuttle.cfg dict of configs. + + direction + Direction of conversion: "path_to_str" or "str_to_path". - config_dict : DataShuttle.cfg dict of configs - direction : "path_to_str" or "str_to_path" """ for path_key in canonical_configs.keys_str_on_file_but_path_in_class(): value = config_dict[path_key] diff --git a/datashuttle/datashuttle_class.py b/datashuttle/datashuttle_class.py index 9bc874b34..1b7f4e268 100644 --- a/datashuttle/datashuttle_class.py +++ b/datashuttle/datashuttle_class.py @@ -62,49 +62,22 @@ class DataShuttle: - """ - DataShuttle is a tool for convenient scientific - project management and data transfer in BIDS format. - - The expected organisation is a central repository - on a central machine ('central') that contains all - project data. This is connected to multiple local - machines ('local'). These can each contain a subset of - the full project (e.g. machine for electrophysiology - collection, machine for behavioural collection). - - On first use on a new profile, show warning prompting - to set configurations with the function make_config_file(). - - Datashuttle will save logs to a .datashuttle folder - in the main local project. These logs contain - detailed information on folder creation / transfer. - To get the path to datashuttle logs, use - cfgs.make_and_get_logging_path(). - - For transferring data between a central data storage - with SSH, use setup setup_ssh_connection(). - This will allow you to check the server key, add host key to - profile if accepted, and setup ssh key pair. - - Parameters - ---------- - - project_name : The project name to use the datashuttle - Folders containing all project files - and folders are specified in make_config_file(). - Datashuttle-related files are stored in - a .datashuttle folder in the user home - folder. Use get_datashuttle_path() to - see the path to this folder. - - print_startup_message : If `True`, a start-up message displaying the - current state of the program (e.g. persistent - settings such as the 'top-level folder') is shown. - """ + """DataShuttle is a tool for neuroscience project management and data transfer.""" def __init__(self, project_name: str, print_startup_message: bool = True): + """Initialise ``DataShuttle``. + Parameters + ---------- + project_name + The project name. + + print_startup_message + If `True`, a start-up message displaying the + current state of the program (e.g. persistent + settings such as the 'top-level folder') is shown. + + """ self._error_on_base_project_name(project_name) self.project_name = project_name ( @@ -131,10 +104,7 @@ def __init__(self, project_name: str, print_startup_message: bool = True): rclone.prompt_rclone_download_if_does_not_exist() def _set_attributes_after_config_load(self) -> None: - """ - Once config file is loaded, update all private attributes - according to config contents. - """ + """Update all private attributes according to config contents.""" self.cfg.init_paths() self._make_project_metadata_if_does_not_exist() @@ -153,50 +123,45 @@ def create_folders( bypass_validation: bool = False, log: bool = True, ) -> Dict[str, List[Path]]: - """ - Create a subject / session folder tree in the project - folder. The passed subject / session names are - formatted and validated. If this succeeds, fully - validation against all subject / session folders in - the local project is performed before making the - folders. + """Create a folder tree in the project folder. + + The passed names are initially formatted and validated, + then folders are created. Parameters ---------- - - top_level_folder : TopLevelFolder - Whether to make the folders in `rawdata` or + top_level_folder + Whether to make the folders within `rawdata` or `derivatives`. - sub_names : Union[str, List[str]] + sub_names subject name / list of subject names to make within the top-level project folder (if not already, these will be prefixed with "sub-") - ses_names : Optional[Union[str, List[str]]] - (Optional). session name / list of session names. + ses_names + session name / list of session names. (if not already, these will be prefixed with "ses-"). If no session is provided, no session-level folders are made. - datatype : Union[str, List[str]] + datatype The datatype to make in the sub / ses folders. (e.g. "ephys", "behav", "anat"). If "" is passed no datatype will be created. Broad or - Narrow canonical NeuroBlueprint datatypes are - accepted. + Narrow NeuroBlueprint datatypes are accepted. - bypass_validation : bool + bypass_validation If `True`, folders will be created even if they are not valid to NeuroBlueprint style. - log : bool + log If `True`, details of folder creation will be logged. Returns ------- - created_paths : + created_paths A dictionary of the full filepaths made during folder creation, where the keys are the type of folder made and the values are a list of created folder paths (Path objects). If datatype were @@ -206,7 +171,6 @@ def create_folders( Notes ----- - sub_names or ses_names may contain formatting tags @TO@ @@ -225,6 +189,7 @@ def create_folders( project.create_folders("rawdata", "sub-001", datatype="behav") project.create_folders("rawdata", "sub-002@TO@005", ["ses-001", "ses-002"], ["ephys", "behav"]) + """ if log: self._start_log("create-folders", local_vars=locals()) @@ -287,11 +252,7 @@ def _format_and_validate_names( bypass_validation: bool, log: bool = True, ) -> Tuple[List[str], List[str]]: - """ - A central method for the formatting and validation of subject / session - names for folder creation. This is called by both DataShuttle and - during TUI validation. - """ + """Central method to format and validate subject and session names.""" format_sub = formatting.check_and_format_names( sub_names, "sub", name_templates, bypass_validation ) @@ -333,50 +294,44 @@ def upload_custom( dry_run: bool = False, init_log: bool = True, ) -> None: - """ - Upload data from a local project to the central project - folder. In the case that a file / folder exists on - the central and local, the central will not be overwritten - even if the central file is an older version. Data - transfer logs are saved to the logging folder). + """Upload data from a local project to the central project folder. Parameters ---------- + top_level_folder + The top-level folder (e.g. `"rawdata"`, `"derivatives"`) to transfer within. - top_level_folder : - The top-level folder (e.g. `"rawdata"`, `"derivatives"`) to transfer files - and folders within. - - sub_names : - a subject name / list of subject names. These must - be prefixed with "sub-", or the prefix will be - automatically added. "@*@" can be used as a wildcard. + sub_names + A subject name / list of subject names. These must + be prefixed with ``"sub-"``, or the prefix will be + automatically added. ``"@*@"`` can be used as a wildcard. "all" will search for all sub-folders in the datatype folder to upload. - ses_names : - a session name / list of session names, similar to - sub_names but requiring a "ses-" prefix. + ses_names + A session name / list of session names, similar to + sub_names but requiring a ``"ses-"`` prefix. - datatype : + datatype The (broad or narrow) NeuroBlueprint datatypes to transfer. - If "all", any broad or narrow datatype folder will be transferred. - - overwrite_existing_files : - If `False`, files on central will never be overwritten - by files transferred from local. If `True`, central files - will be overwritten if there is any difference (date, size) - between central and local files. - - dry_run : - perform a dry-run of transfer. This will output as if file - transfer was taking place, but no files will be moved. Useful - to check which files will be moved on data transfer. - - init_log : - (Optional). Whether to handle logging. This should - always be True, unless logger is handled elsewhere + If ``"all"``, any broad or narrow datatype folder will be transferred. + + overwrite_existing_files + If ``"never"`` files on target will never be overwritten by source. + If ``"always"`` files on target will be overwritten by source if + there is any difference in date or size. + If ``"if_source_newer"`` files on target will only be overwritten + by files on source with newer creation / modification datetime. + + dry_run + Perform a dry-run of transfer. This will output as if file + transfer was taking place, but no files will be moved. + + init_log + Whether to handle logging. This should + always be ``True``, unless logger is handled elsewhere (e.g. in a calling function). + """ if init_log: self._start_log("upload-custom", local_vars=locals()) @@ -410,47 +365,44 @@ def download_custom( dry_run: bool = False, init_log: bool = True, ) -> None: - """ - Download data from the central project folder to the - local project folder. + """Download data from the central project to the local project folder. Parameters ---------- + top_level_folder + The top-level folder (e.g. `"rawdata"`, `"derivatives"`) to transfer within. - top_level_folder : - The top-level folder (e.g. `rawdata`) to transfer files - and folders within. - - sub_names : - a subject name / list of subject names. These must - be prefixed with "sub-", or the prefix will be - automatically added. "@*@" can be used as a wildcard. + sub_names + A subject name / list of subject names. These must + be prefixed with ``"sub-"``, or the prefix will be + automatically added. ``"@*@"`` can be used as a wildcard. "all" will search for all sub-folders in the datatype folder to upload. - ses_names : - a session name / list of session names, similar to - sub_names but requiring a "ses-" prefix. + ses_names + A session name / list of session names, similar to + sub_names but requiring a ``"ses-"`` prefix. - datatype : - see create_folders() + datatype + The (broad or narrow) NeuroBlueprint datatypes to transfer. + If ``"all"``, any broad or narrow datatype folder will be transferred. - overwrite_existing_files : - If "never" files on target will never be overwritten by source. - If "always" files on target will be overwritten by source if + overwrite_existing_files + If ``"never"`` files on target will never be overwritten by source. + If ``"always"`` files on target will be overwritten by source if there is any difference in date or size. - If "if_source_newer" files on target will only be overwritten + If ``"if_source_newer"`` files on target will only be overwritten by files on source with newer creation / modification datetime. - dry_run : - perform a dry-run of transfer. This will output as if file - transfer was taking place, but no files will be moved. Useful - to check which files will be moved on data transfer. + dry_run + Perform a dry-run of transfer. This will output as if file + transfer was taking place, but no files will be moved. - init_log : - (Optional). Whether to handle logging. This should - always be True, unless logger is handled elsewhere + init_log + Whether to handle logging. This should + always be ``True``, unless logger is handled elsewhere (e.g. in a calling function). + """ if init_log: self._start_log("download-custom", local_vars=locals()) @@ -483,24 +435,22 @@ def upload_rawdata( self, overwrite_existing_files: OverwriteExistingFiles = "never", dry_run: bool = False, - ): - """ - Upload files in the `rawdata` top level folder. + ) -> None: + """Upload all files in the `rawdata` top level folder. Parameters ---------- - - overwrite_existing_files : - If "never" files on target will never be overwritten by source. - If "always" files on target will be overwritten by source if + overwrite_existing_files + If ``"never"`` files on target will never be overwritten by source. + If ``"always"`` files on target will be overwritten by source if there is any difference in date or size. - If "if_source_newer" files on target will only be overwritten + If ``"if_source_newer"`` files on target will only be overwritten by files on source with newer creation / modification datetime. - dry_run : - perform a dry-run of transfer. This will output as if file - transfer was taking place, but no files will be moved. Useful - to check which files will be moved on data transfer. + dry_run + Perform a dry-run of transfer. This will output as if file + transfer was taking place, but no files will be moved. + """ self._transfer_top_level_folder( "upload", @@ -515,24 +465,22 @@ def upload_derivatives( self, overwrite_existing_files: OverwriteExistingFiles = "never", dry_run: bool = False, - ): - """ - Upload files in the `derivatives` top level folder. + ) -> None: + """Upload all files in the `derivatives` top level folder. Parameters ---------- - - overwrite_existing_files : - If "never" files on target will never be overwritten by source. - If "always" files on target will be overwritten by source if + overwrite_existing_files + If ``"never"`` files on target will never be overwritten by source. + If ``"always"`` files on target will be overwritten by source if there is any difference in date or size. - If "if_source_newer" files on target will only be overwritten + If ``"if_source_newer"`` files on target will only be overwritten by files on source with newer creation / modification datetime. - dry_run : - perform a dry-run of transfer. This will output as if file - transfer was taking place, but no files will be moved. Useful - to check which files will be moved on data transfer. + dry_run + Perform a dry-run of transfer. This will output as if file + transfer was taking place, but no files will be moved. + """ self._transfer_top_level_folder( "upload", @@ -547,24 +495,22 @@ def download_rawdata( self, overwrite_existing_files: OverwriteExistingFiles = "never", dry_run: bool = False, - ): - """ - Download files in the `rawdata` top level folder. + ) -> None: + """Download all files in the `rawdata` top level folder. Parameters ---------- - - overwrite_existing_files : - If "never" files on target will never be overwritten by source. - If "always" files on target will be overwritten by source if + overwrite_existing_files + If ``"never"`` files on target will never be overwritten by source. + If ``"always"`` files on target will be overwritten by source if there is any difference in date or size. - If "if_source_newer" files on target will only be overwritten + If ``"if_source_newer"`` files on target will only be overwritten by files on source with newer creation / modification datetime. - dry_run : - perform a dry-run of transfer. This will output as if file - transfer was taking place, but no files will be moved. Useful - to check which files will be moved on data transfer. + dry_run + Perform a dry-run of transfer. This will output as if file + transfer was taking place, but no files will be moved.. + """ self._transfer_top_level_folder( "download", @@ -579,24 +525,22 @@ def download_derivatives( self, overwrite_existing_files: OverwriteExistingFiles = "never", dry_run: bool = False, - ): - """ - Download files in the `derivatives` top level folder. + ) -> None: + """Download all files in the `derivatives` top level folder. Parameters ---------- - - overwrite_existing_files : - If "never" files on target will never be overwritten by source. - If "always" files on target will be overwritten by source if + overwrite_existing_files + If ``"never"`` files on target will never be overwritten by source. + If ``"always"`` files on target will be overwritten by source if there is any difference in date or size. - If "if_source_newer" files on target will only be overwritten + If ``"if_source_newer"`` files on target will only be overwritten by files on source with newer creation / modification datetime. - dry_run : - perform a dry-run of transfer. This will output as if file - transfer was taking place, but no files will be moved. Useful - to check which files will be moved on data transfer. + dry_run + Perform a dry-run of transfer. This will output as if file + transfer was taking place, but no files will be moved. + """ self._transfer_top_level_folder( "download", @@ -612,25 +556,23 @@ def upload_entire_project( overwrite_existing_files: OverwriteExistingFiles = "never", dry_run: bool = False, ) -> None: - """ - Upload the entire project (from 'local' to 'central'), - i.e. including every top level folder (e.g. 'rawdata', - 'derivatives', 'code', 'analysis'). + """Upload the entire project. + + Includes every top level folder (e.g. ``rawdata``, ``derivatives``). Parameters ---------- - - overwrite_existing_files : - If "never" files on target will never be overwritten by source. - If "always" files on target will be overwritten by source if + overwrite_existing_files + If ``"never"`` files on target will never be overwritten by source. + If ``"always"`` files on target will be overwritten by source if there is any difference in date or size. - If "if_source_newer" files on target will only be overwritten + If ``"if_source_newer"`` files on target will only be overwritten by files on source with newer creation / modification datetime. - dry_run : - perform a dry-run of transfer. This will output as if file - transfer was taking place, but no files will be moved. Useful - to check which files will be moved on data transfer. + dry_run + Perform a dry-run of transfer. This will output as if file + transfer was taking place, but no files will be moved. + """ self._start_log("upload-entire-project", local_vars=locals()) self._transfer_entire_project( @@ -645,25 +587,23 @@ def download_entire_project( overwrite_existing_files: OverwriteExistingFiles = "never", dry_run: bool = False, ) -> None: - """ - Download the entire project (from 'central' to 'local'), - i.e. including every top level folder (e.g. 'rawdata', - 'derivatives', 'code', 'analysis'). + """Download the entire project. + + Includes every top level folder (e.g. ``rawdata``, ``derivatives``). Parameters ---------- - - overwrite_existing_files : - If "never" files on target will never be overwritten by source. - If "always" files on target will be overwritten by source if + overwrite_existing_files + If ``"never"`` files on target will never be overwritten by source. + If ``"always"`` files on target will be overwritten by source if there is any difference in date or size. - If "if_source_newer" files on target will only be overwritten + If ``"if_source_newer"`` files on target will only be overwritten by files on source with newer creation / modification datetime. - dry_run : - perform a dry-run of transfer. This will output as if file - transfer was taking place, but no files will be moved. Useful - to check which files will be moved on data transfer. + dry_run + Perform a dry-run of transfer. This will output as if file + transfer was taking place, but no files will be moved. + """ self._start_log("download-entire-project", local_vars=locals()) self._transfer_entire_project( @@ -679,31 +619,28 @@ def upload_specific_folder_or_file( overwrite_existing_files: OverwriteExistingFiles = "never", dry_run: bool = False, ) -> None: - """ - Upload a specific file or folder. If transferring - a single file, the path including the filename is - required (see 'filepath' input). If a folder, - wildcards "*" or "**" must be used to transfer - all files in the folder ("*") or all files - and sub-folders ("**"). + """Upload a specific file or folder. + + If transferring a single file, the path including the filename is + required (see 'filepath' input). If a folder, wildcards "*" or "**" must be used to transfer + all files in the folder ("*") or all files and sub-folders ("**"). Parameters ---------- - - filepath : + filepath a string containing the full filepath. - overwrite_existing_files : - If "never" files on target will never be overwritten by source. - If "always" files on target will be overwritten by source if + overwrite_existing_files + If ``"never"`` files on target will never be overwritten by source. + If ``"always"`` files on target will be overwritten by source if there is any difference in date or size. - If "if_source_newer" files on target will only be overwritten + If ``"if_source_newer"`` files on target will only be overwritten by files on source with newer creation / modification datetime. - dry_run : - perform a dry-run of transfer. This will output as if file - transfer was taking place, but no files will be moved. Useful - to check which files will be moved on data transfer. + dry_run + Perform a dry-run of transfer. This will output as if file + transfer was taking place, but no files will be moved. + """ self._start_log("upload-specific-folder-or-file", local_vars=locals()) @@ -721,31 +658,29 @@ def download_specific_folder_or_file( overwrite_existing_files: OverwriteExistingFiles = "never", dry_run: bool = False, ) -> None: - """ - Download a specific file or folder. If transferring - a single file, the path including the filename is - required (see 'filepath' input). If a folder, - wildcards "*" or "**" must be used to transfer - all files in the folder ("*") or all files + """Download a specific file or folder. + + If transferring a single file, the path including the filename is + required (see 'filepath' input). If a folder, wildcards "*" or "**" + must be used to transfer all files in the folder ("*") or all files and sub-folders ("**"). Parameters ---------- - - filepath : + filepath a string containing the full filepath. - overwrite_existing_files : - If "never" files on target will never be overwritten by source. - If "always" files on target will be overwritten by source if + overwrite_existing_files + If ``"never"`` files on target will never be overwritten by source. + If ``"always"`` files on target will be overwritten by source if there is any difference in date or size. - If "if_source_newer" files on target will only be overwritten + If ``"if_source_newer"`` files on target will only be overwritten by files on source with newer creation / modification datetime. - dry_run : - perform a dry-run of transfer. This will output as if file - transfer was taking place, but no files will be moved. Useful - to check which files will be moved on data transfer. + dry_run + Perform a dry-run of transfer. This will output as if file + transfer was taking place, but no files will be moved. + """ self._start_log( "download-specific-folder-or-file", local_vars=locals() @@ -764,10 +699,11 @@ def _transfer_top_level_folder( overwrite_existing_files: OverwriteExistingFiles = "never", dry_run: bool = False, init_log: bool = True, - ): - """ - Core function to upload / download files within a - particular top-level-folder. e.g. `upload_rawdata().` + ) -> None: + """Upload or download files within a particular top-level-folder. + + A centralised function to upload or download data within + a particular top level folder (e.g. ``rawdata``, ``derivatives``). """ if init_log: self._start_log( @@ -795,10 +731,8 @@ def _transfer_top_level_folder( def _transfer_specific_file_or_folder( self, upload_or_download, filepath, overwrite_existing_files, dry_run - ): - """ - Core function for upload/download_specific_folder_or_file(). - """ + ) -> None: + """Core function for upload/download_specific_folder_or_file().""" if isinstance(filepath, str): filepath = Path(filepath) @@ -839,11 +773,10 @@ def _transfer_specific_file_or_folder( @requires_ssh_configs @check_is_not_local_project def setup_ssh_connection(self) -> None: - """ - Setup a connection to the central server using SSH. - Assumes the central_host_id and central_host_username - are set in configs (see make_config_file() and update_config_file()) + """Set up a connection to the central server using SSH. + Assumes the central_host_id and central_host_username + are set in configs (see make_config_file() and update_config_file()). First, the server key will be displayed, requiring verification of the server ID. This will store the hostkey for all future use. @@ -871,17 +804,18 @@ def setup_ssh_connection(self) -> None: @requires_ssh_configs @check_is_not_local_project def write_public_key(self, filepath: str) -> None: - """ - By default, the SSH private key only is stored, in - the datashuttle configs folder. Use this function - to save the public key. + """Save the public SSH key to a specified filepath. + + By default, only the SSH private key is stored in the + datashuttle configs folder. Use this function to save + the public key. Parameters ---------- - - filepath : - full filepath (inc filename) to write the + filepath + Full filepath (including filename) to write the public key to. + """ key: paramiko.RSAKey key = paramiko.RSAKey.from_private_key_file( @@ -904,46 +838,45 @@ def make_config_file( central_host_id: Optional[str] = None, central_host_username: Optional[str] = None, ) -> None: - """ - Initialise the configurations for datashuttle to use on the - local machine. Once initialised, these settings will be - used each time the datashuttle is opened. This method - can also be used to completely overwrite existing configs. + """Initialize the configurations for datashuttle on the local machine. + + Once initialised, these settings will be used each + time the datashuttle is opened. These settings are stored in a config file on the - datashuttle path (not in the project folder) - on the local machine. Use get_config_path() to - get the full path to the saved config file. + datashuttle path (not in the project folder) on the + local machine. Use ``get_config_path()`` to get the + full path to the saved config file. - Use update_config_file() to selectively update settings. + Use ``update_config_file()`` to selectively update settings. Parameters ---------- - - local_path : + local_path path to project folder on local machine - central_path : + central_path Filepath to central project. - If this is local (i.e. connection_method = "local_filesystem"), + If this is local (i.e. ``connection_method = "local_filesystem"``), this is the full path on the local filesystem - Otherwise, if this is via ssh (i.e. connection method = "ssh"), + Otherwise, if this is via ssh (i.e. ``connection method = "ssh"``), this is the path to the project folder on central machine. This should be a full path to central folder i.e. this cannot include ~ home folder syntax, must contain the full path - (e.g. /nfs/nhome/live/jziminski) + (e.g. ``/nfs/nhome/live/jziminski``) - connection_method : + connection_method The method used to connect to the central project filesystem, - e.g. "local_filesystem" (e.g. mounted drive) or "ssh" + e.g. ``"local_filesystem"`` (e.g. mounted drive) or ``"ssh"`` - central_host_id : + central_host_id server address for central host for ssh connection - e.g. "ssh.swc.ucl.ac.uk" + e.g. ``"ssh.swc.ucl.ac.uk"`` - central_host_username : + central_host_username username for which to log in to central host. - e.g. "jziminski" + e.g. ``"jziminski"`` + """ self._start_log( "make-config-file", @@ -991,7 +924,7 @@ def make_config_file( ds_logger.close_log_filehandler() def update_config_file(self, **kwargs) -> None: - """ """ + """Update the configuration file.""" if not self.cfg: utils.log_and_raise_error( "Must have a config loaded before updating configs.", @@ -1019,49 +952,41 @@ def update_config_file(self, **kwargs) -> None: @check_configs_set def get_local_path(self) -> Path: - """ - Get the projects local path. - """ + """Return the projects local path.""" return self.cfg["local_path"] @check_configs_set @check_is_not_local_project def get_central_path(self) -> Path: - """ - Get the project central path. - """ + """Return the project central path.""" return self.cfg["central_path"] def get_datashuttle_path(self) -> Path: - """ - Get the path to the local datashuttle - folder where configs and other - datashuttle files are stored. + """Return the path to the local datashuttle folder. + + This is where configs and other datashuttle files are stored. """ return self._datashuttle_path @check_configs_set def get_config_path(self) -> Path: - """ - Get the full path to the DataShuttle config file. - """ + """Return the full path to the DataShuttle config file.""" return self._config_path @check_configs_set def get_configs(self) -> Configs: + """Return the datashuttle configs.""" return self.cfg @check_configs_set def get_logging_path(self) -> Path: - """ - Get the path where datashuttle logs are written. - """ + """Return the path where datashuttle logs are written.""" return self.cfg.logging_path @staticmethod def get_existing_projects() -> List[Path]: - """ - Get a list of existing project names found on the local machine. + """Return a list of existing project names found on the local machine. + This is based on project folders in the "home / .datashuttle" folder that contain valid config.yaml files. """ @@ -1074,9 +999,7 @@ def get_next_sub( return_with_prefix: bool = True, include_central: bool = False, ) -> str: - """ - Convenience function for `get_next_sub_or_ses` - to find the next subject number. + """Return the next subject number. Parameters ---------- @@ -1090,6 +1013,11 @@ def get_next_sub( If `False, only get names from `local_path`, otherwise from `local_path` and `central_path`. If in local-project mode, this flag is ignored. + + Returns + ------- + The next subject ID. + """ name_template = self.get_name_templates() name_template_regexp = ( @@ -1117,13 +1045,10 @@ def get_next_ses( return_with_prefix: bool = True, include_central: bool = False, ) -> str: - """ - Convenience function for get_next_sub_or_ses - to find the next session number. + """Return the next session number. Parameters ---------- - top_level_folder The top-level folder, "rawdata" or "derivatives". @@ -1134,9 +1059,14 @@ def get_next_ses( If `True`, return with the "ses-" prefix. include_central - If `False, only get names from `local_path`, otherwise from - `local_path` and `central_path`. If in local-project mode, + If ``False``, only get names from ``local_path``, otherwise from + ``local_path`` and ``central_path``. If in local-project mode, this flag is ignored. + + Returns + ------- + The next session ID. + """ name_template = self.get_name_templates() name_template_regexp = ( @@ -1158,8 +1088,9 @@ def get_next_ses( @check_configs_set def is_local_project(self) -> bool: - """ - A project is 'local-only' if it has no `central_path` and `connection_method`. + """Return a bool indicating whether the project is 'local only'. + + A project is 'local-only' if it has no ``central_path`` and ``connection_method``. It can be used to make folders and validate, but not for transfer. """ return self.cfg.is_local_project() @@ -1168,33 +1099,33 @@ def is_local_project(self) -> bool: # ------------------------------------------------------------------------- def get_name_templates(self) -> Dict: - """ - Get the regexp templates used for validation. If - the "on" key is set to `False`, template validation is not performed. + """Return the regexp templates used for validation. + + If the "on" key is set to `False`, template validation is not performed. Returns ------- - - name_templates : Dict + name_templates e.g. {"name_templates": {"on": False, "sub": None, "ses": None}} + """ settings = self._load_persistent_settings() return settings["name_templates"] def set_name_templates(self, new_name_templates: Dict) -> None: - """ - Update the persistent settings with new name templates. + """Update the persistent settings with new name templates. - Name templates are regexp for that, when name_templates["on"] is - set to `True`, "sub" and "ses" names are validated against + Name templates are regexp for that, when ``name_templates["on"]`` is + set to ``True``, ``"sub"`` and ``"ses"`` names are validated against the regexp contained in the dict. Parameters ---------- - new_name_templates : Dict - e.g. {"name_templates": {"on": False, "sub": None, "ses": None}} - where "sub" or "ses" can be a regexp that subject and session + new_name_templates + e.g. ``{"name_templates": {"on": False, "sub": None, "ses": None}}`` + where ``"sub"`` or ``"ses"`` can be a regexp that subject and session names respectively are validated against. + """ self._update_persistent_setting("name_templates", new_name_templates) @@ -1204,9 +1135,7 @@ def set_name_templates(self, new_name_templates: Dict) -> None: @check_configs_set def show_configs(self) -> None: - """ - Print the current configs to the terminal. - """ + """Print the current configs to the terminal.""" utils.print_message_to_user(self._get_json_dumps_config()) # ------------------------------------------------------------------------- @@ -1221,34 +1150,39 @@ def validate_project( include_central: bool = False, strict_mode: bool = False, ) -> List[str]: - """ - Perform validation on the project. This checks the subject - and session level folders to ensure there are no NeuroBlueprint - formatting issues. + """Perform validation on the project. + + This checks the subject and session level folders to + ensure there are no NeuroBlueprint formatting issues. Parameters ---------- - - top_level_folder : TopLevelFolder | None - Folder to check, either "rawdata" or "derivatives". If ``None``, + top_level_folder + Folder to check, either ``"rawdata"`` or ``"derivatives"``. If ``None``, will check both folders. - display_mode : DisplayMode + display_mode The validation issues are displayed as ``"error"`` (raise error) ``"warn"`` (show warning) or ``"print"`` - include_central : bool + include_central If ``False``, only the local project is validated. Otherwise, both local and central projects are validated. If in local-project mode, this flag is ignored. - strict_mode: bool - If `True`, only allow NeuroBlueprint-formatted folders to exist in + strict_mode + If ``True``, only allow NeuroBlueprint-formatted folders to exist in the project. By default, non-NeuroBlueprint folders (e.g. a folder - called 'my_stuff' in the 'rawdata') are allowed, and only folders - starting with sub- or ses- prefix are checked. In `Strict Mode`, + called '`my_stuff'` in the '`rawdata'`) are allowed, and only folders + starting with sub- or ses- prefix are checked. In ``Strict Mode``, any folder not prefixed with sub-, ses- or a valid datatype will raise a validation issue. + + Returns + ------- + error_messages + A list of validation errors found in the project. + """ if include_central and strict_mode: raise ValueError( @@ -1290,22 +1224,23 @@ def validate_project( @staticmethod def check_name_formatting(names: Union[str, list], prefix: Prefix) -> None: - """ + """Format a list of subject or session names. + Pass list of names to check how these will be auto-formatted, - for example as when passed to create_folders() or upload_custom() - or download() + for example as when passed to ``create_folders()`` or ``upload_custom()`` Useful for checking tags e.g. @TO@, @DATE@, @DATETIME@, @DATE@. - This method will print the formatted list of names, + This method will print the formatted list of names. Parameters ---------- - - names : + names A string or list of subject or session names. - prefix: + + prefix The relevant subject or session prefix, - e.g. "sub-" or "ses-" + e.g. ``"sub-"`` or ``"ses-"`` + """ if prefix not in ["sub", "ses"]: utils.log_and_raise_error( @@ -1329,18 +1264,12 @@ def _transfer_entire_project( overwrite_existing_files: OverwriteExistingFiles, dry_run: bool, ) -> None: - """ - Transfer (i.e. upload or download) the entire project (i.e. - every 'top level folder' (e.g. 'rawdata', 'derivatives'). - - Parameters - ---------- + """Transfer the entire project. - upload_or_download : direction to transfer the data, either "upload" (from - local to central) or "download" (from central to local). + i.e. every 'top level folder' (e.g. 'rawdata', 'derivatives'). + See ``upload_custom()`` or ``download_custom()`` for parameters. """ for top_level_folder in canonical_folders.get_top_level_folders(): - utils.log_and_message(f"Transferring `{top_level_folder}`") self._transfer_top_level_folder( @@ -1358,22 +1287,27 @@ def _start_log( store_in_temp_folder: bool = False, verbose: bool = True, ) -> None: - """ - Initialize the logger. This is typically called at - the start of public methods to initialize logging - for a specific function call. + """Initialize the logger. + + This is typically called at the start of public + methods to initialize logging for a specific function call. Parameters ---------- + command_name + Name of the command, for the log output files. - command_name : name of the command, for the log output files. + local_vars + local_vars are passed to fancylog variables argument. + see ds_logger.wrap_variables_for_fancylog for more info - local_vars : local_vars are passed to fancylog variables argument. - see ds_logger.wrap_variables_for_fancylog for more info - - store_in_temp_folder : - if `False`, existing logging path will be used + store_in_temp_folder + If `False`, existing logging path will be used (local project .datashuttle). + + verbose + Print warnings and error messages. + """ if local_vars is None: variables = None @@ -1393,7 +1327,8 @@ def _start_log( ds_logger.start(path_to_save, command_name, variables, verbose) def _move_logs_from_temp_folder(self) -> None: - """ + """Create a temporary logging folder when the project folder is unknown. + Logs are stored within the project folder. Although in some instances, when setting configs, we do not know what the project folder is. In this case, make the logs @@ -1418,7 +1353,7 @@ def _move_logs_from_temp_folder(self) -> None: ) def _clear_temp_log_path(self) -> None: - """""" + """Delete temporary log files.""" log_files = glob.glob(str(self._temp_log_path / "*.log")) for file in log_files: os.remove(file) @@ -1431,8 +1366,8 @@ def _error_on_base_project_name(self, project_name): ) def _log_successful_config_change(self, message: bool = False) -> None: - """ - Log the entire config at the time of config change. + """Log the entire config at the time of config change. + If messaged, just message "update successful" rather than print the entire configs as it becomes confusing. """ @@ -1444,16 +1379,14 @@ def _log_successful_config_change(self, message: bool = False) -> None: ) def _get_json_dumps_config(self) -> str: - """ - Get the config dictionary formatted as json.dumps() - which allows well formatted printing. - """ + """Return the config dictionary formatted as json.dumps() which allows well formatted printing.""" copy_dict = copy.deepcopy(self.cfg.data) load_configs.convert_str_and_pathlib_paths(copy_dict, "path_to_str") return json.dumps(copy_dict, indent=4) def _make_project_metadata_if_does_not_exist(self) -> None: - """ + """Locate the .datashuttle folder within the project local_path. + Within the project local_path is also a .datashuttle folder that contains additional information, e.g. logs. """ @@ -1478,21 +1411,24 @@ def _setup_rclone_central_local_filesystem_config(self) -> None: def _update_persistent_setting( self, setting_name: str, setting_value: Any ) -> None: - """ - Load settings that are stored persistently across datashuttle - sessions. These are stored in yaml dumped to dictionary. + """Load settings that are stored persistently across datashuttle sessions. + + These are stored in yaml dumped to dictionary. Parameters ---------- - setting_name : dictionary key of the persistent setting to change - setting_value : value to change the persistent setting to + setting_name + dictionary key of the persistent setting to change + + setting_value + value to change the persistent setting to + """ settings = self._load_persistent_settings() if setting_name not in settings: utils.log_and_raise_error( - f"Setting key {setting_name} not found in " - f"settings dictionary", + f"Setting key {setting_name} not found in settings dictionary", KeyError, ) @@ -1501,39 +1437,32 @@ def _update_persistent_setting( self._save_persistent_settings(settings) def _init_persistent_settings(self) -> None: - """ - Initialise the default persistent settings - and save to file. - """ + """Initialise the default persistent settings and save to file.""" settings = canonical_configs.get_persistent_settings_defaults() self._save_persistent_settings(settings) def _save_persistent_settings(self, settings: Dict) -> None: - """ - Save the settings dict to file as .yaml - """ + """Save the settings dict to file as ".yaml".""" with open(self._persistent_settings_path, "w") as settings_file: yaml.dump(settings, settings_file, sort_keys=False) def _load_persistent_settings(self) -> Dict: - """ - Load settings that are stored persistently across - datashuttle sessions. - """ + """Return settings that are stored persistently across datashuttle sessions.""" if not self._persistent_settings_path.is_file(): self._init_persistent_settings() - with open(self._persistent_settings_path, "r") as settings_file: + with open(self._persistent_settings_path) as settings_file: settings = yaml.full_load(settings_file) self._update_settings_with_new_canonical_keys(settings) return settings - def _update_settings_with_new_canonical_keys(self, settings: Dict): - """ + def _update_settings_with_new_canonical_keys(self, settings: Dict) -> None: + """Check and update keys within persistent settings if missing. + Perform a check on the keys within persistent settings. - If they do not exist, persistent settings is from older version + If they do not exist, persistent settings is from an older version and the new keys need adding. If changing keys within the top level (e.g. a dict entry in "tui") this method will need to be extended. @@ -1562,10 +1491,8 @@ def _update_settings_with_new_canonical_keys(self, settings: Dict): settings ) - def _check_top_level_folder(self, top_level_folder): - """ - Raise an error if ``top_level_folder`` not correct. - """ + def _check_top_level_folder(self, top_level_folder) -> None: + """Raise an error if ``top_level_folder`` not correct.""" canonical_top_level_folders = canonical_folders.get_top_level_folders() if top_level_folder not in canonical_top_level_folders: diff --git a/datashuttle/datashuttle_functions.py b/datashuttle/datashuttle_functions.py index 647b8b569..5f24b9301 100644 --- a/datashuttle/datashuttle_functions.py +++ b/datashuttle/datashuttle_functions.py @@ -29,38 +29,40 @@ def quick_validate_project( strict_mode: bool = False, name_templates: Optional[Dict] = None, ) -> List[str]: - """ - Perform validation on the project. This checks the subject - and session level folders to ensure there are not - NeuroBlueprint formatting issues. + """Perform validation on a NeuroBlueprint-formatted project. Parameters ---------- - project_path Path to the project to validate. Must include the project name, and hold a "rawdata" or "derivatives" folder. - top_level_folder : TopLevelFolder + top_level_folder The top-level folder ("rawdata" or "derivatives") to perform validation. If `None`, both are checked. - display_mode : DisplayMode - The validation issues are displayed as ``"error"`` (raise error) - ``"warn"`` (show warning) or ``"print"``. + display_mode + The validation issues are displayed as ``"error"`` (raise error), + ``"warn"`` (show warning), or ``"print"``. - strict_mode: bool - If `True`, only allow NeuroBlueprint-formatted folders to exist in + strict_mode + If ``True``, only allow NeuroBlueprint-formatted folders to exist in the project. By default, non-NeuroBlueprint folders (e.g. a folder called 'my_stuff' in the 'rawdata') are allowed, and only folders starting with sub- or ses- prefix are checked. In `Strict Mode`, any folder not prefixed with sub-, ses- or a valid datatype will raise a validation issue. - name_templates : Dict + name_templates A dictionary of templates for subject and session name to validate against. See ``DataShuttle.set_name_templates()`` for details. + + Returns + ------- + error_messages + A list of validation errors found in the project. + """ project_path = Path(project_path) @@ -101,10 +103,21 @@ def quick_validate_project( def _format_top_level_folder( top_level_folder: TopLevelFolder | None, ) -> List[TopLevelFolder]: - """ + """Format the top level folder. + Take a `top_level_folder` ("rawdata" or "derivatives" str) and - convert to list, if `None`, convert it to a list + convert it to a list. If `None`, convert it to a list of both possible top-level folders. + + Parameters + ---------- + top_level_folder + The top-level folder to format. Can be "rawdata", "derivatives", or None. + + Returns + ------- + A list of top-level folder names (e.g. ["rawdata"]). + """ rawdata_and_derivatives: List[TopLevelFolder] = ["rawdata", "derivatives"] diff --git a/datashuttle/tui/app.py b/datashuttle/tui/app.py index 022d74f9d..4507b719f 100644 --- a/datashuttle/tui/app.py +++ b/datashuttle/tui/app.py @@ -33,8 +33,7 @@ class TuiApp(App, inherit_bindings=False): # type: ignore - """ - The main app page for the DataShuttle TUI. + """The main app page for the DataShuttle TUI. This class acts as a base class from which all windows (select existing project, make new project, settings and @@ -50,6 +49,7 @@ class TuiApp(App, inherit_bindings=False): # type: ignore ] def compose(self) -> ComposeResult: + """Set up widgets for the main window.""" yield Container( Label("datashuttle", id="mainwindow_banner_label"), Button( @@ -67,17 +67,26 @@ def compose(self) -> ComposeResult: ) def on_mount(self) -> None: + """Update widgets immediately after creation.""" self.set_dark_mode(self.load_global_settings()["dark_mode"]) id = "#mainwindow_validate_from_project_path" self.query_one(id).tooltip = get_tooltip(id) def set_dark_mode(self, dark_mode: bool) -> None: + """Set the color theme for the application.""" self.theme = "textual-dark" if dark_mode else "textual-light" def on_button_pressed(self, event: Button.Pressed) -> None: - """ - Raise the relevant screen after button press. `push_screen` - second argument is a callback function returned after screen closes. + """Handle a button press on the main window and raise a new screen. + + Note that`push_screen` second argument is a callback function returned + after screen closes. + + Parameters + ---------- + event + A textual event containing information about the button press. + """ if event.button.id == "mainwindow_existing_project_button": self.push_screen( @@ -105,6 +114,16 @@ def on_button_pressed(self, event: Button.Pressed) -> None: self.push_screen(validate_at_path.ValidateScreen(self)) def load_project_page(self, interface: Interface) -> None: + """Load the project manager page. + + This page contains a list of all existing projects. + + Parameters + ---------- + interface + Datashuttle tui Interface object. + + """ if interface: self.push_screen( project_manager.ProjectManagerScreen( @@ -113,20 +132,24 @@ def load_project_page(self, interface: Interface) -> None: ) def show_modal_error_dialog(self, message: str) -> None: + """Show an error in a pop-up window. + + The pop-up window has a red border and is modal (cannot click + elsewhere on the application when it is shown). + + Parameter + --------- + message + Error message to display. + """ self.push_screen(modal_dialogs.MessageBox(message, border_color="red")) def show_modal_error_dialog_from_main_thread(self, message: str) -> None: - """ - Used to call `show_modal_error_dialog from main thread when executing - in another thread. Throws error when called from main thread. - """ + """Call`show_modal_error_dialog from main thread when executing in another thread.""" self.call_from_thread(self.show_modal_error_dialog, message) def handle_open_filesystem_browser(self, path_: Path) -> None: - """ - Open the system file browser to the path with the `showinfm` - package, performing checks that the path exists prior to opening. - """ + """Open the system file browser to the path with the `showinfm` package.""" if not path_.exists(): self.show_modal_error_dialog( f"{path_.as_posix()} cannot be opened as it " @@ -152,15 +175,36 @@ def handle_open_filesystem_browser(self, path_: Path) -> None: self.show_modal_error_dialog(message) - def prompt_rename_file_or_folder(self, path_): - """ """ + def prompt_rename_file_or_folder(self, path_) -> None: + """Display pop-up window to rename a file or folder in a tab DirectoryTree. + + Parameters + ---------- + path_ + Path to the file or folder to rename. + + TODO + ---- + Can this not be moved to the relevant tab page? + + """ self.push_screen( modal_dialogs.RenameFileOrFolderScreen(self, path_), lambda new_name: self.rename_file_or_folder(path_, new_name), ) - def rename_file_or_folder(self, path_, new_name): - """ """ + def rename_file_or_folder(self, path_, new_name) -> None: + """Rename a file or folder within the project. + + Parameters + ---------- + path_ + Path to the file or folder to rename. + + new_name + New name for the file or folder. + + """ if new_name is False: return try: @@ -189,11 +233,11 @@ def rename_file_or_folder(self, path_, new_name): # Global Settings --------------------------------------------------------- def load_global_settings(self) -> Dict: - """ - Load the 'global settings' for the TUI that determine - project-independent settings that are persistent across - sessions. These are stored in the canonical - .datashuttle folder (see `get_global_settings_path`). + """Return the loaded 'global settings' for the TUI. + + These settings determine project-independent settings + that are persistent across sessions. These are stored + in the canonical .datashuttle folder (see `get_global_settings_path`). """ settings_path = self.get_global_settings_path() @@ -201,25 +245,25 @@ def load_global_settings(self) -> Dict: global_settings = self.get_default_global_settings() self.save_global_settings(global_settings) else: - with open(settings_path, "r") as file: + with open(settings_path) as file: global_settings = yaml.full_load(file) return global_settings def get_global_settings_path(self) -> Path: - """ - The canonical path for the TUI's global settings. - """ + """Return the canonical path for the TUI global settings.""" path_ = canonical_folders.get_datashuttle_path() return path_ / "global_tui_settings.yaml" def get_default_global_settings(self) -> Dict: + """Return placeholder default global settings for the TUI.""" return { "dark_mode": True, "show_transfer_tree_status": False, } def save_global_settings(self, global_settings: Dict) -> None: + """Save the TUI global settings to disk.""" settings_path = self.get_global_settings_path() if not settings_path.parent.is_dir(): @@ -228,10 +272,16 @@ def save_global_settings(self, global_settings: Dict) -> None: with open(settings_path, "w") as file: yaml.dump(global_settings, file, sort_keys=False) - def copy_to_clipboard(self, value): - """ - Centralized function to copy to clipboard. - This may fail under some circumstances (e.g., in headless mode on an HPC). + def copy_to_clipboard(self, value) -> None: + """Centralized function to copy to clipboard. + + This may fail in headless mode on an HPC. + + Parameters + ---------- + value + Value to copy to clipboard. + """ try: pyperclip.copy(value) @@ -242,6 +292,7 @@ def copy_to_clipboard(self, value): def main(): + """Start the application.""" TuiApp().run() diff --git a/datashuttle/tui/custom_widgets.py b/datashuttle/tui/custom_widgets.py index 34c8555bc..5ab548fca 100644 --- a/datashuttle/tui/custom_widgets.py +++ b/datashuttle/tui/custom_widgets.py @@ -2,7 +2,6 @@ from typing import ( TYPE_CHECKING, - Iterable, List, Optional, Tuple, @@ -10,6 +9,8 @@ ) if TYPE_CHECKING: + from collections.abc import Iterable + from textual import events from textual.validation import Validator @@ -38,14 +39,15 @@ # ClickableInput # -------------------------------------------------------------------------------------- class ClickableInput(Input): - """ - An input widget which emits a `ClickableInput.Clicked` - signal when clicked, containing the input name - `input` and mouse button index `button`. + """An input widget which emits a `ClickableInput.Clicked` event when clicked. + + The event contains the input name `input` and mouse button index `button`. """ @dataclass class Clicked(Message): + """The event class.""" + input: ClickableInput ctrl: bool @@ -57,6 +59,26 @@ def __init__( validate_on: Optional[List[str]] = None, validators: Optional[List[Validator]] = None, ) -> None: + """Initialise the Clicked event class. + + Parameters + ---------- + mainwindow + The Datashuttle TUI application + + placeholder + Placeholder (i.e. example) text for the Input. + + id + Textual ID of the Input. + + validate_on + Textual keywords specifying actions to validate on (e.g. ["changed", "submitted"]) + + validators + Textual validator objects to apply. + + """ super(ClickableInput, self).__init__( placeholder=placeholder, id=id, @@ -67,12 +89,15 @@ def __init__( self.mainwindow = mainwindow def _on_click(self, event: events.Click) -> None: + """Handle when the input is clicked.""" self.post_message(self.Clicked(self, event.ctrl)) def as_names_list(self) -> List[str]: + """Return the contents of the input as a list split by ','.""" return self.value.replace(" ", "").split(",") def on_key(self, event: events.Key) -> None: + """Handle keyboard press on the Input.""" if event.key == "ctrl+q": self.mainwindow.copy_to_clipboard(self.value) @@ -86,36 +111,67 @@ def on_key(self, event: events.Key) -> None: class CustomDirectoryTree(DirectoryTree): - """ - Base class for directory tree with some customised additions: - - filter out top-level folders that are not canonical - - add additional keyboard shortcuts defined in `on_key`. + """Base class for a custom directory tree. + + This has some customised additions: + - filter out top-level folders that are not canonical + - add additional keyboard shortcuts defined in `on_key`. """ @dataclass class DirectoryTreeSpecialKeyPress(Message): + """Event to handle a key press on the CustomDirectoryTree.""" + key: str node_path: Optional[Path] def __init__( self, mainwindow: TuiApp, path: Path, id: Optional[str] = None ) -> None: + """Initialise the Directory Tree. + + Parameters + ---------- + mainwindow + Datashuttle TUI application. + + path + Root path for the CustomDirectoryTree. + + id + Textual ID for the CustomDirectoryTree. + + """ super(CustomDirectoryTree, self).__init__(path=path, id=id) self.mainwindow = mainwindow def filter_paths(self, paths: Iterable[Path]) -> Iterable[Path]: - """ - Filter out all hidden folders and files from DirectoryTree - display. + """Filter out all hidden folders and files from CustomDirectoryTree display. + + Parameters + ---------- + paths + Paths to be filtered before being added to the CustomDirectoryTree. + + Returns + ------- + A list of filtered paths + """ return [path for path in paths if not path.name.startswith(".")] def on_key(self, event: events.Key) -> None: - """ - Handle key presses on the CustomDirectoryTree. Depending on the keys pressed, - copy the path under the cursor, refresh the directorytree or - emit a DirectoryTreeSpecialKeyPress event. + """Handle key presses on the CustomDirectoryTree. + + Depending on the keys pressed, copy the path under the cursor, + refresh the CustomDirectoryTree or emit a DirectoryTreeSpecialKeyPress event. + + Parameters + ---------- + event + Textual event containing information on the key press. + """ if event.key == "ctrl+q": path_ = self.get_node_at_line(self.hover_line).data.path @@ -143,8 +199,9 @@ def on_key(self, event: events.Key) -> None: def _render_line( self, y: int, x1: int, x2: int, base_style: Style ) -> Strip: - """ - This function is overridden from textual's `Tree` class to stop + """Overridden function that renders CustomDirectoryTree lines. + + Overridden from textual's `Tree` class to stop CSS styling on hovering and clicking which was distracting / changed the default color used for transfer status, respectively. @@ -195,11 +252,15 @@ def _render_line( def get_guides(style: Style) -> tuple[str, str, str, str]: """Get the guide strings for a given style. - Args: - style: A Style object. + Parameters + ---------- + style + A Style object. + + Returns + ------- + Strings for space, vertical, terminator and cross. - Returns: - Strings for space, vertical, terminator and cross. """ lines: tuple[ Iterable[str], Iterable[str], Iterable[str], Iterable[str] @@ -287,7 +348,8 @@ def get_guides(style: Style) -> tuple[str, str, str, str]: class TreeAndInputTab(TabPane): - """ + """High level class for Custom DirectoryTrees and Inputs. + A parent class that defined common methods for screens with a directory tree and sub / session inputs, .e. the Create tab and the Transfer tab. @@ -296,7 +358,8 @@ class TreeAndInputTab(TabPane): def handle_fill_input_from_directorytree( self, sub_input_key: str, ses_input_key: str, event: events.Key ) -> None: - """ + """Handle interactions between CustomDirectoryTree and Inputs. + When a CustomDirectoryTree key is pressed, we typically want to perform an action that involves an Input. These are coordinated here. Note that the 'copy' and 'refresh' @@ -315,16 +378,16 @@ def handle_fill_input_from_directorytree( Parameters ---------- - - sub_input_key : str + sub_input_key The textual widget id for the subject input (prefixed with #) - ses_input_key : str + ses_input_key The textual widget id for the session input (prefixed with #) - event : DirectoryTreeSpecialKeyPress + event A DirectoryTreeSpecialKeyPress event triggered from the CustomDirectoryTree. + """ if event.key == "ctrl+a": self.append_sub_or_ses_name_to_input( @@ -342,12 +405,19 @@ def handle_fill_input_from_directorytree( def insert_sub_or_ses_name_to_input( self, sub_input_key: str, ses_input_key: str, name: str ) -> None: - """ - see `handle_directorytree_key_pressed` for `sub_input_key` and - `ses_input_key`. + """Fill an input with the CustomDirectoryTree subject or session folder name under mouse. - name : str + Parameters + ---------- + sub_input_key + The textual widget id for the subject input (prefixed with #) + + ses_input_key + The textual widget id for the session input (prefixed with #) + + name The sub or ses name to append to the input. + """ if name.startswith("sub-"): self.query_one(sub_input_key).value = name @@ -357,9 +427,7 @@ def insert_sub_or_ses_name_to_input( def append_sub_or_ses_name_to_input( self, sub_input_key: str, ses_input_key: str, name: str ) -> None: - """ - see `insert_sub_or_ses_name_to_input`. - """ + """See `insert_sub_or_ses_name_to_input`.""" if name.startswith("sub-"): if not self.query_one(sub_input_key).value: self.query_one(sub_input_key).value = name @@ -375,9 +443,7 @@ def append_sub_or_ses_name_to_input( def get_sub_ses_names_and_datatype( self, sub_input_key: str, ses_input_key: str ) -> Tuple[List[str], List[str], List[str]]: - """ - see `handle_fill_input_from_directorytree` for parameters. - """ + """See `handle_fill_input_from_directorytree` for parameters.""" sub_names = self.query_one(sub_input_key).as_names_list() ses_names = self.query_one(ses_input_key).as_names_list() datatype = self.query_one("DatatypeCheckboxes").selected_datatypes() @@ -386,9 +452,9 @@ def get_sub_ses_names_and_datatype( class TopLevelFolderSelect(Select): - """ - A Select widget for display and updating of top-level-folders. The - Create tab and transfer tabs (custom, top-level-folder) all have + """A Select widget for display and updating of top-level-folders. + + The Create tab and transfer tabs (custom, top-level-folder) all have top level folder selects that perform the same function. This widget unifies these in a single place. @@ -398,17 +464,28 @@ class TopLevelFolderSelect(Select): Parameters ---------- - - existing_only : bool + existing_only If `True`, only top level folders that actually exist in the project are displayed. Otherwise, all possible canonical top-level-folders are displayed. - id : str + id Textualize widget id + """ def __init__(self, interface: Interface, id: str) -> None: + """Initialise the TopLevelFolderSelect. + + Parameters + ---------- + interface + Datashuttle Interface object. + + id + Textual id for the Select widget. + + """ self.interface = interface top_level_folders = [ @@ -440,37 +517,31 @@ def __init__(self, interface: Interface, id: str) -> None: ) def get_top_level_folder(self, init: bool = False) -> str: - """ - Get the top level folder from `persistent_settings`, - performing a confidence-check that it matches the textual display. + """Return the top level folder from `persistent_settings`. + + Performs a confidence-check that it matches the textual display. """ top_level_folder = self.interface.tui_settings[ "top_level_folder_select" ][self.settings_key] if not init: - assert ( - top_level_folder == self.get_displayed_top_level_folder() - ), "config and widget should never be out of sync." + assert top_level_folder == self.get_displayed_top_level_folder(), ( + "config and widget should never be out of sync." + ) return top_level_folder def get_displayed_top_level_folder(self) -> str: - """ - Get the top level folder that is currently selected - on the select widget. - """ + """Return the top level folder that is currently selected on the select widget.""" assert self.value in canonical_folders.get_top_level_folders() return self.value def on_select_changed(self, event: Select.Changed) -> None: - """ - When the select is changed, update the linked persistent setting. - """ + """When the select is changed, update the linked persistent setting.""" top_level_folder = event.value if event.value != Select.BLANK: - self.interface.save_tui_settings( top_level_folder, "top_level_folder_select", self.settings_key ) diff --git a/datashuttle/tui/interface.py b/datashuttle/tui/interface.py index aedd2815e..cc7c91ef9 100644 --- a/datashuttle/tui/interface.py +++ b/datashuttle/tui/interface.py @@ -15,11 +15,11 @@ class Interface: - """ - An interface class between the TUI and datashuttle API. Takes input - to all datashuttle functions as passed from the TUI, outputs - success status (True or False) and optional data, in the case - of False. + """An interface class between the TUI and datashuttle API. + + Takes input to all datashuttle functions as passed from the TUI, + outputs success status (True or False) and optional data, in the + case of False. `self.project` is initialised when project is loaded. @@ -31,21 +31,20 @@ class Interface: """ def __init__(self) -> None: - + """Initialise the Interface class.""" self.project: DataShuttle self.name_templates: Dict = {} self.tui_settings: Dict = {} def select_existing_project(self, project_name: str) -> InterfaceOutput: - """ - Load an existing project into `self.project`. + """Load an existing project into `self.project`. Parameters ---------- - - project_name : str + project_name The name of the datashuttle project to load. Must already exist. + """ try: project = DataShuttle(project_name, print_startup_message=False) @@ -58,17 +57,16 @@ def select_existing_project(self, project_name: str) -> InterfaceOutput: def setup_new_project( self, project_name: str, cfg_kwargs: Dict ) -> InterfaceOutput: - """ - Set up a new project and load into `self.project`. + """Set up a new project and load into `self.project`. Parameters ---------- - - project_name : str + project_name Name of the project to set up. - cfg_kwargs : Dict + cfg_kwargs The configurations to set the new project to. + """ try: project = DataShuttle(project_name, print_startup_message=False) @@ -85,15 +83,15 @@ def setup_new_project( def set_configs_on_existing_project( self, cfg_kwargs: Dict ) -> InterfaceOutput: - """ - Update the settings on an existing project. Only the settings - passed in `cfg_kwargs` are updated. + """Update the settings on an existing project. + + Only the settings passed in `cfg_kwargs` are updated. Parameters ---------- - - cfg_kwargs : Dict + cfg_kwargs The configs and new values to update. + """ try: self.project.update_config_file(**cfg_kwargs) @@ -108,20 +106,19 @@ def create_folders( ses_names: Optional[List[str]], datatype: List[str], ) -> InterfaceOutput: - """ - Create folders through datashuttle. + """Create folders through datashuttle. Parameters ---------- - - sub_names : List[str] + sub_names A list of un-formatted / unvalidated subject names to create. - ses_names : List[str] + ses_names A list of un-formatted / unvalidated session names to create. - datatype : List[str] + datatype A list of canonical datatype names to create. + """ top_level_folder = self.tui_settings["top_level_folder_select"][ "create_tab" @@ -144,21 +141,21 @@ def create_folders( def validate_names( self, sub_names: List[str], ses_names: Optional[List[str]] ) -> InterfaceOutput: - """ - Validate a list of subject / session names. This is used - to populate the Input tooltips with validation errors. + """Validate a list of subject / session names. + + This is used to populate the Input tooltips with validation errors. Uses a central `_format_and_validate_names()` that is also called during folder creation itself, to ensure these a results always match. Parameters ---------- - - sub_names : List[str] + sub_names List of subject names to format. - ses_names : List[str] + ses_names List of session names to format. + """ top_level_folder = self.tui_settings["top_level_folder_select"][ "create_tab" @@ -187,13 +184,12 @@ def validate_project( include_central: bool, strict_mode: bool, ) -> tuple[bool, list[str] | str]: - """ - Wrap the validate project function. This returns a list of validation - errors (empty if there are none). + """Wrap the validate project function. + + This returns a list of validation errors (empty if there are none). Parameters ---------- - top_level_folder The "rawdata" or "derivatives" folder to validate. If `None`, both will be validated. @@ -201,6 +197,15 @@ def validate_project( If `True`, the central project is also validated. strict_mode If `True`, validation will be run in strict mode. + + Returns + ------- + success + A bool inicating whether the validation was run successfully + + issues + A message or list of discovered validation issues. + """ try: results = self.project.validate_project( @@ -218,15 +223,14 @@ def validate_project( # ---------------------------------------------------------------------------------- def transfer_entire_project(self, upload: bool) -> InterfaceOutput: - """ - Transfer the entire project (all canonical top-level folders). + """Transfer the entire project (all canonical top-level folders). Parameters ---------- - - upload : bool + upload Upload from local to central if `True`, otherwise download from central to remote. + """ try: if upload: @@ -249,16 +253,14 @@ def transfer_entire_project(self, upload: bool) -> InterfaceOutput: def transfer_top_level_only( self, selected_top_level_folder: str, upload: bool ) -> InterfaceOutput: - """ - Transfer all files within a selected top level folder. + """Transfer all files within a selected top level folder. Parameters ---------- - - selected_top_level_folder : str + selected_top_level_folder The top level folder selected in the TUI for this transfer window. - upload : bool + upload Upload from local to central if `True`, otherwise download from central to remote. @@ -299,27 +301,26 @@ def transfer_custom_selection( datatype: List[str], upload: bool, ) -> InterfaceOutput: - """ - Transfer a custom selection of subjects / sessions / datatypes. + """Transfer a custom selection of subjects / sessions / datatypes. Parameters ---------- - - selected_top_level_folder : str + selected_top_level_folder The top level folder selected in the TUI for this transfer window. - sub_names : List[str] + sub_names Subject names or subject-level canonical transfer keys to transfer. - ses_names : List[str] + ses_names Session names or session-level canonical transfer keys to transfer. - datatype : List[str] + datatype Datatypes or datatype-level canonical transfer keys to transfer. - upload : bool + upload Upload from local to central if `True`, otherwise download from central to remote. + """ try: if upload: @@ -347,12 +348,12 @@ def transfer_custom_selection( # ---------------------------------------------------------------------------------- def get_name_templates(self) -> Dict: - """ - Get the `name_templates` defining templates to validate - against. These are stored in a variable to avoid constantly + """Return the `name_templates` defining templates to validate against. + + These are stored in a variable to avoid constantly reading these values from disk where they are stored in `persistent_settings`. It is critical this variable - and the file contetns are in sync, so when changed + and the file contents are in sync, so when changed on the TUI side they are updated also, in `get_tui_settings`. """ if not self.name_templates: @@ -361,9 +362,9 @@ def get_name_templates(self) -> Dict: return self.name_templates def set_name_templates(self, name_templates: Dict) -> InterfaceOutput: - """ - Set the `name_templates` here and on disk. See `get_name_templates` - for more information. + """Set the `name_templates` here and on disk. + + See `get_name_templates` for more information. """ try: self.project.set_name_templates(name_templates) @@ -374,10 +375,10 @@ def set_name_templates(self, name_templates: Dict) -> InterfaceOutput: return False, str(e) def get_tui_settings(self) -> Dict: - """ - Get the "tui" field of `persistent_settings`. Similar to - `get_name_templates`, there are held on the class to avoid - constantly reading from disk. + """Return the "tui" field of `persistent_settings`. + + Similar to `get_name_templates`, there are held on the + class to avoid constantly reading from disk. """ if not self.tui_settings: self.tui_settings = self.project._load_persistent_settings()["tui"] @@ -387,22 +388,21 @@ def get_tui_settings(self) -> Dict: def save_tui_settings( self, value: Any, key: str, key_2: Optional[str] = None ) -> None: - """ - Update the "tui" field of the `persistent_settings` on disk. + """Update the "tui" field of the `persistent_settings` on disk. Parameters ---------- - - value : Any + value Value to set the `persistent_settings` tui field to - key_1 : str + key First key of the tui `persistent_settings` to update e.g. "top_level_folder_select" - key_2 : str + key_2 Optionals second level of the dictionary to update. e.g. "create_tab" + """ if key_2 is None: self.tui_settings[key] = value @@ -415,17 +415,19 @@ def save_tui_settings( # ---------------------------------------------------------------------------------- def get_central_host_id(self) -> str: + """Return the central host id for ssh.""" return self.project.cfg["central_host_id"] def get_configs(self) -> Configs: + """Return Datashuttle Configs.""" return self.project.cfg def get_textual_compatible_project_configs(self) -> Configs: - """ - Datashuttle configs keeps paths saved as pathlib.Path - objects. In some cases textual requires str representation. - This method returns datashuttle configs with all paths that - are Path converted to str. + """Return Datashuttle configs with paths stored as str. + + In some cases textual requires str representation. This method + returns datashuttle configs with all paths that are Path + converted to str. """ cfg_to_load = copy.deepcopy(self.project.cfg) load_configs.convert_str_and_pathlib_paths(cfg_to_load, "path_to_str") @@ -434,6 +436,7 @@ def get_textual_compatible_project_configs(self) -> Configs: def get_next_sub( self, top_level_folder: TopLevelFolder, include_central: bool ) -> InterfaceOutput: + """Return the next subject ID in the project.""" try: next_sub = self.project.get_next_sub( top_level_folder, @@ -447,7 +450,7 @@ def get_next_sub( def get_next_ses( self, top_level_folder: TopLevelFolder, sub: str, include_central: bool ) -> InterfaceOutput: - + """Return the next session ID for the `sub` in the project.""" try: next_ses = self.project.get_next_ses( top_level_folder, @@ -460,6 +463,7 @@ def get_next_ses( return False, str(e) def get_ssh_hostkey(self) -> InterfaceOutput: + """Return the SSH remote server host key.""" try: key = ssh.get_remote_server_key( self.project.cfg["central_host_id"] @@ -469,6 +473,7 @@ def get_ssh_hostkey(self) -> InterfaceOutput: return False, str(e) def save_hostkey_locally(self, key: paramiko.RSAKey) -> InterfaceOutput: + """Save the SSH hostkey to disk.""" try: ssh.save_hostkey_locally( key, @@ -483,6 +488,7 @@ def save_hostkey_locally(self, key: paramiko.RSAKey) -> InterfaceOutput: def setup_key_pair_and_rclone_config( self, password: str ) -> InterfaceOutput: + """Set up SSH key pair and associated rclone configuration.""" try: ssh.add_public_key_to_central_authorized_keys( self.project.cfg, password, log=False diff --git a/datashuttle/tui/screens/create_folder_settings.py b/datashuttle/tui/screens/create_folder_settings.py index 952f8318f..5448d7ea5 100644 --- a/datashuttle/tui/screens/create_folder_settings.py +++ b/datashuttle/tui/screens/create_folder_settings.py @@ -3,7 +3,6 @@ from typing import TYPE_CHECKING, Dict, Optional if TYPE_CHECKING: - from textual.app import ComposeResult from datashuttle.tui.app import TuiApp @@ -28,8 +27,9 @@ class CreateFoldersSettingsScreen(ModalScreen): - """ - This screen handles setting datashuttle's `name_template`'s, as well + """Settings for the Create Folders tab. + + Handles setting datashuttle's `name_template`, as well as the top-level-folder select and option to bypass all validation. Name Templates @@ -46,15 +46,26 @@ class CreateFoldersSettingsScreen(ModalScreen): Attributes ---------- - Because the Input for `name_templates` is shared between subject and session, the values are held in the `input_values` attribute. These are loaded from `persistent_settings` on init. + """ TITLE = "Create Folders Settings" def __init__(self, mainwindow: TuiApp, interface: Interface) -> None: + """Initialise the CreateFoldersSettingsScreen. + + Parameters + ---------- + mainwindow + The main TUI app. + + interface + Datashuttle Interface object. + + """ super(CreateFoldersSettingsScreen, self).__init__() self.mainwindow = mainwindow @@ -69,9 +80,11 @@ def __init__(self, mainwindow: TuiApp, interface: Interface) -> None: } def action_link_docs(self) -> None: + """Link to datashuttle documentation.""" webbrowser.open(links.get_docs_link()) def compose(self) -> ComposeResult: + """Add widgets to the CreateFoldersSettingsScreen.""" sub_on = True if self.input_mode == "sub" else False ses_on = not sub_on @@ -153,6 +166,7 @@ def compose(self) -> ComposeResult: ) def on_mount(self) -> None: + """Update widgets immediately after mounting.""" for id in [ "#create_folders_settings_toplevel_select", "#create_folders_settings_bypass_validation_checkbox", @@ -166,21 +180,23 @@ def on_mount(self) -> None: self.switch_template_container_disabled() def init_input_values_holding_variable(self) -> None: + """Add the project Name Templates to the relevant Inputs.""" name_templates = self.interface.get_name_templates() self.input_values["sub"] = name_templates["sub"] self.input_values["ses"] = name_templates["ses"] def switch_template_container_disabled(self) -> None: + """Switch the name template widgets disabled / enabled.""" is_on = self.query_one( "#template_settings_validation_on_checkbox" ).value self.query_one("#template_inner_container").disabled = not is_on def fill_input_from_template(self) -> None: - """ - Fill the `name_templates` Input, that is shared - between subject and session, depending on the - current radioset value. + """Fill the `name_templates` Input. + + The Input is shared between subject and session, + depending on the current RadioSet value. """ input = self.query_one("#template_settings_input") value = self.input_values[self.input_mode] @@ -192,7 +208,8 @@ def fill_input_from_template(self) -> None: input.value = value def on_button_pressed(self, event: Button.Pressed) -> None: - """ + """Handle button press on the screen. + On close, update the `name_templates` stored in `persistent_settings` with those set on the TUI. @@ -212,6 +229,7 @@ def on_button_pressed(self, event: Button.Pressed) -> None: self.interface.save_tui_settings(False, "bypass_validation") def make_name_templates_from_widgets(self) -> Dict: + """Return a canonical `name_templates` entry based on the current widget settings.""" return { "on": self.query_one( "#template_settings_validation_on_checkbox" @@ -221,9 +239,7 @@ def make_name_templates_from_widgets(self) -> Dict: } def on_checkbox_changed(self, event: Checkbox.Changed) -> None: - """ - Turn `name_templates` on or off and update the TUI accordingly. - """ + """Turn `name_templates` on or off and update the TUI accordingly.""" is_on = event.value if event.checkbox.id == "template_settings_validation_on_checkbox": @@ -245,17 +261,19 @@ def on_checkbox_changed(self, event: Checkbox.Changed) -> None: disable_container = not self.query_one( "#template_settings_validation_on_checkbox" ).value - self.query_one("#template_inner_container").disabled = ( - disable_container - ) + + self.query_one( + "#template_inner_container" + ).disabled = disable_container elif event.checkbox.id == "suggest_next_sub_ses_central_checkbox": self.interface.save_tui_settings( is_on, "suggest_next_sub_ses_central" ) def on_radio_set_changed(self, event: RadioSet.Changed) -> None: - """ - Update the displayed SSH widgets when the `connection_method` + """Update the displayed SSH widgets. + + These are updated when the `connection_method` radiobuttons are changed. """ label = str(event.pressed.label) @@ -265,6 +283,7 @@ def on_radio_set_changed(self, event: RadioSet.Changed) -> None: self.fill_input_from_template() def on_input_changed(self, message: Input.Changed) -> None: + """Store the current Input value in the attribute to be saved later.""" if message.input.id == "template_settings_input": val = None if message.value == "" else message.value self.input_values[self.input_mode] = val diff --git a/datashuttle/tui/screens/datatypes.py b/datashuttle/tui/screens/datatypes.py index df184d4e6..7c21f84df 100644 --- a/datashuttle/tui/screens/datatypes.py +++ b/datashuttle/tui/screens/datatypes.py @@ -3,7 +3,6 @@ from typing import TYPE_CHECKING, List, Literal, Optional if TYPE_CHECKING: - from textual.app import ComposeResult from datashuttle.tui.interface import Interface @@ -67,8 +66,7 @@ class DisplayedDatatypesScreen(ModalScreen): - """ - Screen to select the which datatype checkboxes to show on the Create / Transfer tabs. + """Screen to select the which datatype checkboxes to show on the Create / Transfer tabs. Display a SessionList widget which all canonical broad and narrow-type datatypes. When selected, this will update DatatypeCheckboxes (coordinates @@ -85,6 +83,7 @@ class DisplayedDatatypesScreen(ModalScreen): checkboxes are so close together. Testing indicate that when writing to file after each click, syncing could get messed up and the wrong checkboxes displayed on the window. + """ def __init__( @@ -92,6 +91,17 @@ def __init__( create_or_transfer: Literal["create", "transfer"], interface: Interface, ) -> None: + """Initialise the DisplayedDatatypesScreen. + + Parameters + ---------- + create_or_transfer + Whether we are on the "create" or "transfer" tab. + + interface + Datashuttle Interface object. + + """ super(DisplayedDatatypesScreen, self).__init__() self.interface = interface @@ -104,10 +114,7 @@ def __init__( ) def compose(self) -> ComposeResult: - """ - Collect the datatypes names and status from - the persistent settings and display. - """ + """Collect the datatypes names and status from the persistent settings and display.""" selections = [] for idx, (datatype, setting) in enumerate( self.datatype_config.items() @@ -141,14 +148,9 @@ def compose(self) -> ComposeResult: id="display_datatypes_screen_container", ) - def on_mount(self): - pass - - # self.query_one("#display_datatypes_screen_container").action_scroll_up() - # assert False, f"{dir(self.query_one('#displayed_datatypes_selection_list'))}" - def on_button_pressed(self, event): - """ + """Handle button press on the DisplayedDatatypesScreen. + When 'Save' is pressed, the configs copied on this class are updated back onto the interface configs, and written to disk. Otherwise, close the screen without saving. @@ -165,10 +167,7 @@ def on_button_pressed(self, event): def on_selection_list_selection_toggled( self, event: SelectionList.SelectionMessage.SelectionToggled ): - """ - When a selection is toggled, update the configs with - the 'displayed' status and save to disk. - """ + """Update the configs with the 'displayed' status and save to disk when Select is changed.""" datatype_name = event.selection.prompt.plain datatype_name = datatype_name.split(" ")[0] is_checked = not event.selection.initial_state @@ -184,25 +183,23 @@ def on_selection_list_selection_toggled( class DatatypeCheckboxes(Static): - """ - Dynamically-populated checkbox widget for convenient datatype - selection during folder creation. + """Dynamically-populated checkbox widget for convenient datatype selection. Parameters ---------- - - settings_key : 'create' if datatype checkboxes for the create tab, - 'transfer' for the transfer tab. Transfer tab includes - additional datatype options (e.g. "all"). + settings_key + 'create' if datatype checkboxes for the create tab, + 'transfer' for the transfer tab. Transfer tab includes + additional datatype options (e.g. "all"). Attributes ---------- - - datatype_config : a Dictionary containing datatype as key (e.g. "ephys", "behav") - and values are `bool` indicating whether the checkbox is on / off. - If 'transfer', then transfer datatype arguments (e.g. "all") - are also included. This structure mirrors - the `persistent_settings` dictionaries. + datatype_config + A Dictionary containing datatype as key (e.g. "ephys", "behav") + and values are `bool` indicating whether the checkbox is on / off. + If 'transfer', then transfer datatype arguments (e.g. "all") + are also included. This structure mirrors + the `persistent_settings` dictionaries. Notes ----- @@ -210,6 +207,7 @@ class DatatypeCheckboxes(Static): however because this screen persists through the lifetime of the app there is no clear time point in which to save the checkbox status. Therefore, the configs are updated (written to disk) on each click. + """ def __init__( @@ -218,6 +216,20 @@ def __init__( create_or_transfer: Literal["create", "transfer"] = "create", id: Optional[str] = None, ) -> None: + """Initialise the DatatypeCheckboxes. + + Parameters + ---------- + interface + Datashuttle Interface object. + + create_or_transfer + Whether we are on the "create" or "transfer" tab. + + id + Textual ID for the DatatypeCheckboxes widget. + + """ super(DatatypeCheckboxes, self).__init__(id=id) self.interface = interface @@ -232,6 +244,7 @@ def __init__( ] def compose(self) -> ComposeResult: + """Add widgets to the DatatypeCheckboxes.""" for datatype, setting in self.datatype_config.items(): if setting["displayed"]: yield Checkbox( @@ -242,7 +255,8 @@ def compose(self) -> ComposeResult: @on(Checkbox.Changed) def on_checkbox_changed(self) -> None: - """ + """Handle a datatype checkbox change. + When a checkbox is changed, update the `self.datatype_config` to contain new boolean values for each datatype. Also update the stored `persistent_settings`. @@ -258,7 +272,7 @@ def on_checkbox_changed(self) -> None: ) def on_mount(self) -> None: - """ """ + """Add widgets to the DatatypeCheckboxes.""" for datatype in self.datatype_config.keys(): if self.datatype_config[datatype]["displayed"]: self.query_one( @@ -266,10 +280,7 @@ def on_mount(self) -> None: ).tooltip = tooltips[datatype] def selected_datatypes(self) -> List[str]: - """ - Get the names of the datatype options for which the - checkboxes are switched on. - """ + """Return the names of the datatype options for which the checkboxes are switched on.""" selected_datatypes = [ datatype for datatype, settings in self.datatype_config.items() @@ -284,15 +295,18 @@ def selected_datatypes(self) -> List[str]: def get_checkbox_name( create_or_transfer: Literal["create", "transfer"], datatype -): +) -> str: + """Return a canonical formatted checkbox name.""" return f"{create_or_transfer}_{datatype}_checkbox" def get_tui_settings_key_name( create_or_transfer: Literal["create", "transfer"], ) -> str: + """Return the canonical tui settings key.""" if create_or_transfer == "create": settings_key = "create_checkboxes_on" else: settings_key = "transfer_checkboxes_on" + return settings_key diff --git a/datashuttle/tui/screens/get_help.py b/datashuttle/tui/screens/get_help.py index e177b7d42..511f59958 100644 --- a/datashuttle/tui/screens/get_help.py +++ b/datashuttle/tui/screens/get_help.py @@ -18,9 +18,10 @@ class GetHelpScreen(ModalScreen): - """ """ + """A screen with helpful information.""" def __init__(self) -> None: + """Initialise the GetHelpScreen.""" super(GetHelpScreen, self).__init__() self.text = """ @@ -35,19 +36,23 @@ def __init__(self) -> None: """ def action_link_docs(self) -> None: + """Link to datashuttle documentation.""" webbrowser.open(links.get_docs_link()) def action_link_github(self) -> None: + """Link to datashuttle github.""" webbrowser.open(links.get_github_link()) def action_link_github_issues(self) -> None: + """Link to datashuttle github issues.""" webbrowser.open(links.get_link_github_issues()) - def action_link_zulip(self): + def action_link_zulip(self) -> None: + """Link to datashuttle zulip.""" webbrowser.open(links.get_link_zulip()) def compose(self) -> ComposeResult: - + """Add widgets to the GetHelpScreen.""" yield Container( Static(self.text, id="get_help_label"), Button("Main Menu", id="all_main_menu_buttons"), @@ -55,5 +60,6 @@ def compose(self) -> ComposeResult: ) def on_button_pressed(self, event: Button.Pressed) -> None: + """Handle a button press on the GetHelpScreen.""" if event.button.id == "all_main_menu_buttons": self.dismiss() diff --git a/datashuttle/tui/screens/modal_dialogs.py b/datashuttle/tui/screens/modal_dialogs.py index 82d7da21d..c7ca57c3e 100644 --- a/datashuttle/tui/screens/modal_dialogs.py +++ b/datashuttle/tui/screens/modal_dialogs.py @@ -36,24 +36,38 @@ class MessageBox(ModalScreen): - """ - A screen for rendering error messages. + """A screen for rendering error messages. - message : str + Parameters + ---------- + message The message to display in the message box - border_color : str + border_color The color to pass to the `border` style on the widget. Note that the keywords 'red' 'grey' 'green' are overridden for custom style. + """ def __init__(self, message: str, border_color: str) -> None: + """Initialise the MessageBox. + + Parameters + ---------- + message + Message to display on the MessageBox. + + border_color + Color of the MessageBox border (e.g. green if the message is positive). + + """ super(MessageBox, self).__init__() self.message = message self.border_color = border_color def compose(self) -> ComposeResult: + """Add widgets to the MessageBox.""" yield Container( Container( Static(self.message, id="messagebox_message_label"), @@ -64,6 +78,7 @@ def compose(self) -> ComposeResult: ) def on_mount(self) -> None: + """Update widgets immediately after mounting.""" if self.border_color == "red": color = "rgb(140, 12, 0)" elif self.border_color == "green": @@ -79,12 +94,12 @@ def on_mount(self) -> None: ) def on_button_pressed(self) -> None: + """Handle button press.""" self.dismiss(True) class ConfirmAndAwaitTransferPopup(ModalScreen): - """ - A popup screen for confirming, awaiting and finishing a Transfer. + """A popup screen for confirming, awaiting and finishing a Transfer. When users select Transfer, this screen pops up to a) allow users to confirm transfer b) display a `LoadingIndicator` while the transfer runs in a separate worker c) indicate the transfer is finished. @@ -96,12 +111,24 @@ def __init__( message: str, transfer_func: Callable[[], Worker[InterfaceOutput]], ) -> None: + """Initialise the ConfirmAndAwaitTransferPopup. + + Parameters + ---------- + message + Message to display while running the transfer. + + transfer_func + Function to run in a worker that performs the transfer. + + """ super().__init__() self.transfer_func = transfer_func self.message = message def compose(self) -> ComposeResult: + """Add widgets to the ConfirmAndAwaitTransferPopup.""" yield Container( Label(self.message, id="confirm_message_label"), Horizontal( @@ -113,6 +140,7 @@ def compose(self) -> ComposeResult: ) def on_button_pressed(self, event: Button.Pressed) -> None: + """Handle button press on the ConfirmAndAwaitTransferPopup.""" if event.button.id == "confirm_ok_button": self.query_one("#confirm_button_container").remove() @@ -129,8 +157,7 @@ def on_button_pressed(self, event: Button.Pressed) -> None: self.dismiss() async def handle_transfer_and_update_ui_when_complete(self) -> None: - """Runs the data transfer worker and updates the UI on completion""" - + """Run the data transfer worker and updates the UI on completion.""" data_transfer_worker = self.transfer_func() await data_transfer_worker.wait() success, output = data_transfer_worker.result @@ -151,19 +178,23 @@ async def handle_transfer_and_update_ui_when_complete(self) -> None: class SearchingCentralForNextSubSesPopup(ModalScreen): - """ - A popup to show message and a loading indicator when awaiting search next sub/ses across - the folders present in both local and central machines. This search happens in a separate - thread so as to allow TUI to display the loading indicate without freezing. + """Show message and a loading indicator for suggesting the next subject or session. + + Used to await searching next sub/ses across including folders + present on the central machine. This search happens in a separate + thread to allow TUI to display the loading indicate without freezing. - Only displayed when the `include_central` flag is checked and the connection method is "ssh". + Only displayed when the `include_central` flag is checked and the + connection method is "ssh". """ def __init__(self, sub_or_ses: Prefix) -> None: + """Initialise SearchingCentralForNextSubSesPopup.""" super().__init__() self.message = f"Searching central for next {sub_or_ses}" def compose(self) -> ComposeResult: + """Add widgets to SearchingCentralForNextSubSesPopup.""" yield Container( Label(self.message, id="searching_message_label"), LoadingIndicator(id="searching_animated_indicator"), @@ -172,26 +203,36 @@ def compose(self) -> ComposeResult: class SelectDirectoryTreeScreen(ModalScreen): - """ - A modal screen that includes a DirectoryTree to browse - and select folders. If a folder is double-clicked, - the path to the folder is returned through 'dismiss' - callback mechanism. + """Screen that includes a DirectoryTree to browse and select folders. + + If a folder is double-clicked, the path to the folder is + returned through 'dismiss' callback mechanism. Parameters ---------- - - mainwindow : TuiApp + mainwindow Textual main app screen - path_ : Optional[Path] + path_ Path to use as the DirectoryTree root, if `None` set to the system user home. + """ def __init__( self, mainwindow: TuiApp, path_: Optional[Path] = None ) -> None: + """Initialise SelectDirectoryTreeScreen. + + Parameters + ---------- + mainwindow + The main TUI app. + + path_ + Root path for the DirectoryTree. + + """ super(SelectDirectoryTreeScreen, self).__init__() self.mainwindow = mainwindow @@ -202,7 +243,7 @@ def __init__( self.click_info = ClickInfo() def compose(self) -> ComposeResult: - + """Add widgets to the SelectDirectoryTreeScreen.""" label_message = ( "Select (double click) a folder with the same name as the project.\n" "If the project folder does not exist, select the parent folder and it will be created." @@ -226,9 +267,9 @@ def compose(self) -> ComposeResult: ) @staticmethod - def get_drives(): - """ - Get drives available on the machine to switch between. + def get_drives() -> list[str]: + """Return drives available on the machine to switch between. + For Windows, use `psutil` to get the list of drives. Otherwise, assume root is "/" and take all folders from that level. """ @@ -248,49 +289,65 @@ def get_drives(): f"/{dir.name}" for dir in Path("/").iterdir() if dir.is_dir() ] - def get_selected_drive(self): - """ - Get the default drive which the select starts on. For windows, - use the .drive attribute but for macOS and Linux this is blank. - On these Os use the first folder (e.g. /Users) as the default drive. + def get_selected_drive(self) -> str: + """Return the default drive which the select starts on. + + For windows, use the .drive attribute but for macOS and Linux + this is blank. On these Os use the first folder (e.g. /Users) + as the default drive. """ if platform.system() == "Windows": selected_drive = f"{self.path_.drive}\\" else: selected_drive = f"/{self.path_.parts[1]}" + return selected_drive def on_select_changed(self, event: Select.Changed) -> None: - """Updates the directory tree when the drive is changed.""" + """Update the directory tree when the drive is changed.""" self.path_ = Path(event.value) - self.query_one("#select_directory_tree_directory_tree").path = ( - self.path_ - ) + self.query_one( + "#select_directory_tree_directory_tree" + ).path = self.path_ @require_double_click def on_directory_tree_directory_selected( self, event: DirectoryTree.DirectorySelected ) -> None: + """Handle a node on the DirectoryTree selected.""" if event.path.is_file(): return else: self.dismiss(event.path) def on_button_pressed(self, event: Button.Pressed) -> None: + """Handle cancel button pressed.""" if event.button.id == "cancel_button": self.dismiss(False) class RenameFileOrFolderScreen(ModalScreen): - """ """ + """A screen to handle the renaming of a file or folder selected through the DirectoryTree.""" def __init__(self, mainwindow: TuiApp, path_: Path) -> None: + """Initialise RenameFileOrFolderScreen. + + Parameters + ---------- + mainwindow + The main TUI app. + + path_ + Path of the file or folder to rename. + + """ super(RenameFileOrFolderScreen, self).__init__() self.mainwindow = mainwindow self.path_ = path_ def compose(self) -> ComposeResult: + """Add widgets to the RenameFileOrFolderScreen.""" yield Container( Label("Input the new name:", id="rename_screen_label"), Input(value=self.path_.stem, id="rename_screen_input"), @@ -303,7 +360,7 @@ def compose(self) -> ComposeResult: ) def on_button_pressed(self, event: Button.Pressed) -> None: - """""" + """Handle button pressed on the RenameFileOrFolderScreen.""" if event.button.id == "rename_screen_okay_button": self.dismiss(self.query_one("#rename_screen_input").value) diff --git a/datashuttle/tui/screens/new_project.py b/datashuttle/tui/screens/new_project.py index 006e6bcbd..38bb51a93 100644 --- a/datashuttle/tui/screens/new_project.py +++ b/datashuttle/tui/screens/new_project.py @@ -14,10 +14,7 @@ class NewProjectScreen(Screen): - """ - Screen for setting up a new datashuttle project, by - inputting the desired configs. This uses the - ConfigsContent window to display and set the configs. + """Screen for setting up a new datashuttle project. If "Main Manu" button is pressed, the callback function returns None, so the project screen is not switched to. @@ -26,22 +23,24 @@ class NewProjectScreen(Screen): project is in ConfigsContent. ConfigsContent calls the dismiss method of this class to return an initialised project to mainwindow. - See ConfigsContent.on_button_pressed() for more details Parameters ---------- + mainwindow + The main TUI app - mainwindow : TuiApp """ TITLE = "Make New Project" def __init__(self, mainwindow: TuiApp) -> None: + """Initialise the NewProjectScreen.""" super(NewProjectScreen, self).__init__() self.mainwindow = mainwindow def compose(self) -> ComposeResult: + """Add widgets to the NewProjectScreen.""" yield Header() yield Button("Main Menu", id="all_main_menu_buttons") yield configs_content.ConfigsContent( @@ -49,5 +48,6 @@ def compose(self) -> ComposeResult: ) def on_button_pressed(self, event: Button.Pressed) -> None: + """Handle a button press on the NewProjectScreen.""" if event.button.id == "all_main_menu_buttons": self.dismiss(None) diff --git a/datashuttle/tui/screens/project_manager.py b/datashuttle/tui/screens/project_manager.py index e25c6f5fa..8de148106 100644 --- a/datashuttle/tui/screens/project_manager.py +++ b/datashuttle/tui/screens/project_manager.py @@ -23,10 +23,10 @@ class ProjectManagerScreen(Screen): - """ - Screen containing the Create, Transfer and Configs tabs. This is - the primary screen within which the user interacts with - a pre-configured project. + """Screen containing the Create, Transfer and Configs tabs. + + This is the primary screen within which the user interacts + with a pre-configured project. The 'Create' tab interacts with Datashuttle's `create_folders()` method to create new project folders. @@ -43,6 +43,7 @@ class ProjectManagerScreen(Screen): """ def __init__(self, mainwindow: TuiApp, interface: Interface, id) -> None: + """Initialise the ProjectManagerScreen.""" super(ProjectManagerScreen, self).__init__(id=id) self.mainwindow = mainwindow @@ -53,6 +54,7 @@ def __init__(self, mainwindow: TuiApp, interface: Interface, id) -> None: self.tabbed_content_mount_signal = True def compose(self) -> ComposeResult: + """Add widgets to the ProjectManagerScreen.""" yield Header() yield Button("Main Menu", id="all_main_menu_buttons") with TabbedContent( @@ -90,23 +92,21 @@ def compose(self) -> ComposeResult: ) def on_button_pressed(self, event: Button.Pressed) -> None: - """ - Dismisses the TabScreen (and returns to the main menu) once - the 'Main Menu' button is pressed. - """ + """Dismiss the TabScreen and return to the main menu.""" if event.button.id == "all_main_menu_buttons": self.dismiss() def on_tabbed_content_tab_activated( self, event: TabbedContent.TabActivated ) -> None: - """ - Refresh the directorytree for create or transfer tabs whenever - the tabbedcontent is switched to one of these tabs. + """Handle a tab switch. + + Refresh the DirectoryTree for create or transfer tabs whenever + the TabbedContent is switched to one of these tabs. This is also triggered on mount, leading to it being reloaded twice, leading to a strange flicker. Ideally no trigger - would be sent on mount. Therefore the ugly `tabbed_content_mount_signal` + would be sent on mount. Therefore, the ugly `tabbed_content_mount_signal` variable is introduced to track this. """ if self.tabbed_content_mount_signal: @@ -125,12 +125,14 @@ def on_tabbed_content_tab_activated( "#tabscreen_logging_tab" ).update_most_recent_label() - def update_active_tab_tree(self): + def update_active_tab_tree(self) -> None: + """Reload the CustomDirectoryTree on the now-active tab.""" active_tab_id = self.query_one("#tabscreen_tabbed_content").active self.query_one(f"#{active_tab_id}").reload_directorytree() def on_configs_content_configs_saved(self) -> None: - """ + """Handle saving of the configs tab. + When configs are saved, we may switch between a 'full' project and a 'local only' project (no `central_path` or `connection_method` set). In such a case we need to refresh the ProjectManager screen to add / remove @@ -153,7 +155,6 @@ def on_configs_content_configs_saved(self) -> None: ) if old_project_type == project_type: - if project_type == "full": self.query_one( "#tabscreen_transfer_tab" @@ -171,9 +172,10 @@ def on_configs_content_configs_saved(self) -> None: self.wrap_dismiss, ) - def wrap_dismiss(self, _): - """ - Need to wrap dismiss as cannot include it directly - in push_screen callback, or even wrapped in lambda. + def wrap_dismiss(self, _) -> None: + """Wrap the dismiss function for push screen callbacks. + + Need to wrap dismiss as cannot include it directly in + push_screen callback, or even wrapped in lambda. """ self.dismiss() diff --git a/datashuttle/tui/screens/project_selector.py b/datashuttle/tui/screens/project_selector.py index 2cb43f205..a3fb00afc 100644 --- a/datashuttle/tui/screens/project_selector.py +++ b/datashuttle/tui/screens/project_selector.py @@ -18,9 +18,9 @@ class ProjectSelectorScreen(Screen): - """ - The project selection screen. Finds and displays DataShuttle - projects present on the local system. + """The project selection screen. + + Finds and displays DataShuttle projects present on the local system. `self.dismiss()` returns an initialised project if initialisation was successful. Otherwise, in case `Main Menu` button is pressed, @@ -28,8 +28,7 @@ class ProjectSelectorScreen(Screen): Parameters ---------- - - mainwindow : TuiApp + mainwindow The main TUI app, functions on which are used to coordinate screen display. @@ -38,6 +37,7 @@ class ProjectSelectorScreen(Screen): TITLE = "Select Project" def __init__(self, mainwindow: TuiApp) -> None: + """Initialise the ProjectSelectorScreen.""" super(ProjectSelectorScreen, self).__init__() self.project_names = [ @@ -46,6 +46,7 @@ def __init__(self, mainwindow: TuiApp) -> None: self.mainwindow = mainwindow def compose(self) -> ComposeResult: + """Add widgets to the ProjectSelectorScreen.""" yield Header(id="project_select_header") yield Button("Main Menu", id="all_main_menu_buttons") yield Container( @@ -54,8 +55,8 @@ def compose(self) -> ComposeResult: ) def on_button_pressed(self, event: Button.Pressed) -> None: + """Handle a button press on ProjectSelectorScreen.""" if event.button.id in self.project_names: - project_name = event.button.id interface = Interface() diff --git a/datashuttle/tui/screens/settings.py b/datashuttle/tui/screens/settings.py index b9a53cf9d..c8b018bde 100644 --- a/datashuttle/tui/screens/settings.py +++ b/datashuttle/tui/screens/settings.py @@ -20,20 +20,21 @@ class SettingsScreen(ModalScreen): - """ - Screen accessible from the main window that contains - 'global' settings for the TUI. 'Global' settings are non-project - specific settings (e.g. dark mode) and are handled independently - of the main datashuttle API. + """Screen accessible that contains 'global' settings for the TUI. + + 'Global' settings are non-project specific settings (e.g. dark mode) + and are handled independently of the main datashuttle API. """ def __init__(self, mainwindow: TuiApp) -> None: + """Initialise the SettingsScreen.""" super(SettingsScreen, self).__init__() self.mainwindow = mainwindow self.global_settings = self.mainwindow.load_global_settings() def compose(self) -> ComposeResult: + """Add widgets to the SettingsScreen.""" dark_mode = self.global_settings["dark_mode"] yield Container( RadioSet( @@ -59,11 +60,12 @@ def compose(self) -> ComposeResult: ) def on_mount(self) -> None: - """""" + """Update widgets immediately after they have been mounted.""" id = "#show_transfer_tree_status_checkbox" self.query_one(id).tooltip = get_tooltip(id) def on_radio_set_changed(self, event: RadioSet.Changed) -> None: + """Handle a radio set widget changed on SettingsScreen.""" label = str(event.pressed.label) assert label in ["Light Mode", "Dark Mode"] @@ -74,9 +76,11 @@ def on_radio_set_changed(self, event: RadioSet.Changed) -> None: self.mainwindow.save_global_settings(self.global_settings) def on_checkbox_changed(self, event: Checkbox.Changed) -> None: + """Handle checkbox changed on SettingsScreen.""" self.global_settings["show_transfer_tree_status"] = event.value self.mainwindow.save_global_settings(self.global_settings) def on_button_pressed(self, event: Button.Pressed) -> None: + """Handle button pressed on SettingsScreen.""" if event.button.id == "all_main_menu_buttons": self.dismiss() diff --git a/datashuttle/tui/screens/setup_ssh.py b/datashuttle/tui/screens/setup_ssh.py index 296865f2a..e02915699 100644 --- a/datashuttle/tui/screens/setup_ssh.py +++ b/datashuttle/tui/screens/setup_ssh.py @@ -18,11 +18,10 @@ class SetupSshScreen(ModalScreen): - """ - This dialog windows handles the TUI equivalent of API's - setup_ssh_connection(). This asks to - confirm the central hostkey, and takes password to setup - SSH key pair. + """Dialog window that sets up an SSH connection. + + This asks to confirm the central hostkey, and takes password to setup + SSH key pair. Under the hood uses `project.setup_ssh_connection()`. This is the one instance in which it is not possible for the TUI to nearly wrap the API, because the logic flow is @@ -30,6 +29,7 @@ class SetupSshScreen(ModalScreen): """ def __init__(self, interface: Interface) -> None: + """Initialise the SetupSshScreen.""" super(SetupSshScreen, self).__init__() self.interface = interface @@ -39,6 +39,7 @@ def __init__(self, interface: Interface) -> None: self.key: paramiko.RSAKey def compose(self) -> ComposeResult: + """Add widgets to the SetupSshScreen.""" yield Container( Horizontal( Static( @@ -57,10 +58,12 @@ def compose(self) -> ComposeResult: ) def on_mount(self) -> None: + """Update widgets immediately after they are mounted.""" self.query_one("#setup_ssh_password_input").visible = False def on_button_pressed(self, event: Button.pressed) -> None: - """ + """Handle button press on the SetupSshScreen. + When each stage is successfully progressed by clicking the "ok" button, `self.stage` is iterated by 1. For saving and excepting hostkey, if there is a problem (error or user declines) the 'OK' button @@ -84,8 +87,8 @@ def on_button_pressed(self, event: Button.pressed) -> None: self.dismiss() def ask_user_to_accept_hostkeys(self) -> None: - """ - The central server is identified by a hostkey. + """Ask the user to accept the hostkey that identifies the central server. + Get this hostkey and present it to user, clicking 'OK' is they are happy. If there is an error, block process (because it most likely is necessary to edit the central host id) and @@ -116,10 +119,10 @@ def ask_user_to_accept_hostkeys(self) -> None: self.stage += 1 def save_hostkeys_and_prompt_password_input(self) -> None: - """ - Once the hostkey is accepted, get the user password - for the central server. When 'OK' is pressed we go - straight to 'use_password_to_setup_ssh_key_pairs'. + """Get the user password for the central server. + + When 'OK' is pressed we go straight to + 'use_password_to_setup_ssh_key_pairs'. """ success, output = self.interface.save_hostkey_locally(self.key) @@ -141,10 +144,10 @@ def save_hostkeys_and_prompt_password_input(self) -> None: self.stage += 1 def use_password_to_setup_ssh_key_pairs(self) -> None: - """ - Get the user password for the central server. If correct, - SSH key pair is setup and 'OK' button changed to 'Finish'. - Otherwise, continue allowing failed password attempts. + """Get the user password for the central server. + + If correct, SSH key pair is set up and 'OK' button changed + to 'Finish'. Otherwise, continue allowing failed password attempts. """ password = self.query_one("#setup_ssh_password_input").value diff --git a/datashuttle/tui/screens/validate_at_path.py b/datashuttle/tui/screens/validate_at_path.py index b4157add1..2b88e087e 100644 --- a/datashuttle/tui/screens/validate_at_path.py +++ b/datashuttle/tui/screens/validate_at_path.py @@ -14,9 +14,8 @@ class ValidateScreen(Screen): - """ - Screen to hold the validation window for - validating an existing project at a given path. + """Screen to the validate project from path window. + All widgets are stored in `ValidateContent`, which is shared between here and the validation tab on the project manager. """ @@ -24,11 +23,13 @@ class ValidateScreen(Screen): TITLE = "Validate Project" def __init__(self, mainwindow: TuiApp) -> None: + """Initialise the ValidateScreen.""" super(ValidateScreen, self).__init__() self.mainwindow = mainwindow def compose(self) -> ComposeResult: + """Add widgets to the ValidateScreen.""" yield Header() yield Button("Main Menu", id="all_main_menu_buttons") yield validate_content.ValidateContent( @@ -36,5 +37,6 @@ def compose(self) -> ComposeResult: ) def on_button_pressed(self, event: Button.Pressed) -> None: + """Handle button press on the ValidateScreen.""" if event.button.id == "all_main_menu_buttons": self.dismiss(None) diff --git a/datashuttle/tui/shared/configs_content.py b/datashuttle/tui/shared/configs_content.py index 974ee08af..be3b3444e 100644 --- a/datashuttle/tui/shared/configs_content.py +++ b/datashuttle/tui/shared/configs_content.py @@ -30,8 +30,8 @@ class ConfigsContent(Container): - """ - This screen holds widgets and logic for setting datashuttle configs. + """Holds widgets and logic for setting datashuttle configs. + It is used in `NewProjectPage` to instantiate a new project and initialise configs, or in `TabbedContent` to update an existing project's configs. @@ -41,10 +41,21 @@ class ConfigsContent(Container): additional information. Otherwise, widgets are filled with the existing projects configs. + + Note: + ---- + The natural design would be to have two classes (one for a new project, + one for an existing project) with a shared base class. The issue is that + for a new project, the screen becomes a set-up project once the config is saved. + Therefore, because the class mutates from a new-project config to existing-project + configs, this shared design is used. + """ @dataclass class ConfigsSaved(Message): + """An event signalling when the configs are saved.""" + pass def __init__( @@ -53,6 +64,20 @@ def __init__( interface: Optional[Interface], id: str, ) -> None: + """Initialise the ConfigsContent. + + Parameters + ---------- + parent_class + The Screen on which the contents container is mounted. + + interface + Datashuttle Interface object. + + id + Textual ID for the configs container. + + """ super(ConfigsContent, self).__init__(id=id) self.parent_class = parent_class @@ -60,11 +85,12 @@ def __init__( self.config_ssh_widgets: List[Any] = [] def compose(self) -> ComposeResult: - """ + """Set up the Configs widgets. + `self.config_ssh_widgets` are SSH-setup related widgets that are only required when the user selects the SSH connection method. These are displayed / hidden based on the - `connection_method` + `connection_method`. `config_screen_widgets` are core config-related widgets that are always displayed. @@ -169,7 +195,8 @@ def compose(self) -> ComposeResult: yield Container(*config_screen_widgets, id="configs_container") def on_mount(self) -> None: - """ + """Handle logic immediately following widget mounting. + When we have mounted the widgets, the following logic depends on whether we are setting up a new project (`self.project is `None`) or have an instantiated project. @@ -185,13 +212,13 @@ def on_mount(self) -> None: if self.interface: self.fill_widgets_with_project_configs() else: - self.query_one("#configs_local_filesystem_radiobutton").value = ( - True - ) + self.query_one( + "#configs_local_filesystem_radiobutton" + ).value = True self.switch_ssh_widgets_display(display_ssh=False) - self.query_one("#configs_setup_ssh_connection_button").visible = ( - False - ) + self.query_one( + "#configs_setup_ssh_connection_button" + ).visible = False # Setup tooltips if not self.interface: @@ -222,9 +249,7 @@ def on_mount(self) -> None: self.query_one(id).tooltip = get_tooltip(id) def on_radio_set_changed(self, event: RadioSet.Changed) -> None: - """ - Update the displayed SSH widgets when the `connection_method` - radiobuttons are changed. + """Update the SSH widgets when the `connection_method` radiobuttons are changed. When SSH is set, ssh config-setters are shown. Otherwise, these are hidden. @@ -242,25 +267,22 @@ def on_radio_set_changed(self, event: RadioSet.Changed) -> None: if label == "No connection (local only)": self.query_one("#configs_central_path_input").value = "" self.query_one("#configs_central_path_input").disabled = True - self.query_one("#configs_central_path_select_button").disabled = ( - True - ) + self.query_one( + "#configs_central_path_select_button" + ).disabled = True display_ssh = False else: self.query_one("#configs_central_path_input").disabled = False - self.query_one("#configs_central_path_select_button").disabled = ( - False - ) + self.query_one( + "#configs_central_path_select_button" + ).disabled = False display_ssh = True if label == "SSH" else False self.switch_ssh_widgets_display(display_ssh) self.set_central_path_input_tooltip(display_ssh) def set_central_path_input_tooltip(self, display_ssh: bool) -> None: - """ - Use a different tooltip depending on whether connection method - is ssh or local filesystem. - """ + """Set tooltip depending on whether connection method is SSH or local filesystem.""" id = "#configs_central_path_input" if display_ssh: self.query_one(id).tooltip = get_tooltip( @@ -274,7 +296,17 @@ def set_central_path_input_tooltip(self, display_ssh: bool) -> None: def get_platform_dependent_example_paths( self, local_or_central: Literal["local", "central"], ssh: bool = False ) -> str: - """ """ + """Get example paths for the local or central Inputs depending on operating system. + + Parameters + ---------- + local_or_central + The "local" or "central" input to fill. + + ssh + If the user has selected SSH (which changes the central input). + + """ assert local_or_central in ["local", "central"] # Handle the ssh central case separately @@ -292,45 +324,47 @@ def get_platform_dependent_example_paths( return example_path def switch_ssh_widgets_display(self, display_ssh: bool) -> None: - """ - Show or hide SSH-related configs based on whether the current - `connection_method` widget is "ssh" or "local_filesystem". + """Show or hide SSH-related configs. + + This is based on whether the current `connection_method` + widget is "ssh" or "local_filesystem". Parameters ---------- - - display_ssh : bool + display_ssh If `True`, display the SSH-related widgets. + """ for widget in self.config_ssh_widgets: widget.display = display_ssh - self.query_one("#configs_central_path_select_button").display = ( - not display_ssh - ) + self.query_one( + "#configs_central_path_select_button" + ).display = not display_ssh if self.interface is None: - self.query_one("#configs_setup_ssh_connection_button").visible = ( - False - ) + self.query_one( + "#configs_setup_ssh_connection_button" + ).visible = False else: - self.query_one("#configs_setup_ssh_connection_button").visible = ( - display_ssh - ) + self.query_one( + "#configs_setup_ssh_connection_button" + ).visible = display_ssh if not self.query_one("#configs_central_path_input").value: if display_ssh: placeholder = f"e.g. {self.get_platform_dependent_example_paths('central', ssh=True)}" else: placeholder = f"e.g. {self.get_platform_dependent_example_paths('central', ssh=False)}" - self.query_one("#configs_central_path_input").placeholder = ( - placeholder - ) + self.query_one( + "#configs_central_path_input" + ).placeholder = placeholder def on_button_pressed(self, event: Button.Pressed) -> None: - """ - Enables the Create Folders button to read out current input values - and use these to call project.create_folders(). + """Handle a button press event. + + Enables the Create Folders button to read out current input + values and use these to call project.create_folders(). """ if event.button.id == "configs_save_configs_button": if not self.interface: @@ -366,36 +400,32 @@ def on_button_pressed(self, event: Button.Pressed) -> None: def handle_input_fill_from_select_directory( self, path_: Path, local_or_central: Literal["local", "central"] ) -> None: - """ - Update the `local` or `central` path inputs after - `SelectDirectoryTreeScreen` returns a path. + """Update the `local` or `central` Inputs after `SelectDirectoryTreeScreen` returns a path. Parameters ---------- - - path_ : Union[Literal[False], Path] + path_ The path returned from `SelectDirectoryTreeScreen`. If `False`, the screen exited with no directory selected. - local_or_central : str + local_or_central The Input to fill with the path. + """ if path_ is False: return if local_or_central == "local": - self.query_one("#configs_local_path_input").value = ( - path_.as_posix() - ) + self.query_one( + "#configs_local_path_input" + ).value = path_.as_posix() elif local_or_central == "central": - self.query_one("#configs_central_path_input").value = ( - path_.as_posix() - ) + self.query_one( + "#configs_central_path_input" + ).value = path_.as_posix() def setup_ssh_connection(self) -> None: - """ - Set up the `SetupSshScreen` screen, - """ + """Set up the `SetupSshScreen` screen.""" assert self.interface is not None, "type narrow flexible `interface`" if not self.widget_configs_match_saved_configs(): @@ -410,10 +440,9 @@ def setup_ssh_connection(self) -> None: ) def widget_configs_match_saved_configs(self): - """ - Check that the configs currently stored in the widgets - on the screen match those stored in the app. This check - is to avoid user starting to set up SSH with unexpected + """Ensure configs as set on screen match those stored in the project object. + + This check is to avoid user starting to set up SSH with unexpected settings. It is a little fiddly as the Input for local and central path may or may not contain the project name. Therefore, need to check the stored values against @@ -433,7 +462,8 @@ def widget_configs_match_saved_configs(self): return True def setup_configs_for_a_new_project(self) -> None: - """ + """Set up configs when the project does not exist. + If a project does not exist, we are in NewProjectScreen. We need to instantiate a new project based on the project name, create configs based on the current widget settings, and display @@ -453,17 +483,15 @@ def setup_configs_for_a_new_project(self) -> None: success, output = interface.setup_new_project(project_name, cfg_kwargs) if success: - self.interface = interface - self.query_one("#configs_go_to_project_screen_button").visible = ( - True - ) + self.query_one( + "#configs_go_to_project_screen_button" + ).visible = True # Could not find a neater way to combine the push screen # while initiating the callback in one case but not the other. if cfg_kwargs["connection_method"] == "ssh": - self.query_one( "#configs_setup_ssh_connection_button" ).visible = True @@ -495,7 +523,8 @@ def setup_configs_for_a_new_project(self) -> None: self.parent_class.mainwindow.show_modal_error_dialog(output) def setup_configs_for_an_existing_project(self) -> None: - """ + """Set up configs when the project already exists. + If the project already exists, we are on the TabbedContent screen. We need to get the configs to set from the current widget values and display the set values (or an error if @@ -524,7 +553,8 @@ def setup_configs_for_an_existing_project(self) -> None: self.parent_class.mainwindow.show_modal_error_dialog(output) def fill_widgets_with_project_configs(self) -> None: - """ + """Fill widgets on screen with content from the project config file. + If a configured project already exists, we want to fill the widgets with the current project configs. This in some instances requires recasting to a new type of changing the value. @@ -587,10 +617,7 @@ def fill_widgets_with_project_configs(self) -> None: input.value = value def get_datashuttle_inputs_from_widgets(self) -> Dict: - """ - Get the configs to pass to `make_config_file()` from - the current TUI settings. - """ + """Get the configs to pass to `make_config_file()` from the current TUI settings.""" cfg_kwargs: Dict[str, Any] = {} cfg_kwargs["local_path"] = Path( diff --git a/datashuttle/tui/shared/validate_content.py b/datashuttle/tui/shared/validate_content.py index a8cce2747..bbf53f65d 100644 --- a/datashuttle/tui/shared/validate_content.py +++ b/datashuttle/tui/shared/validate_content.py @@ -27,6 +27,19 @@ class ValidateContent(Container): + """A container containing widgets for project validation. + + This is shared between the Validate Project from Path + and validation tab on the project manager. It takes a similar + approach to ConfigsContent. + + Todo: + ---- + In contrast to ConfigsContent, these contents are always + split. This should probably be factored into two classes + with a shared base class. + + """ def __init__( self, @@ -36,13 +49,24 @@ def __init__( interface: Optional[Interface], id: str, ) -> None: + """Initialise the ValidateContent class. + + parent_class + The parent screen (project manager or validate from path). + + interface + Datashuttle Interface class. + + id + Textual ID for the container. + """ super(ValidateContent, self).__init__(id=id) self.parent_class = parent_class self.interface = interface def compose(self) -> ComposeResult: - + """Set up the widgets for the container.""" if platform.system() == "Windows": example_path = r"C:\path\to\project\project_name" else: @@ -89,7 +113,7 @@ def compose(self) -> ComposeResult: yield Container(*widgets, id="validate_top_container") def on_mount(self) -> None: - """ """ + """Handle the widgets immediately after they are mounted.""" for id in [ "validate_path_input", "validate_top_level_folder_select", @@ -112,14 +136,9 @@ def on_mount(self) -> None: else: self.query_one("#validate_include_central_checkbox").remove() - def set_select_path(self, path_): - if path_: - self.query_one("#validate_path_input").value = path_.as_posix() - def on_button_pressed(self, event: Button.Pressed) -> None: - + """Handle a button press event.""" if event.button.id == "validate_select_button": - self.parent_class.mainwindow.push_screen( modal_dialogs.SelectDirectoryTreeScreen( self.parent_class.mainwindow @@ -128,7 +147,6 @@ def on_button_pressed(self, event: Button.Pressed) -> None: ) elif event.button.id == "validate_validate_button": - select_value = self.query_one( "#validate_top_level_folder_select" ).value @@ -140,7 +158,6 @@ def on_button_pressed(self, event: Button.Pressed) -> None: # assert False, f"strict mode: {strict_mode}" if self.interface: - if self.interface.project.is_local_project(): include_central = False else: @@ -159,11 +176,10 @@ def on_button_pressed(self, event: Button.Pressed) -> None: ) else: self.write_results_to_richlog(output) - self.query_one("#validate_logs_label").value = ( - f"Logs output to: {self.interface.project.get_logging_path()}" - ) + self.query_one( + "#validate_logs_label" + ).value = f"Logs output to: {self.interface.project.get_logging_path()}" else: - path_ = self.query_one("#validate_path_input").value if path_ == "": @@ -185,7 +201,13 @@ def on_button_pressed(self, event: Button.Pressed) -> None: ) self.write_results_to_richlog(output) + def set_select_path(self, path_): + """Fill the Input with the path_ if it is not None.""" + if path_: + self.query_one("#validate_path_input").value = path_.as_posix() + def write_results_to_richlog(self, results): + """Display the validation results on the Rich Log widget.""" text_log = self.query_one("#validate_richlog") text_log.clear() if any(results): diff --git a/datashuttle/tui/tabs/create_folders.py b/datashuttle/tui/tabs/create_folders.py index 036a61dec..2ec76e728 100644 --- a/datashuttle/tui/tabs/create_folders.py +++ b/datashuttle/tui/tabs/create_folders.py @@ -44,11 +44,20 @@ class CreateFoldersTab(TreeAndInputTab): - """ - Create new project files formatted according to the NeuroBlueprint specification. - """ + """Create new project files formatted according to the NeuroBlueprint specification.""" def __init__(self, mainwindow: TuiApp, interface: Interface) -> None: + """Initialise the CreateFoldersTab. + + Parameters + ---------- + mainwindow + The main TUI application. + + interface + Datashuttle Interface object. + + """ super(CreateFoldersTab, self).__init__( "Create", id="tabscreen_create_tab" ) @@ -61,6 +70,7 @@ def __init__(self, mainwindow: TuiApp, interface: Interface) -> None: self.click_info = ClickInfo() def compose(self) -> ComposeResult: + """Add widgets to the Create Folders tab.""" yield CustomDirectoryTree( self.mainwindow, self.interface.get_configs()["local_path"], @@ -105,7 +115,7 @@ def compose(self) -> ComposeResult: ) def on_mount(self) -> None: - """""" + """Handle the widgets immediately after mounting.""" if not self.interface: self.query_one("#configs_name_input").tooltip = get_tooltip( "#configs_name_input" @@ -122,18 +132,19 @@ def on_mount(self) -> None: self.query_one(id).tooltip = get_tooltip(id) def on_button_pressed(self, event: Button.Pressed) -> None: - """ - Enables the Create Folders button to read out current input values + """Handle a button press event. + + The Create Folders button to read out current Input values and use these to call project.create_folders(). - `unused_bool` is necessary to get dismiss to call - the callback. + `unused_bool` is necessary as dismiss() on the pushed screen + returns a bool. We use this to trigger revalidation regardless + of the bool state. """ if event.button.id == "create_folders_create_folders_button": self.create_folders() elif event.button.id == "create_folders_displayed_datatypes_button": - self.mainwindow.push_screen( DisplayedDatatypesScreen("create", self.interface), self.refresh_after_datatypes_changed, @@ -145,7 +156,8 @@ def on_button_pressed(self, event: Button.Pressed) -> None: lambda unused_bool: self.revalidate_inputs(["sub", "ses"]), ) - async def refresh_after_datatypes_changed(self, ignore): + async def refresh_after_datatypes_changed(self, ignore) -> None: + """Redisplay the datatype checkboxes.""" await self.recompose() self.on_mount() @@ -153,10 +165,9 @@ async def refresh_after_datatypes_changed(self, ignore): def on_clickable_input_clicked( self, event: ClickableInput.Clicked ) -> None: - """ - Handled a double click on the custom ClickableInput widget, - which indicates the input should be filled with a suggested value. + """Handle a double click on the custom ClickableInput widget. + A double click indicates the input should be filled with a suggested value. Determine if we have the subject or session input, and if it was a left or right click. Then, fill with either a generic suggestion or suggestion based on next sub / ses number. @@ -176,11 +187,11 @@ def on_clickable_input_clicked( def on_custom_directory_tree_directory_tree_special_key_press( self, event: CustomDirectoryTree.DirectoryTreeSpecialKeyPress - ): - """ - Handle a key press on the directory tree, which can refresh the - directorytree or fill / append subject/session folder name to - the relevant input widget. + ) -> None: + """Handle a key press on the CustomDirectoryTree. + + This can refresh the CustomDirectoryTree or fill / append + subject/session folder name to the relevant input widget. """ if event.key == "ctrl+r": self.reload_directorytree() @@ -196,10 +207,18 @@ def on_custom_directory_tree_directory_tree_special_key_press( self.mainwindow.prompt_rename_file_or_folder(event.node_path) def fill_input_with_template(self, prefix: Prefix, input_id: str) -> None: - """ - Given the `name_template`, fill the sub or ses - Input with the template (based on `prefix`). + """Fill the sub or ses Input with the name template. + If `self.templates` is off, then just suggest "sub-" or "ses-". + + Parameters + ---------- + prefix + "sub" or "ses" + + input_id + Textual ID of the Input to fill. + """ if self.templates_on(prefix): fill_value = self.interface.get_name_templates()[prefix] @@ -210,6 +229,7 @@ def fill_input_with_template(self, prefix: Prefix, input_id: str) -> None: input.value = fill_value def templates_on(self, prefix: Prefix) -> bool: + """Return `True` if the name templates are used in the project.""" return ( self.interface.get_name_templates()["on"] and self.interface.get_name_templates()[prefix] is not None @@ -219,10 +239,7 @@ def templates_on(self, prefix: Prefix) -> bool: # ---------------------------------------------------------------------------------- def revalidate_inputs(self, all_prefixes: List[str]) -> None: - """ - Revalidate and style both subject and session - inputs based on their value. - """ + """Revalidate and style both subject and session inputs based on their value.""" input_names = { "sub": "#create_folders_subject_input", "ses": "#create_folders_session_input", @@ -233,18 +250,15 @@ def revalidate_inputs(self, all_prefixes: List[str]) -> None: value = self.query_one(key).value self.query_one(key).validate(value=value) - def update_input_tooltip(self, message: List[str], prefix: Prefix) -> None: - """ - Update the value of a subject or session tooltip, which - indicates the validation status of the input value. - """ + def update_input_tooltip(self, message: str, prefix: Prefix) -> None: + """Update the value of a subject or session tooltip indicating the validation status.""" id = ( "#create_folders_subject_input" if prefix == "sub" else "#create_folders_session_input" ) input = self.query_one(id) - input.tooltip = message if any(message) else None + input.tooltip = message # ---------------------------------------------------------------------------------- # Datashuttle Callers @@ -254,10 +268,7 @@ def update_input_tooltip(self, message: List[str], prefix: Prefix) -> None: # ---------------------------------------------------------------------------------- def create_folders(self) -> None: - """ - Create project folders based on current widget input - through the datashuttle API. - """ + """Create project folders based on the information in TUI widgets.""" ses_names: Optional[List[str]] sub_names, ses_names, datatype = self.get_sub_ses_names_and_datatype( @@ -277,10 +288,13 @@ def create_folders(self) -> None: self.mainwindow.show_modal_error_dialog(output) def reload_directorytree(self) -> None: - """ - This reloads the directorytree and also updates validation. - Not now a good method name but done for consistency with other - tab refresh methods. + """Reload the DirectoryTree and also updates validation. + + Notes + ----- + No longer a good method name due to extended responsibilities + but done for consistency with other tab refresh methods. + """ self.revalidate_inputs(["sub", "ses"]) self.query_one("#create_folders_directorytree").reload() @@ -290,14 +304,26 @@ def reload_directorytree(self) -> None: def suggest_next_sub_ses( self, prefix: Prefix, input_id: str, include_central: bool - ): - """ - This handles suggesting next sub/ses for the project. Shows - a pop up screen in cases when searching for next sub/ses takes + ) -> None: + """Suggests the next sub/ses name for the project. + + Shows a pop up screen in cases when searching for next sub/ses takes time such as searching central in SSH connection method. Creates an asyncio task which handles the suggestion logic and dismissing the pop up. + + Parameters + ---------- + prefix + Suggest the next "sub" or "ses". + + input_id + Textual ID of the widget in which to fill the suggested sub or ses. + + include_central + If `True`, search central project as well to generate the suggestion. + """ assert self.interface.project.cfg["connection_method"] in [ None, @@ -323,14 +349,16 @@ def suggest_next_sub_ses( async def fill_suggestion_and_dismiss_popup( self, prefix, input_id, include_central - ): - """ - This handles running the `fill_input_with_next_sub_or_ses_template` - worker and waiting for it to complete. If an error occurs in - `fill_input_with_next_sub_or_ses_template`, it dismisses the popup itself. + ) -> None: + """Run the `fill_input_with_next_sub_or_ses_template` worker. + + Awaits completion. If an error occurs in `fill_input_with_next_sub_or_ses_template`, + it dismisses the popup itself. Else, if the worker successfully exits, this function handles dismissal of the popup. + + see `suggest_next_sub_ses()` for parameters. """ worker = self.fill_input_with_next_sub_or_ses_template( prefix, input_id, include_central @@ -343,26 +371,32 @@ async def fill_suggestion_and_dismiss_popup( @work(exclusive=True, thread=True) def fill_input_with_next_sub_or_ses_template( self, prefix: Prefix, input_id: str, include_central: bool - ) -> Worker: - """ - This fills a sub / ses Input with a suggested name based on the - next subject / session in the project (local). + ) -> Worker[None]: + """Fill an Input the next subject / session in the project (local). If `name_templates` are set, then the sub- or ses- first key of the template name will be replaced with the suggested sub or ses key-value. Otherwise, the sub/ses key-value pair only will be suggested. - It runs in a worker thread so as to allow the TUI to show a loading + It runs in a worker thread as to allow the TUI to show a loading animation. Parameters - - prefix : Prefix + ---------- + prefix Whether to fill the subject or session Input - input_id : str + input_id The textual input name to update. + + include_central + If `True`, the central project is also validated. + + Returns + ------- + A textual Worker object for the thread in which the function is run. + """ top_level_folder = self.interface.tui_settings[ "top_level_folder_select" @@ -424,11 +458,10 @@ def fill_input_with_next_sub_or_ses_template( def dismiss_popup_and_show_modal_error_dialog_from_thread( self, message: str ) -> None: - """ - This is a utility function that the `fill_input_with_next_sub_or_ses_template` - worker calls to display error dialog an if an error occurs while suggesting - the next sub/ses. Handles the TUI widget manipulation from the main thread - when called from within a worker thread. + """Handle an error while suggesting the next session or subject. + + Called by `fill_input_with_next_sub_or_ses_template` to display error + dialog an if an error occurs while suggesting the next sub/ses. """ if self.searching_central_popup_widget: self.mainwindow.call_from_thread( @@ -441,11 +474,8 @@ def dismiss_popup_and_show_modal_error_dialog_from_thread( # Validation # ---------------------------------------------------------------------------------- - def run_local_validation(self, prefix: Prefix): - """ - Run validation of the values stored in the - sub / ses Input according to the passed prefix - using core datashuttle functions. + def run_local_validation(self, prefix: Prefix) -> tuple[bool, str]: + """Run validation of the values stored in the subject / session Inputs. First, format the subject name (and session if required) which also performs quick name format validations. Then, @@ -464,8 +494,17 @@ def run_local_validation(self, prefix: Prefix): Parameters ---------- + prefix + Whether to run validation on the "sub" or "ses". + + Returns + ------- + bool + Indicate whether validation passed successfully. + If `False`, there was a validation error. + output + Str containing the validation error, or successfully formatted name. - prefix : Prefix """ sub_names = self.query_one( "#create_folders_subject_input" @@ -493,7 +532,5 @@ def run_local_validation(self, prefix: Prefix): return True, f"Formatted names: {names}" def update_directorytree_root(self, new_root_path: Path) -> None: - """ - Will automatically refresh the tree through the reactive attribute `path`. - """ + """Refresh the tree through the reactive attribute `path`.""" self.query_one("#create_folders_directorytree").path = new_root_path diff --git a/datashuttle/tui/tabs/logging.py b/datashuttle/tui/tabs/logging.py index 770387f00..1e3e3e475 100644 --- a/datashuttle/tui/tabs/logging.py +++ b/datashuttle/tui/tabs/logging.py @@ -6,8 +6,12 @@ if TYPE_CHECKING: from textual import events + from textual.app import ComposeResult from textual.widgets import DirectoryTree + from datashuttle import DataShuttle + from datashuttle.tui.app import TuiApp + from textual.containers import Container, Horizontal from textual.screen import ModalScreen from textual.widgets import Button, Label, RichLog, TabPane @@ -22,29 +26,56 @@ class RichLogScreen(ModalScreen): + """Screen to display the log output.""" + def __init__(self, log_file): + """Initialise the RichLogScreen.""" super(RichLogScreen, self).__init__() - with open(log_file, "r") as file: + with open(log_file) as file: self.log_contents = "".join(file.readlines()) - def compose(self): + def compose(self) -> ComposeResult: + """Set the widgets for the screen.""" yield Container( RichLog(highlight=True, markup=True, id="richlog_screen_rich_log"), Button("Close", id="richlog_screen_close_button"), ) - def on_mount(self): + def on_mount(self) -> None: + """Update widgets immediately after mount.""" text_log = self.query_one(RichLog) text_log.write(self.log_contents) - def on_button_pressed(self, event): + def on_button_pressed(self, event: Button.Pressed) -> None: + """Handle a button press on the screen.""" if event.button.id == "richlog_screen_close_button": self.dismiss() class LoggingTab(TabPane): - def __init__(self, title, mainwindow, project, id): + """The logging tab on the project manager screen.""" + + def __init__( + self, title: str, mainwindow: TuiApp, project: DataShuttle, id: str + ): + """Initialise the Logging Tab. + + Parameters + ---------- + title + Title for the tab. + + mainwindow + Tui main appl + + project + DataShuttle project. + + id + Textual ID for the LoggingTab. + + """ super(LoggingTab, self).__init__(title=title, id=id) self.mainwindow = mainwindow @@ -57,6 +88,7 @@ def __init__(self, title, mainwindow, project, id): self.click_info = ClickInfo() def update_latest_log_path(self): + """Set the `latest_log_path` attribute that can be opened through a button.""" logs = list(self.project.get_logging_path().glob("*.log")) self.latest_log_path = ( max(logs, key=os.path.getctime) @@ -64,7 +96,8 @@ def update_latest_log_path(self): else Path("None found.") ) - def compose(self): + def compose(self) -> ComposeResult: + """Set with widgets on the LoggingTab.""" yield Container( Label( "Double click logging file to select:", @@ -89,23 +122,27 @@ def compose(self): ) def _on_mount(self, event: events.Mount) -> None: + """Update the widgets immediately after mounting.""" self.update_most_recent_label() def update_most_recent_label(self): + """Update the label indicating the most recently saved log.""" self.update_latest_log_path() self.query_one("#logging_most_recent_label").update( f"or open most recent: {self.latest_log_path.stem}" ) self.refresh() - def on_button_pressed(self, event): + def on_button_pressed(self, event: Button.Pressed) -> None: + """Handle button press on the tab.""" if event.button.id == "logging_tab_open_most_recent_button": self.push_rich_log_screen(self.latest_log_path) @require_double_click def on_directory_tree_file_selected( self, event: DirectoryTree.FileSelected - ): + ) -> None: + """Handle a click on the DirectoryTree showing the log files.""" if not event.path.is_file(): self.mainwindow.show_modal_error_dialog( "Log file no longer exists. Refresh the directory tree" @@ -116,14 +153,19 @@ def on_directory_tree_file_selected( self.push_rich_log_screen(event.path) def push_rich_log_screen(self, log_path): + """Push the screen that displays the log file contents.""" self.mainwindow.push_screen( RichLogScreen( log_path, ) ) - def reload_directorytree(self): + def reload_directorytree(self) -> None: + """Refresh the DirectoryTree (e.g. if a new log file is saved).""" self.query_one("#logging_tab_custom_directory_tree").reload() - def on_custom_directory_tree_directory_tree_special_key_press(self): + def on_custom_directory_tree_directory_tree_special_key_press( + self, + ) -> None: + """Handle the CTRL+R refresh of the directory tree.""" self.reload_directorytree() diff --git a/datashuttle/tui/tabs/transfer.py b/datashuttle/tui/tabs/transfer.py index 1973f05e6..63dd78112 100644 --- a/datashuttle/tui/tabs/transfer.py +++ b/datashuttle/tui/tabs/transfer.py @@ -43,37 +43,12 @@ class TransferTab(TreeAndInputTab): - """ - This tab handles the upload / download of files between local + """The Project Manager's Transfer tab. + + Handles the upload / download of files between local and central folders. It contains a TransferDirectoryTree that displays the transfer status of the files in the local folder, and calls underlying datashuttle transfer functions. - - Parameters - ---------- - - title : str - - mainwindow : TuiApp - - interface : Interface - TUI-datashuttle interface object - - id : str - The textual widget id. - - Attributes - ---------- - - show_legend : bool - Convenience attribute linked to a global setting exists that - turns off / on styling of directorytree nodes based on transfer status. ` - - `self.mainwindow.load_global_settings()[ - "show_transfer_tree_status" - ]` - - When on, the legend must be hidden. """ def __init__( @@ -83,6 +58,34 @@ def __init__( interface: Interface, id: Optional[str] = None, ) -> None: + """Initialise the TransferTab. + + Parameters + ---------- + title + The title of the tab + + mainwindow + The main TUI app + + interface + TUI-datashuttle interface object + + id + The textual widget id. + + Attributes + ---------- + show_legend + Convenience attribute linked to a global setting exists that + turns off / on styling of DirectoryTree nodes based on transfer status. ` + + `self.mainwindow.load_global_settings()[ + "show_transfer_tree_status" + ]` + When on, the legend must be hidden. + + """ super(TransferTab, self).__init__(title, id=id) self.mainwindow = mainwindow self.interface = interface @@ -95,6 +98,7 @@ def __init__( # ---------------------------------------------------------------------------------- def compose(self) -> ComposeResult: + """Set the widgets on the Transfer Tab.""" self.transfer_all_widgets = [ Label( "All data from: \n\n - Rawdata \n - Derivatives \n\nwill be transferred.", @@ -208,7 +212,7 @@ def compose(self) -> ComposeResult: yield Label("â­• Legend", id="transfer_legend") def on_mount(self) -> None: - + """Update the widgets immediately after mounting.""" for id in [ "#transfer_directorytree", "#transfer_switch_container", @@ -240,6 +244,7 @@ def on_mount(self) -> None: ) def on_select_changed(self, event: Select.Changed) -> None: + """Handle a Select widget changed on the tab.""" if event.select.id == "transfer_tab_overwrite_select": assert event.select.value in ["Never", "Always", "If Source Newer"] format_select = event.select.value.lower().replace(" ", "_") @@ -249,6 +254,7 @@ def on_select_changed(self, event: Select.Changed) -> None: ) def on_checkbox_changed(self, event: Checkbox.Changed) -> None: + """Handle a Checkbox widget changed on the tab.""" if event.checkbox.id == "transfer_tab_dry_run_checkbox": self.interface.save_tui_settings( event.checkbox.value, @@ -259,9 +265,9 @@ def on_checkbox_changed(self, event: Checkbox.Changed) -> None: # ---------------------------------------------------------------------------------- def switch_transfer_widgets_display(self) -> None: - """ - Show or hide transfer parameters based on whether the transfer mode - currently selected in `transfer_radioset`. + """Show or hide transfer parameters based on whether the transfer mode. + + The transfer mode is selected by the radiobutton e.g. Custom. """ for widget in self.transfer_all_widgets: widget.display = self.query_one("#transfer_all_radiobutton").value @@ -277,16 +283,14 @@ def switch_transfer_widgets_display(self) -> None: ).value def on_radio_set_changed(self, event: RadioSet.Changed) -> None: - """ - Update the displayed transfer parameter widgets when the - `transfer_radioset` radiobuttons are changed. - """ + """Update the transfer parameter widgets when the `transfer_radioset` are changed.""" label = str(event.pressed.label) assert label in ["All", "Top Level", "Custom"], "Unexpected label." self.switch_transfer_widgets_display() def on_button_pressed(self, event: Button.Pressed) -> None: - """ + """Handle a button press on a tab. + If the Transfer button is clicked, opens a modal dialog to confirm that the user wishes to transfer their data (in the direction selected). If "Yes" is selected, @@ -321,6 +325,7 @@ def on_button_pressed(self, event: Button.Pressed) -> None: ) async def refresh_after_datatype_changed(self, ignore): + """Refresh Checkboxes after the shown datatypes have changed.""" await self.recompose() self.on_mount() self.query_one("#transfer_custom_radiobutton").value = True @@ -329,6 +334,7 @@ async def refresh_after_datatype_changed(self, ignore): def on_custom_directory_tree_directory_tree_special_key_press( self, event: CustomDirectoryTree.DirectoryTreeSpecialKeyPress ) -> None: + """Handle a key press on the CustomDirectoryTree.""" if event.key == "ctrl+r": self.reload_directorytree() @@ -342,13 +348,11 @@ def on_custom_directory_tree_directory_tree_special_key_press( self.reload_directorytree() def reload_directorytree(self) -> None: + """Refresh the CustomDirectoryTree.""" self.query_one("#transfer_directorytree").update_transfer_tree() def update_directorytree_root(self, new_root_path: Path) -> None: - """ - This will automatically refresh the tree through the - reactive variable `path`. - """ + """Automatically refresh the tree through the reactive variable `path`.""" self.query_one("#transfer_directorytree").path = new_root_path # Transfer @@ -356,8 +360,7 @@ def update_directorytree_root(self, new_root_path: Path) -> None: @work(exclusive=True, thread=True) def transfer_data(self) -> Worker[InterfaceOutput]: - """ - A threaded worker to transfer data + """Transfer data in a threaded worker. This function transfers data based on the config provided by the radio buttons such as a) the data to be transferred (all / top-level-folders / custom) b) the @@ -366,6 +369,12 @@ def transfer_data(self) -> Worker[InterfaceOutput]: This function is passed to `ConfirmAndAwaitTransferPopup` which calls it to handle data transfer in a worker thread. The UI updates during and after transfer are handled by `ConfirmAndAwaitTransferPopup`. + + Returns + ------- + An InterfaceOutput object that indicates whether the transfer was a success. + Wrapped in an awaitable Textual Worker for the thread in the function is run. + """ upload = not self.query_one("#transfer_switch").value @@ -373,7 +382,6 @@ def transfer_data(self) -> Worker[InterfaceOutput]: success, output = self.interface.transfer_entire_project(upload) elif self.query_one("#transfer_toplevel_radiobutton").value: - selected_top_level_folder = self.query_one( "#transfer_toplevel_select" ).get_top_level_folder() @@ -383,7 +391,6 @@ def transfer_data(self) -> Worker[InterfaceOutput]: ) elif self.query_one("#transfer_custom_radiobutton").value: - selected_top_level_folder = self.query_one( "#transfer_custom_select" ).get_top_level_folder() diff --git a/datashuttle/tui/tabs/transfer_status_tree.py b/datashuttle/tui/tabs/transfer_status_tree.py index 8777dbb78..9993849a1 100644 --- a/datashuttle/tui/tabs/transfer_status_tree.py +++ b/datashuttle/tui/tabs/transfer_status_tree.py @@ -3,7 +3,6 @@ from typing import TYPE_CHECKING, Dict, Optional if TYPE_CHECKING: - from rich.style import Style from textual.widgets._directory_tree import DirEntry @@ -24,17 +23,10 @@ class TransferStatusTree(CustomDirectoryTree): - """ - A directorytree in which the nodes are styled depending on their - transfer status. e.g. indicates whether files are changed between - local or central, or appear in local only. + """A DirectoryTree in which the nodes are styled depending on their transfer status. - Attributes - ---------- - - Keep the local path as a string, linked to project.cfg["local_path"], - so that no conversion to string is necessary in `format_transfer_label` - which is called many times. + Foe example, indicates whether files are changed between + local or central, or appear in local only. """ def __init__( @@ -43,7 +35,26 @@ def __init__( interface: Interface, id: Optional[str] = None, ): + """Initialise the TransferStatusTree. + Parameters + ---------- + mainwindow + The main TUI application. + + interface + Datashuttle Interface object. + + id + Textual ID for the TransferStatusTree. + + Attributes + ---------- + Keep the local path as a string, linked to project.cfg["local_path"], + so that no conversion to string is necessary in `format_transfer_label` + which is called many times. + + """ self.interface = interface self.local_path_str = self.interface.get_configs()[ "local_path" @@ -55,13 +66,11 @@ def __init__( ) def on_mount(self) -> None: + """Update the directory tree after the widget is mounted.""" self.update_transfer_tree(init=True) def update_transfer_tree(self, init: bool = False) -> None: - """ - Updates tree styling to reflect the current TUI state - and project transfer status. - """ + """Update tree styling to reflect the current TUI state and project transfer status.""" self.local_path_str = self.interface.get_configs()[ "local_path" ].as_posix() @@ -75,9 +84,7 @@ def update_transfer_tree(self, init: bool = False) -> None: self.reload() def update_local_transfer_paths(self) -> None: - """ - Compiles a list of all project files and paths. - """ + """Compiles a list of all project files and paths.""" paths_list = [] for top_level_folder in canonical_folders.get_top_level_folders(): @@ -91,9 +98,7 @@ def update_local_transfer_paths(self) -> None: self.transfer_paths = paths_list def update_transfer_diffs(self) -> None: - """ - Updates the transfer diffs used to style the DirectoryTree. - """ + """Update the transfer diffs used to style the DirectoryTree.""" self.transfer_diffs = get_local_and_central_file_differences( self.interface.get_configs(), top_level_folders_to_check=["rawdata", "derivatives"], @@ -105,7 +110,8 @@ def update_transfer_diffs(self) -> None: def render_label( self, node: TreeNode[DirEntry], base_style: Style, style: Style ) -> Text: - """ + """Handle label rendering on the TransferStatusTree. + Extends the `DirectoryTree.render_label()` method to allow custom styling of file nodes according to their transfer status. """ @@ -148,7 +154,8 @@ def render_label( return text def format_transfer_label(self, node_label, node_path) -> None: - """ + """Format the folder label depending on its transfer status. + Takes nodes being formatted using `render_label` and applies custom formatting according to the node's transfer status. """ diff --git a/datashuttle/tui/tooltips.py b/datashuttle/tui/tooltips.py index f1e33f910..f39bb9a4b 100644 --- a/datashuttle/tui/tooltips.py +++ b/datashuttle/tui/tooltips.py @@ -1,8 +1,5 @@ def get_tooltip(id: str) -> str: - """ - Master function to get tooltips for all widgets, - based on their widget (textual) id. - """ + """Return tooltip for a widget based on its textual id.""" # Main App Window # ------------------------------------------------------------------------- diff --git a/datashuttle/tui/utils/tui_decorators.py b/datashuttle/tui/utils/tui_decorators.py index 9b95735be..8a2e493d6 100644 --- a/datashuttle/tui/utils/tui_decorators.py +++ b/datashuttle/tui/utils/tui_decorators.py @@ -13,22 +13,21 @@ class ClickInfo: - """ - A class to hold click-info to checking - double clicks are within the time threshold - and match the widget id. + """A class to hold click-info. + + This stores click history to allow later checking + that double clicks occur within a time threshold + and that the same widget is clicked twice. """ def __init__(self): - + """Initialise the ClickInfo.""" self.prev_click_time = 0.0 self.prev_click_widget_id = "" def require_double_click(func): - """ - A decorator that calls the decorated function - on a double click, otherwise will not do anything. + """Call the decorated function on a double click. Requires the first argument (`self` on the class) to have the attribute `click_info`). Any class holding a widget diff --git a/datashuttle/tui/utils/tui_validators.py b/datashuttle/tui/utils/tui_validators.py index 792fc1c5e..916eb6241 100644 --- a/datashuttle/tui/utils/tui_validators.py +++ b/datashuttle/tui/utils/tui_validators.py @@ -1,6 +1,4 @@ -""" -Tools for live validation of user inputs in the DataShuttle TUI. -""" +"""Tools for live validation of user inputs in the DataShuttle TUI.""" from __future__ import annotations @@ -14,21 +12,30 @@ class NeuroBlueprintValidator(Validator): + """A textual validator subclass for validating Input text. + + Takes the sub / ses prefix as input. Runs validation of + the name against the project and propagates + any error message through the Input tooltip. + """ + def __init__(self, prefix: Prefix, parent: CreateFoldersTab) -> None: - """ - Custom Validator() class that takes - sub / ses prefix as input. Runs validation of - the name against the project and propagates - any error message through the Input tooltip. + """Initialise the NeuroBlueprintValidator. + + prefix + "sub" or "ses". + + parent + The tab on which the validated input exists. """ super(NeuroBlueprintValidator, self).__init__() self.parent = parent self.prefix = prefix def validate(self, name: str) -> ValidationResult: - """ - Run validation and update the tooltip with the error, - if no error then the formatted sub / ses name is displayed. + """Run validation and update the tooltip with the error. + + If no error then the formatted sub / ses name is displayed. This is set on an Input widget. """ valid, message = self.parent.run_local_validation(self.prefix) diff --git a/datashuttle/tui_launcher.py b/datashuttle/tui_launcher.py index fb2e05ff9..f98f4b0be 100644 --- a/datashuttle/tui_launcher.py +++ b/datashuttle/tui_launcher.py @@ -29,9 +29,7 @@ def main() -> None: - """ - Launch the datashuttle tui. - """ + """Launch the datashuttle tui.""" args = parser.parse_args() if args.launch == "launch": diff --git a/datashuttle/utils/custom_exceptions.py b/datashuttle/utils/custom_exceptions.py index 1d3d8e86a..05be4b6d0 100644 --- a/datashuttle/utils/custom_exceptions.py +++ b/datashuttle/utils/custom_exceptions.py @@ -1,6 +1,10 @@ class ConfigError(Exception): + """Raise an error relating to a configuration problem.""" + pass class NeuroBlueprintError(Exception): + """Raise an error when something doesn't conform to the NeuroBlueprint pattern.""" + pass diff --git a/datashuttle/utils/data_transfer.py b/datashuttle/utils/data_transfer.py index 21121b8e6..1db712859 100644 --- a/datashuttle/utils/data_transfer.py +++ b/datashuttle/utils/data_transfer.py @@ -12,51 +12,15 @@ class TransferData: - """ - Class to perform data transfers. This works by first building - a large list of all files to transfer. Then, rclone is called + """Class to perform data transfers. + + This works by first building a large list of all + files to transfer. Then, rclone is called once with this list to perform the transfer. The properties on this class are to be read during generation of transfer lists and should never be changed during the lifetime of the class. - - Parameters - ---------- - - cfg : Configs, - datashuttle configs UserDict. - - upload_or_download : Literal["upload", "download"] - Direction to perform the transfer. - - top_level_folder: TopLevelFolder - - sub_names : Union[str, List[str]] - List of subject names or single subject to transfer. This - can include transfer keywords (e.g. "all_non_sub"). - - ses_names : Union[str, List[str]] - List of sessions or single session to transfer, for each - subject. May include session-level transfer keywords. - - datatype : Union[str, List[str]] - List of datatypes to transfer, for the sessions / subjects - specified. Can include datatype-level tranfser keywords. - - overwrite_existing_files : OverwriteExistingFiles - If "never" files on target will never be overwritten by source. - If "always" files on target will be overwritten by source if - there is any difference in date or size. - If "if_source_newer" files on target will only be overwritten - by files on source with newer creation / modification datetime. - - dry_run : bool, - If `True`, transfer will not actually occur but will be logged - as if it did (to see what would happen for a transfer). - - log : bool, - if `True`, log and print the transfer output. """ def __init__( @@ -71,6 +35,46 @@ def __init__( dry_run: bool, log: bool, ): + """Initialise TransferData. + + Parameters + ---------- + cfg + datashuttle configs UserDict. + + upload_or_download + Direction to perform the transfer. + + top_level_folder + The top-level folder structure where data is organized. + + sub_names + List of subject names or single subject to transfer. This + can include transfer keywords (e.g. "all_non_sub"). + + ses_names + List of sessions or single session to transfer, for each + subject. May include session-level transfer keywords. + + datatype + List of datatypes to transfer, for the sessions / subjects + specified. Can include datatype-level tranfser keywords. + + overwrite_existing_files + If ``"never"`` files on target will never be overwritten by source. + If ``"always"`` files on target will be overwritten by source if + there is any difference in date or size. + If ``"if_source_newer"`` files on target will only be overwritten + by files on source with newer creation / modification datetime. + + dry_run + Perform a dry-run of transfer. This will output as if file + transfer was taking place, but no files will be moved. + + log + if `True`, log and print the transfer output. + + """ self.__cfg = cfg self.__upload_or_download = upload_or_download self.__top_level_folder = top_level_folder @@ -111,20 +115,23 @@ def __init__( # ------------------------------------------------------------------------- def build_a_list_of_all_files_and_folders_to_transfer(self) -> List[str]: - """ - Build a list of every file to transfer based on the user-passed - arguments. This cycles through every subject, session and datatype + """Build a list of every file to transfer based on the user-passed arguments. + + This cycles through every subject, session and datatype and adds the outputs to three lists: - `sub_ses_dtype_include` - files within datatype folders - `extra_folder_names` - folders that do not fall within datatype folders - `extra_file_names` - files that do not fall within datatype folders + `sub_ses_dtype_include` + files within datatype folders + `extra_folder_names` + folders that do not fall within datatype folders + `extra_file_names` + files that do not fall within datatype folders Returns ------- - - include_list : List[str] + include_list A list of paths to pass to rclone's `--include` flag. + """ # Find sub names to transfer processed_sub_names = self.get_processed_names(self.sub_names) @@ -185,10 +192,7 @@ def build_a_list_of_all_files_and_folders_to_transfer(self) -> List[str]: def make_include_arg( self, list_of_paths: List[str], recursive: bool = True ) -> List[str]: - """ - Format the list of paths to rclone's required - `--include` flag format. - """ + """Return the list of paths formatted to rclone's required `--include` flag format.""" if not any(list_of_paths): return [] @@ -211,9 +215,9 @@ def include_arg(ele: str) -> str: def update_list_with_non_sub_top_level_folders( self, extra_folder_names: List[str], extra_filenames: List[str] ) -> None: - """ - Search the subject level for all files and folders in the - top-level-folder. Split the output based onto files / folders + """Search the subject level for all files and folders in the top-level-folder. + + Splits the output based onto files / folders within "sub-" prefixed folders or not. """ top_level_folders: List[str] @@ -239,10 +243,7 @@ def update_list_with_non_ses_sub_level_folders( extra_filenames: List[str], sub: str, ) -> None: - """ - For the subject, get a list of files / folders that are - not within "ses-" prefixed folders. - """ + """Get a list of files / folders that for the subject that are not within "ses-" prefixed folders.""" sub_level_folders: List[str] sub_level_folders, sub_level_files = folders.search_sub_or_ses_level( # type: ignore self.__cfg, @@ -276,8 +277,9 @@ def update_list_with_non_dtype_ses_level_folders( sub: str, ses: str, ) -> None: - """ - For a specific subject and session, get a list of files / folders + """Return non-datatype files and folders. + + Returns a list of files / folders for a specific subject and session that are not in canonical datashuttle datatype folders. """ ses_level_folders: List[str] @@ -321,10 +323,7 @@ def update_list_with_dtype_paths( sub: str, ses: Optional[str] = None, ) -> None: - """ - Given a particular subject and session, get a list of all - canonical datatype folders. - """ + """Get a list of all canonical datatype folders for a particular subject and session.""" datatype = list(filter(lambda x: x != "all_non_datatype", datatype)) datatype_items = folders.items_from_datatype_input( @@ -352,6 +351,7 @@ def update_list_with_dtype_paths( # ------------------------------------------------------------------------- def to_list(self, names: Union[str, List[str]]) -> List[str]: + """Return a name or list of names as a list.""" if isinstance(names, str): names = [names] return names @@ -359,11 +359,12 @@ def to_list(self, names: Union[str, List[str]]) -> List[str]: def check_input_arguments( self, ) -> None: - """ - Check the sub / session names passed. The checking here - is stricter than for create_folders / formatting.check_and_format_names - because we want to ensure that a) non-datatype arguments are not - passed at the wrong input (e.g. all_non_ses as a subject name). + """Check the sub / session names passed. + + The checking here is stricter than for create_folders and + formatting.check_and_format_names because we want to ensure that + non-datatype arguments are not passed at the wrong input + (e.g. all_non_ses as a subject name). We also want to limit the possible combinations of inputs, such that is a user inputs "all" subjects, or "all_sub", they should @@ -372,8 +373,8 @@ def check_input_arguments( Parameters ---------- - see update_list_with_dtype_paths() + """ if len(self.sub_names) > 1 and any( [name in ["all", "all_sub"] for name in self.sub_names] @@ -421,8 +422,8 @@ def get_processed_names( names_checked: List[str], sub: Optional[str] = None, ) -> List[str]: - """ - Process the list of subject session names. + """Process the list of subject session names. + If they are pre-defined (e.g. ["sub-001", "sub-002"]) they will be checked and formatted as per formatting.check_and_format_names() and @@ -433,10 +434,12 @@ def get_processed_names( will be searched to determine what files exist to transfer, and the sub / ses names list generated. - Parameters - ---------- + see transfer_sub_ses_data() for list of parameters. - see transfer_sub_ses_data() + Returns + ------- + A list of folder names generated from the original + names list that included search wildcards, "all" keys etc. """ prefix: Prefix @@ -477,10 +480,7 @@ def get_processed_names( return processed_names def transfer_non_datatype(self, datatype_checked: List[str]) -> bool: - """ - Convenience function, bool if all non-datatype folders - are to be transferred - """ + """Return bool indicating whether all non-datatype folders are to be transferred.""" return any( [name in ["all_non_datatype", "all"] for name in datatype_checked] ) diff --git a/datashuttle/utils/decorators.py b/datashuttle/utils/decorators.py index cacf54914..6fecb6ce0 100644 --- a/datashuttle/utils/decorators.py +++ b/datashuttle/utils/decorators.py @@ -5,9 +5,10 @@ def requires_ssh_configs(func): - """ - Decorator to check file is loaded. Used on Mainwindow class - methods only as first arg is assumed to be self (containing cfgs) + """Check ssh configs have been set. + + Used on Mainwindow class methods only as first + arg is assumed to be self (containing cfgs). """ @wraps(func) @@ -29,11 +30,7 @@ def wrapper(*args, **kwargs): def check_configs_set(func): - """ - Check that configs have been loaded (i.e. - project.cfg is not None) before the - func is run. - """ + """Check configs have been set.""" @wraps(func) def wrapper(*args, **kwargs): @@ -50,9 +47,7 @@ def wrapper(*args, **kwargs): def check_is_not_local_project(func): - """ - Decorator to check that the project is not - a local project. If it is, raise. + """Check that the project is not a local project. This decorator should be placed above methods which require `central_path` and `connection_method` to be set. diff --git a/datashuttle/utils/ds_logger.py b/datashuttle/utils/ds_logger.py index ca1c6bf94..d1591038d 100644 --- a/datashuttle/utils/ds_logger.py +++ b/datashuttle/utils/ds_logger.py @@ -3,6 +3,7 @@ from typing import TYPE_CHECKING, Any, List, Optional if TYPE_CHECKING: + from logging import Logger from pathlib import Path from datashuttle.configs.configs import Configs @@ -17,15 +18,18 @@ from datashuttle.utils import utils -def get_logger_name(): +def get_logger_name() -> str: + """Return the name of the logger.""" return "datashuttle" -def get_logger(): +def get_logger() -> Logger: + """Return the instance of the logger object.""" return logging.getLogger(get_logger_name()) -def logging_is_active(): +def logging_is_active() -> bool: + """Return a bool indicating if the logger is active.""" logger_exists = get_logger_name() in logging.root.manager.loggerDict if logger_exists and get_logger().handlers != []: return True @@ -38,8 +42,23 @@ def start( variables: Optional[List[Any]], verbose: bool = True, ) -> None: - """ - Call fancylog to initialise logging. + """Call fancylog to initialise logging. + + Parameters + ---------- + path_to_log + Path to save the log file to. + + command_name + Name of the datashuttle command run, which is included + in the log filename. + + variables + Local variables to log. + + verbose + Verbosity passed to ``fancylog``. + """ filename = get_logging_filename(command_name) @@ -60,39 +79,54 @@ def start( def get_logging_filename(command_name: str) -> str: - """ - Get the filename to which the log will be saved. This - starts with ISO8601-formatted datetime, so logs are stored - in datetime order. + """Return the log filename. + + This starts with ISO8601-formatted datetime, so logs + are stored in datetime order. + + Parameters + ---------- + command_name + Name of the datashuttle command run, which is included + in the log filename. + """ filename = datetime.now().strftime(f"%Y%m%dT%H%M%S_{command_name}") return filename def log_names(list_of_headers: List[Any], list_of_names: List[Any]) -> None: - """ - Log a list of subject or session names. + """Log a list of subject or session names. Parameters ---------- + list_of_headers + a list of titles that the names + will be printed under, e.g. "sub_names", "ses_names" - list_of_headers : a list of titles that the names - will be printed under, e.g. "sub_names", "ses_names" + list_of_names + list of names to print to log - list_of_names : list of names to print to log """ for header, names in zip(list_of_headers, list_of_names): utils.log(f"{header}: {names}") def wrap_variables_for_fancylog(local_vars: dict, cfg: Configs) -> List: - """ - Wrap the locals from the original function call to log - and the datashuttle.cfg in a wrapper class with __dict__ + """Wrap the locals from the original function call for fancylog. + + Fancylog will log these variables as well as the + datashuttle.cfg in a wrapper class with __dict__ attribute for fancylog writing. Delete the self attribute (which is DataShuttle class) to keep the logs neat, as it adds no information. + + Returns + ------- + A list holding a wrapper class that holds all variable + state for fancylog to log. + """ class VariablesState: @@ -108,9 +142,7 @@ def __init__(self, local_vars_, cfg_): def close_log_filehandler() -> None: - """ - Remove handlers from all loggers. - """ + """Remove handlers from all loggers.""" logger = get_logger() logger.debug("Finished logging.") handlers = logger.handlers[:] diff --git a/datashuttle/utils/folder_class.py b/datashuttle/utils/folder_class.py index 6ada726c2..71fab22a5 100644 --- a/datashuttle/utils/folder_class.py +++ b/datashuttle/utils/folder_class.py @@ -1,7 +1,5 @@ class Folder: - """ - Folder class used to contain details of canonical - folders in the project folder tree. + """Contains details of canonical folders in the project folder tree. see configs.canonical_folders.py for details. """ @@ -11,5 +9,16 @@ def __init__( name: str, level: str, ): + """Initialise the Folder class. + + Parameters + ---------- + name + the name of the folder. + + level + level to make the folder at. + + """ self.name = name self.level = level diff --git a/datashuttle/utils/folders.py b/datashuttle/utils/folders.py index 56852640c..a8ff8827b 100644 --- a/datashuttle/utils/folders.py +++ b/datashuttle/utils/folders.py @@ -36,9 +36,9 @@ def create_folder_trees( datatype: Union[List[str], str], log: bool = True, ) -> Dict[str, List[Path]]: - """ - Entry method to make a full folder tree. It will - iterate through all passed subjects, then sessions, then + """Entry method to make a full folder tree. + + Iterate through all passed subjects, then sessions, then subfolders within a datatype folder. This permits flexible creation of folders (e.g. to make subject only, do not pass session name. @@ -48,11 +48,19 @@ def create_folder_trees( Parameters ---------- + cfg + datashuttle config UserDict + + top_level_folder + either "rawdata" or "derivatives" - sub_names, ses_names, datatype : see create_folders() + sub_names, ses_names, datatype + see create_folders() - log : whether to log or not. If True, logging must + log + whether to log or not. If True, logging must already be initialised. + """ datatype_passed = datatype not in [[""], ""] @@ -121,36 +129,42 @@ def make_datatype_folders( save_paths: Dict, log: bool = True, ): - """ - Make datatype folder (e.g. behav) at the sub or ses - level. Checks folder_class.Folders attributes, - whether the datatype is used and at the current level. + """Make datatype folder (e.g. behav) at the sub or ses level. + + Checks folder_class.Folders attributes, whether the datatype + is used and at the current level. Parameters ---------- - cfg : ConfigsClass + cfg + datashuttle configs - datatype : datatype (e.g. "behav", "all") to use. Use + datatype + datatype (e.g. "behav", "all") to use. Use empty string ("") for none. - sub_or_ses_level_path : Full path to the subject + sub_or_ses_level_path + Full path to the subject or session folder where the new folder will be written. - level : The folder level that the + level + The folder level that the folder will be made at, "sub" or "ses" - save_paths : A dictionary, which will be filled + save_paths + A dictionary, which will be filled with created paths split by datatype name. - log : whether to log on or not (if True, logging must + log + whether to log on or not (if True, logging must already be initialised). + """ datatype_items = cfg.get_datatype_as_dict_items(datatype) for datatype_key, datatype_folder in datatype_items: # type: ignore if datatype_folder.level == level: - datatype_name = datatype_folder.name datatype_path = sub_or_ses_level_path / datatype_name @@ -168,17 +182,17 @@ def make_datatype_folders( def create_folders(paths: Union[Path, List[Path]], log: bool = True) -> None: - """ - For path or list of paths, make them if - they do not already exist. + """Make a path or list of paths if they do not already exist. Parameters ---------- + paths + Path or list of Paths to create - paths : Path or list of Paths to create - - log : if True, log all made folders. This + log + if True, log all made folders. This requires the logger to already be initialised. + """ if isinstance(paths, Path): paths = [paths] @@ -206,18 +220,44 @@ def search_project_for_sub_or_ses_names( include_central: bool, return_full_path: bool = False, ) -> Dict: - """ - If sub is None, the top-level level folder will be - searched (i.e. for subjects). The search string "sub-*" is suggested - in this case. Otherwise, the subject, level folder for the specified - subject will be searched. The search_str "ses-*" is suggested in this case. + """If sub is None, the top-level level folder will be searched (i.e. for subjects). + + The search string "sub-*" is suggested in this case. Otherwise, the subject, + level folder for the specified subject will be searched. + The search_str "ses-*" is suggested in this case. Note `verbose` argument of `search_sub_or_ses_level()` is set to `False`, as session folders for local subjects that are not yet on central will be searched for on central, showing a confusing 'folder not found' message. - """ + Parameters + ---------- + cfg + Datashuttle Configs object. + + top_level_folder + "rawdata" or "derivatives". + + sub + Subject name (if provided, search for a session within that sub) + + search_str + Glob-style search to perform e.g. "sub-*" + + include_central + If `True`, central project is also searched. + + return_full_path + If True, the full path to the discovered folders is provided. + Otherwise, just the name. + + Returns + ------- + A dictionary with "local" and "central" keys, where values + are the discovered folders. "central" is `None` if include_central is `False`. + + """ # Search local and central for folders that begin with "sub-*" local_foldernames, _ = search_sub_or_ses_level( cfg, @@ -259,15 +299,19 @@ def items_from_datatype_input( sub: str, ses: Optional[str] = None, ) -> Union[ItemsView, zip]: - """ - Get the list of datatypes to transfer, either - directly from user input, or by searching + """Return the list of datatypes to transfer. + + Take these directly from user input, or by searching what is available if "all" is passed. - Parameters - ---------- + see _transfer_datatype() for full parameters list. + + Returns + ------- + Datatypes as a dictionary items() or zip that mimics that structure. + The dictionary is in the form datatype name: Folder() struct. + See `canonical_folders.py`. - see _transfer_datatype() for parameters. """ base_folder = cfg.get_base_folder(local_or_central, top_level_folder) @@ -299,9 +343,9 @@ def search_for_datatype_folders( sub: str, ses: Optional[str] = None, ) -> zip: - """ - Search a subject or session folder specifically - for datatypes. First searches for all folders / files + """Search a subject or session folder specifically for datatypes. + + First searches for all folders / files in the folder, and then returns any folders that match datatype name. @@ -312,6 +356,7 @@ def search_for_datatype_folders( ------- Find the datatype files and return in a format that mirrors dict.items() + """ search_results = search_sub_or_ses_level( cfg, base_folder, local_or_central, sub, ses @@ -328,16 +373,16 @@ def process_glob_to_find_datatype_folders( folder_names: list, datatype_folders: dict, ) -> zip: - """ - Process the results of glob on a sub or session level, - which could contain any kind of folder / file. + """Process the results of glob on a sub or session level. + The results could contain any type of folder / file. see project.search_sub_or_ses_level() for inputs. Returns ------- Find the datatype files and return in a format that mirrors dict.items() + """ ses_folder_keys = [] ses_folder_values = [] @@ -366,8 +411,7 @@ def search_for_wildcards( all_names: List[str], sub: Optional[str] = None, ) -> List[str]: - """ - Handle wildcard flag in upload or download. + """Handle wildcard flag in upload or download. All names in name are searched for @*@ string, and replaced with single * for glob syntax. If sub is passed, it is @@ -376,28 +420,38 @@ def search_for_wildcards( if sub is None it is assumed all_names are sub names and the level above is searched. - Outputs a new list of names including all original names - but where @*@-containing names have been replaced with - search results. - Parameters ---------- + cfg + datashuttle configs - project : initialised datashuttle project + project + initialised datashuttle project - base_folder : folder to search for wildcards in + base_folder + folder to search for wildcards in - local_or_central : "local" or "central" project path to + local_or_central + "local" or "central" project path to search in - all_names : list of subject or session names that + all_names + list of subject or session names that may or may not include the wildcard flag. If sub (below) is passed, it is assumed these are session names. Otherwise, it is assumed these are subject names. - sub : optional subject to search for sessions in. If not provided, + sub + optional subject to search for sessions in. If not provided, will search for subjects rather than sessions. + Returns + ------- + new_all_names + A new list of names including all original names + but where @*@-containing names have been replaced with + search results. + """ new_all_names: List[str] = [] for name in all_names: @@ -441,38 +495,52 @@ def search_sub_or_ses_level( verbose: bool = True, return_full_path: bool = False, ) -> Tuple[List[str] | List[Path], List[str]]: - """ - Search project folder at the subject or session level. - Only returns folders + """Search project folder at the subject or session level. Parameters ---------- - - cfg : datashuttle project cfg. Currently, this is used + cfg + datashuttle project cfg. Currently, this is used as a holder for ssh configs to avoid too many arguments, but this is not nice and breaks the general rule that these functions should operate project-agnostic. - local_or_central : search in local or central project + base_folder + the path to the base folder. If sub is None, the search is + performed on this folder - sub : either a subject name (string) or None. If None, the search - is performed at the top_level_folder level + local_or_central + search in local or central project - ses : either a session name (string) or None, This must not + sub + either a subject name (string) or None. If None, the search + is performed at the base_folder level + + ses + either a session name (string) or None, This must not be a session name if sub is None. If provided (with sub) then the session folder is searched - str : glob-format search string to search at the + search_str + glob-format search string to search at the folder level. - verbose : If `True`, if a search folder cannot be found, a message - will be printed with the un-found path. + verbose + If `True`, if a search folder cannot be found, a message + will be printed with the un-found path. + + return_full_path + include the search_path in the returned paths + + Returns + ------- + Discovered folders (`all_folder_names`) and files (`all_filenames`). + """ if ses and not sub: utils.log_and_raise_error( - "cannot pass session to " - "search_sub_or_ses_level() without subject", + "cannot pass session to search_sub_or_ses_level() without subject", ValueError, ) @@ -502,18 +570,33 @@ def search_for_folders( verbose: bool = True, return_full_path: bool = False, ) -> Tuple[List[Any], List[Any]]: - """ - Wrapper to determine the method used to search for search - prefix folders in the search path. + """Determine the method used to search for search prefix folders in the search path. Parameters ---------- + cfg + datashuttle configs + + local_or_central + "local" or "central" + + search_path + full filepath to search in + + search_prefix + file / folder name to search (e.g. "sub-*") + + verbose + If `True`, when a search folder cannot be found, a message + will be printed with the missing path. + + return_full_path + include the search_path in the returned paths + + Returns + ------- + Discovered folders (`all_folder_names`) and files (`all_filenames`). - local_or_central : "local" or "central" - search_path : full filepath to search in - search_prefix : file / folder name to search (e.g. "sub-*") - verbose : If `True`, when a search folder cannot be found, a message - will be printed with the missing path. """ if local_or_central == "central" and cfg["connection_method"] == "ssh": all_folder_names, all_filenames = ssh.search_ssh_central_for_folders( @@ -541,9 +624,24 @@ def search_for_folders( def search_filesystem_path_for_folders( search_path_with_prefix: Path, return_full_path: bool = False ) -> Tuple[List[Path | str], List[Path | str]]: - """ + r"""Search a folder through the local filesystem. + Use glob to search the full search path (including prefix) with glob. Files are filtered out of results, returning folders only. + + Parameters + ---------- + search_path_with_prefix + Path to search along with search prefix e.g. "C:\drive\project\sub-*" + + return_full_path + If `True` returns the path to the discovered folder or file, + otherwise just the name. + + Returns + ------- + Discovered folders (`all_folder_names`) and files (`all_filenames`). + """ all_folder_names = [] all_filenames = [] @@ -552,7 +650,6 @@ def search_filesystem_path_for_folders( sorter_files_and_folders = sorted(all_files_and_folders) for file_or_folder_str in sorter_files_and_folders: - file_or_folder = Path(file_or_folder_str) if file_or_folder.is_dir(): diff --git a/datashuttle/utils/formatting.py b/datashuttle/utils/formatting.py index 1cebf49d0..625174218 100644 --- a/datashuttle/utils/formatting.py +++ b/datashuttle/utils/formatting.py @@ -22,9 +22,9 @@ def check_and_format_names( name_templates: Optional[Dict] = None, bypass_validation: bool = False, ) -> List[str]: - """ - Format a list of subject or session names, e.g. - by ensuring all have sub- or ses- prefix, checking + """Format a list of subject or session names. + + This ensures all have sub- or ses- prefix, checks for tags, that names do not include spaces and that there are not duplicates. @@ -38,21 +38,25 @@ def check_and_format_names( Parameters ---------- - - names : Union[list, str] + names str or list containing sub or ses names (e.g. to create folders) - prefix : Prefix + prefix "sub" or "ses" - this defines the prefix checks. - name_templates : Dict + name_templates A dictionary of templates to validate subject and session name against. e.g. {"name_templates": {"on": False, "sub": None, "ses": None}} where the "sub" and "ses" may contain a regexp to validate against. - bypass_validation : Dict + bypass_validation If `True`, NeuroBlueprint validation will be performed on the passed names. + + Returns + ------- + A list of formatted names. + """ if isinstance(names, str): names = [names] @@ -79,8 +83,7 @@ def check_and_format_names( def format_names(names: List, prefix: Prefix) -> List[str]: - """ - Check a single or list of input session or subject names. + """Check a single or list of input session or subject names. First check the type is correct, next prepend the prefix sub- or ses- to entries that do not have the relevant prefix. @@ -88,10 +91,17 @@ def format_names(names: List, prefix: Prefix) -> List[str]: with required inputs e.g. date, time Parameters - ----------- - names: str or list containing sub or ses names (e.g. to make folders) + ---------- + names + str or list containing sub or ses names (e.g. to make folders) + + prefix + "sub" or "ses" - this defines the prefix checks. + + Returns + ------- + A list of formatted names. - prefix: "sub" or "ses" - this defines the prefix checks. """ assert prefix in ["sub", "ses"], "`prefix` must be 'sub' or 'ses'." @@ -114,13 +124,13 @@ def format_names(names: List, prefix: Prefix) -> List[str]: def update_names_with_range_to_flag( names: List[str], prefix: str ) -> List[str]: - """ - Given a list of names, check if they contain the @TO@ keyword. + """Given a list of names, check if they contain the @TO@ keyword. + If so, expand to a range of names. Names including the @TO@ keyword must be in the form prefix-num1@num2. The maximum number of leading zeros are used to pad the output e.g. - sub-01@003 becomes ["sub-001", "sub-002", "sub-003"] + sub-01@003 becomes ["sub-001", "sub-002", "sub-003"]. Input can also be a mixed list e.g. names = ["sub-01", "sub-02@TO@04", "sub-05@TO@10"] @@ -132,7 +142,9 @@ def update_names_with_range_to_flag( if tags("to") in name: check_name_with_to_tag_is_formatted_correctly(name, prefix) - prefix_tag = re.search(f"{prefix}-[0-9]+{tags('to')}[0-9]+", name)[0] # type: ignore + prefix_tag = re.search(f"{prefix}-[0-9]+{tags('to')}[0-9]+", name)[ + 0 + ] # type: ignore tag_number = prefix_tag.split(f"{prefix}-")[1] name_start_str, name_end_str = name.split(tag_number) @@ -169,10 +181,7 @@ def update_names_with_range_to_flag( def check_name_with_to_tag_is_formatted_correctly( name: str, prefix: str ) -> None: - """ - Check the input string is formatted with the @TO@ key - as expected. - """ + """Check the input string is formatted with the @TO@ key as expected.""" first_key_value_pair = name.split("_")[0] expected_format = re.compile(f"{prefix}-[0-9]+{tags('to')}[0-9]+") @@ -188,7 +197,8 @@ def check_name_with_to_tag_is_formatted_correctly( def make_list_of_zero_padded_names_across_range( left_number: str, right_number: str, name_start_str: str, name_end_str: str ) -> List[str]: - """ + """Make a list of subject or session names across a range. + Numbers formatted with the @TO@ keyword need to have standardised leading zeros on the output. Here we take the maximum number of leading zeros and apply for @@ -197,15 +207,25 @@ def make_list_of_zero_padded_names_across_range( Parameters ---------- + left_number + left (start) number from the range, e.g. "001" - left_number : left (start) number from the range, e.g. "001" + right_number + right (end) number from the range, e.g. "005" - right_number : right (end) number from the range, e.g. "005" + name_start_str + part of the name before the flag, usually "sub-" - name_start_str : part of the name before the flag, usually "sub-" - - name_end_str : rest of the name after the flag, i.e. all other + name_end_str + rest of the name after the flag, i.e. all other key-value pairs. + + Returns + ------- + A list of subject or session names expanded across a range. + e.g. sub-001@TO@002_date-20220101 becomes + ["sub-001_date-20220101", "sub-002_date-20220101"]. + """ max_leading_zeros = max( utils.num_leading_zeros(left_number), @@ -230,9 +250,9 @@ def make_list_of_zero_padded_names_across_range( def update_names_with_datetime(names: List[str]) -> None: - """ - Replace @DATE@ and @DATETIME@ flag with date and datetime respectively. + """Replace @DATE@ and @DATETIME@ flag with date and datetime respectively. + `names` is a list of subject or session names. Format using key-value pair for bids, i.e. date-20221223_time- """ date = str(datetime.datetime.now().date().strftime("%Y%m%d")) @@ -253,10 +273,23 @@ def replace_date_time_tags_in_name( datetime_with_key: str, date_with_key: str, time_with_key: str, -): - """ - For all names in the list, do the replacement of tags - with their final values. +) -> None: + """Replace tags with their final value for every name in a list. + + Parameters + ---------- + names + A list of subject or session names. + + datetime_with_key + Formatted datetime key-value pair .e.g datetime-20220101T010101. + + date_with_key + Formatted date key-value pair .e.g date-20220101. + + time_with_key + Formatted time key-value pair .e.g time-010101. + """ for i, name in enumerate(names): # datetime conditional must come first. @@ -276,23 +309,27 @@ def replace_date_time_tags_in_name( def format_date(date: str) -> str: + """Return the date formatted as `date-`.""" return f"date-{date}" def format_time(time_: str) -> str: + """Return the time `time_` formatted as `time-`.""" return f"time-{time_}" def format_datetime(date: str, time_: str) -> str: + """Return the `date` and `time_` formatted as `datetime-T`.""" return f"datetime-{date}T{time_}" def add_underscore_before_after_if_not_there(string: str, key: str) -> str: - """ + """Handle tags that are not perfectly formatted with underscores. + If names are passed with @DATE@, @TIME@, or @DATETIME@ but not surrounded by underscores, check and insert if required. e.g. sub-001@DATE@ becomes sub-001_@DATE@ - or sub-001@DATEid-101 becomes sub-001_@DATE_id-101 + or sub-001@DATEid-101 becomes sub-001_@DATE_id-101. """ key_len = len(key) key_start_idx = string.index(key) @@ -300,9 +337,9 @@ def add_underscore_before_after_if_not_there(string: str, key: str) -> str: # Handle left edge if string[key_start_idx - 1] != "_": string_split = string.split(key) # assumes key only in string once - assert ( - len(string_split) == 2 - ), f"{key} must not appear in string more than once." + assert len(string_split) == 2, ( + f"{key} must not appear in string more than once." + ) string = f"{string_split[0]}_{key}{string_split[1]}" @@ -318,12 +355,7 @@ def add_underscore_before_after_if_not_there(string: str, key: str) -> str: def add_missing_prefixes_to_names( all_names: Union[List[str], str], prefix: str ) -> List[str]: - """ - Make sure all elements in the list of names are - prefixed with the prefix, typically "sub-" or "ses-" - - Use expanded list for readability - """ + """Ensure all elements in the list of names are prefixed with "sub-" or "ses-".""" prefix = prefix + "-" n_chars = len(prefix) diff --git a/datashuttle/utils/getters.py b/datashuttle/utils/getters.py index c3b357dab..33ee68f86 100644 --- a/datashuttle/utils/getters.py +++ b/datashuttle/utils/getters.py @@ -37,10 +37,11 @@ def get_next_sub_or_ses( default_num_value_digits: int = 3, name_template_regexp: Optional[str] = None, ) -> str: - """ - Suggest the next available subject or session number. This function will - search the local repository, and the central repository, for all subject - or session folders (subject or session depending on inputs). + """Suggest the next available subject or session number. + + This function will search the local repository, and the central + repository, for all subject or session folders (subject or session + depending on inputs). It will take the union of all folder names, find the relevant key-value pair values, and return the maximum value + 1 as the new number. @@ -50,37 +51,43 @@ def get_next_sub_or_ses( Parameters ---------- - cfg : Configs + cfg datashuttle configs class - top_level_folder: TopLevelFolder + top_level_folder The top-level folder (e.g. `"rawdata"`, `"derivatives"`) - sub : Optional[str] - subject name to search within if searching for sessions, otherwise None + sub + Subject name to search within if searching for sessions, otherwise None to search for subjects - search_str : str - the string to search for within the top-level or subject-level + search_str + The string to search for within the top-level or subject-level folder ("sub-*") or ("ses-*") are suggested, respectively. - include_central : bool + include_central If `False, only get names from `local_path`, otherwise from `local_path` and `central_path`. - return_with_prefix : bool + return_with_prefix If `True`, the next sub or ses value will include the prefix e.g. "sub-001", otherwise the value alone will be returned (e.g. "001") - default_num_value_digits : int + default_num_value_digits If no sub or ses exist in the project, the starting number is 1. Because the number of digits for the project is not accessible, the desired value can be entered here. e.g. if 3 (the default), if no subjects are found the subject returned will be "sub-001". + name_template_regexp + the name template to try and get the num digits from. + If unspecified, the number of digits will be default_num_value_digits. + Returns ------- - suggested_new_num : the new suggested sub / ses. + suggested_new_num + the new suggested sub / ses. + """ prefix: Prefix @@ -121,28 +128,26 @@ def get_max_sub_or_ses_num_and_value_length( default_num_value_digits: Optional[int] = None, name_template_regexp: Optional[str] = None, ) -> Tuple[int, int]: - """ - Given a list of BIDS-style folder names, find the maximum subject or - session value (sub or ses depending on `prefix`). Also, find the - number of value digits across the project, so a new suggested number - can be formatted consistency. If the list is empty, set the value + """Find the maximum subject or session value given a list of BIDS-style folder names. + + Also, find the number of value digits across the project, so a new suggested + number can be formatted consistency. If the list is empty, set the value to 0 and a default number of value digits. Parameters ---------- - - all_folders : List[str] + all_folders A list of BIDS-style formatted folder names. - see `get_next_sub_or_ses()` for other arguments. + prefix, default_num_value_digits, name_template_regexp + see `get_next_sub_or_ses()`. Returns ------- - - max_existing_num : int + max_existing_num The largest number sub / ses value in the past list. - num_value_digits : int + num_value_digits The length of the value in all sub / ses values within the passed list. If these are not consistent, an error is raised. @@ -151,10 +156,9 @@ def get_max_sub_or_ses_num_and_value_length( """ if len(all_folders) == 0: - - assert isinstance( - default_num_value_digits, int - ), "`default_num_value_digits` must be int`" + assert isinstance(default_num_value_digits, int), ( + "`default_num_value_digits` must be int`" + ) max_existing_num = 0 @@ -212,10 +216,16 @@ def get_max_sub_or_ses_num_and_value_length( def get_num_value_digits_from_project( all_values_str: List[str], prefix: Prefix ) -> int: - """ - Find the number of digits for the sub or ses key within the project. - `all_values_str` is a list of all the sub or ses values from within - the project. + """Return the number of digits for the sub or ses key within the project. + + Parameters + ---------- + all_values_str + A list of all the sub or ses values from within the project. + + prefix + "sub" or "ses". + """ all_num_value_digits = [len(value) for value in all_values_str] @@ -233,12 +243,26 @@ def get_num_value_digits_from_project( def get_num_value_digits_from_regexp( prefix: Prefix, name_template_regexp: str ) -> Union[Literal[False], int]: - r""" - Given a name template regexp, find the number of values for the - sub or ses key. These will be fixed with "\d" (digit) or ".?" (wildcard). + r"""Given a name template regexp, find the number of values for the sub or ses key. + + These will be fixed with "\d" (digit) or ".?" (wildcard). If there is length-unspecific wildcard (.*) in the sub key, then skip. In practice, there should never really be a .* in the sub or ses key of a name template, but handle it just in case. + + Parameters + ---------- + prefix + "sub" or "ses". + + name_template_regexp + Regexp for the name template to validate against. + + Returns + ------- + num_digits + Number of digits in the sub- or ses- value, or `False` if wildcard searching. + """ all_values_str = utils.get_values_from_bids_formatted_name( [name_template_regexp], prefix, return_as_int=False @@ -260,9 +284,9 @@ def get_num_value_digits_from_regexp( def get_existing_project_paths() -> List[Path]: - """ - Return full path and names of datashuttle projects on - this local machine. A project is determined by a project + """Return full path and names of datashuttle projects on this local machine. + + A project is determined by a project folder in the home / .datashuttle folder that contains a config.yaml file. Returns in order of most recently modified first. @@ -299,26 +323,31 @@ def get_all_sub_and_ses_paths( top_level_folder: TopLevelFolder, include_central: bool, ) -> Dict: - """ - Get a list of every subject and session name in the - local and central project folders. Local and central names are combined - into a single list, separately for subject and sessions. + """Return a dict including filepaths to all subjects and sessions. + + Local and central names are combined into a single list, + separately for subject and sessions. Note this only finds local sub and ses names on this machine. Other local machines are not searched. Parameters ---------- + cfg + Datashuttle Configs. - cfg : Configs - datashuttle Configs - - top_level_folder: TopLevelFolder + top_level_folder The top-level folder (e.g. `"rawdata"`, `"derivatives"`) - include_central : bool + include_central If `False, only get names from `local_path`, otherwise from `local_path` and `central_path`. + + Returns + ------- + A dictionary with "sub" key (path to all subject folders) + and "ses" key (path to all session folders). + """ sub_folder_paths = folders.search_project_for_sub_or_ses_names( cfg, @@ -338,7 +367,6 @@ def get_all_sub_and_ses_paths( all_ses_folder_paths = {} for sub_path in all_sub_folder_paths: - sub = sub_path.name ses_folder_paths = folders.search_project_for_sub_or_ses_names( diff --git a/datashuttle/utils/rclone.py b/datashuttle/utils/rclone.py index 49d7da826..2d66a8cc1 100644 --- a/datashuttle/utils/rclone.py +++ b/datashuttle/utils/rclone.py @@ -12,15 +12,20 @@ def call_rclone(command: str, pipe_std: bool = False) -> CompletedProcess: - """ - Call rclone with the specified command. Current mode is double-verbose. - Return the completed process from subprocess. + """Call rclone with the specified command. Parameters ---------- - command: Rclone command to be run + command + Rclone command to be run + + pipe_std + if True, do not output anything to stdout. + + Returns + ------- + subprocess.CompletedProcess with `stdout` and `stderr` attributes. - pipe_std: if True, do not output anything to stdout. """ command = "rclone " + command if pipe_std: @@ -34,10 +39,20 @@ def call_rclone(command: str, pipe_std: bool = False) -> CompletedProcess: def call_rclone_through_script(command: str) -> CompletedProcess: - """ - Call rclone through a script, to avoid limits on command-line calls - (in particular on Windows). Used for transfers due to generation of - large call strings. + """Call rclone through a script. + + This is to avoid limits on command-line calls (in particular on Windows). + Used for transfers due to generation of large call strings. + + Parameters + ---------- + command + Full command to run with RClone. + + Returns + ------- + subprocess.CompletedProcess with `stdout` and `stderr` attributes. + """ system = platform.system() @@ -80,8 +95,9 @@ def call_rclone_through_script(command: str) -> CompletedProcess: def setup_rclone_config_for_local_filesystem( rclone_config_name: str, log: bool = True, -): - """ +) -> None: + """Set the RClone remote config for local filesystem. + RClone sets remote targets in a config file that are used at transfer. For local filesystem, this is essentially a placeholder and that is not linked to a particular filepath. @@ -96,12 +112,13 @@ def setup_rclone_config_for_local_filesystem( Parameters ---------- + rclone_config_name + canonical config name, generated by + datashuttle.cfg.get_rclone_config_name() - rclone_config_name : rclone config name - canonical config name, generated by - datashuttle.cfg.get_rclone_config_name() + log + whether to log, if True logger must already be initialised. - log : whether to log, if True logger must already be initialised. """ call_rclone(f"config create {rclone_config_name} local", pipe_std=True) @@ -114,26 +131,29 @@ def setup_rclone_config_for_ssh( rclone_config_name: str, ssh_key_path: Path, log: bool = True, -): - """ - RClone sets remote targets in a config file that are - used at transfer. For SSH, this must contain the central path, - username and ssh key. The relative path is supplied at transfer time. +) -> None: + """Set the RClone remote config for ssh. - Parameters - ---------- + RClone sets remote targets in a config file that are + used at transfer. For SSH, this must contain the central path, + username and ssh key. The relative path is supplied at transfer time. + + Parameters + ---------- + cfg + datashuttle configs UserDict. - cfg : Configs - datashuttle configs UserDict. + rclone_config_name + canonical config name, generated by + datashuttle.cfg.get_rclone_config_name() - rclone_config_name : rclone config name - canonical config name, generated by - datashuttle.cfg.get_rclone_config_name() + ssh_key_path + path to the ssh key used for connecting to + ssh central filesystem - ssh_key_path : path to the ssh key used for connecting to - ssh central filesystem, + log + whether to log, if True logger must already be initialised. - log : whether to log, if True logger must already be initialised. """ call_rclone( f"config create " @@ -150,18 +170,16 @@ def setup_rclone_config_for_ssh( log_rclone_config_output() -def log_rclone_config_output(): +def log_rclone_config_output() -> None: + """Log the output from creating Rclone config.""" output = call_rclone("config file", pipe_std=True) utils.log( - f"Successfully created rclone config. " - f"{output.stdout.decode('utf-8')}" + f"Successfully created rclone config. {output.stdout.decode('utf-8')}" ) def check_rclone_with_default_call() -> bool: - """ - Check to see whether rclone is installed. - """ + """Return a bool indicating whether rclone is installed.""" try: output = call_rclone("-h", pipe_std=True) except FileNotFoundError: @@ -170,10 +188,7 @@ def check_rclone_with_default_call() -> bool: def prompt_rclone_download_if_does_not_exist() -> None: - """ - Check that rclone is installed. If it does not - (e.g. first time using datashuttle) then download. - """ + """Check that rclone is installed.""" if not check_rclone_with_default_call(): raise BaseException( "RClone installation not found. Install by entering " @@ -194,28 +209,31 @@ def transfer_data( include_list: List[str], rclone_options: Dict, ) -> subprocess.CompletedProcess: - """ - Transfer data by making a call to Rclone. + """Transfer data by making a call to Rclone. Parameters ---------- - - cfg: Configs + cfg datashuttle configs - upload_or_download : Literal["upload", "download"] + upload_or_download If "upload", transfer from `local_path` to `central_path`. "download" proceeds in the opposite direction. - top_level_folder: Literal["rawdata", "derivatives"] + top_level_folder The top-level-folder to transfer files within. - include_list : List[str] + include_list A list of filepaths to include in the transfer - rclone_options : Dict + rclone_options A list of options to pass to Rclone's copy function. see `cfg.make_rclone_transfer_options()`. + + Returns + ------- + subprocess.CompletedProcess with `stdout` and `stderr` attributes. + """ assert upload_or_download in [ "upload", @@ -251,28 +269,31 @@ def get_local_and_central_file_differences( cfg: Configs, top_level_folders_to_check: List[TopLevelFolder], ) -> Dict: - """ - Convert the output of rclone's check (with `--combine`) flag - to a dictionary separating each case. + """Format a structure of all changes between local and central. Rclone output comes as a list of files, separated by newlines, with symbols indicating whether the file paths are same across local and central, different, or found in local / central only. + Convert the output of Rclone's check (with `--combine`) flag + to a dictionary separating each case. + Parameters ---------- + cfg + datashuttle configs UserDict. - top_level_folders_to_check : + top_level_folders_to_check List of top-level folders to check. Returns ------- - - parsed_output : Dict[str, List] + parsed_output A dictionary where the keys are the cases (e.g. "same" across local and central) and the values are lists of paths that fall into these cases. Note the paths are relative to the "rawdata" folder. + """ convert_symbols = { "=": "same", @@ -286,7 +307,6 @@ def get_local_and_central_file_differences( parsed_output = {val: [] for val in convert_symbols.values()} for top_level_folder in top_level_folders_to_check: - rclone_output = perform_rclone_check(cfg, top_level_folder) # type: ignore split_rclone_output = rclone_output.split("\n") @@ -307,10 +327,10 @@ def get_local_and_central_file_differences( def assert_rclone_check_output_is_as_expected(result, symbol, convert_symbols): - """ - Ensure the output of Rclone check is as expected. Currently, the "error" - case is untested and a test case is required. Once the test case is - obtained this should most likely be moved to tests. + """Ensure the output of Rclone check is as expected. + + Currently, the "error" case is untested and a test case is required. + Once the test case is obtained this should most likely be moved to tests. """ assert result[1] == " ", ( "`rclone check` output does not contain a " @@ -327,10 +347,11 @@ def assert_rclone_check_output_is_as_expected(result, symbol, convert_symbols): def perform_rclone_check( cfg: Configs, top_level_folder: TopLevelFolder ) -> str: - """ + r"""Run RClone check to find differences in files between local and central. + Use Rclone's `check` command to build a list of files that are the same ("="), different ("*"), found in local only ("+") - or central only ("-"). The output is formatted as " \n". + or central only ("-"). The output is formatted as "\ \\n". """ local_filepath = cfg.get_base_folder( "local", top_level_folder @@ -340,7 +361,7 @@ def perform_rclone_check( ).parent.as_posix() output = call_rclone( - f'{rclone_args("check")} ' + f"{rclone_args('check')} " f'"{local_filepath}" ' f'"{cfg.get_rclone_config_name()}:{central_filepath}"' f" --combined -", @@ -353,8 +374,21 @@ def perform_rclone_check( def handle_rclone_arguments( rclone_options: Dict, include_list: List[str] ) -> str: - """ - Construct the extra arguments to pass to RClone, + """Construct the extra arguments to pass to RClone. + + Parameters + ---------- + rclone_options + A list of option keywords to be passed to + + include_list + The (already formatted) list of filepaths for the + rclone `--include` option. + + Returns + ------- + A full list of arguments to pass to rclone. + """ extra_arguments_list = [] @@ -385,9 +419,7 @@ def handle_rclone_arguments( def rclone_args(name: str) -> str: - """ - Central function to hold rclone commands - """ + """Return list of Rclone commands.""" valid_names = [ "dry_run", "copy", diff --git a/datashuttle/utils/ssh.py b/datashuttle/utils/ssh.py index 8f6de6785..587e9416a 100644 --- a/datashuttle/utils/ssh.py +++ b/datashuttle/utils/ssh.py @@ -28,7 +28,23 @@ def connect_client_core( client: paramiko.SSHClient, cfg: Configs, password: Optional[str] = None, -): +) -> None: + """Connect to the client. + + A centralised function to connect to a paramiko client. + + Parameters + ---------- + client + Paramiko client to connect to. + + cfg + Datashuttle Configs. + + password + Password (if required) to establish the connection. + + """ client.get_host_keys().load(cfg.hostkeys_path.as_posix()) client.set_missing_host_key_policy(paramiko.RejectPolicy()) @@ -48,8 +64,19 @@ def connect_client_core( def add_public_key_to_central_authorized_keys( cfg: Configs, password: str, log=True ) -> None: - """ - Append the public part of key to central server ~/.ssh/authorized_keys. + """Append the public part of key to central server ~/.ssh/authorized_keys. + + Parameters + ---------- + cfg + Datashuttle Configs object. + + password + Password to the central server. + + log + If `True`, log the client connection process. + """ generate_and_write_ssh_key(cfg.ssh_key_path) @@ -73,14 +100,26 @@ def add_public_key_to_central_authorized_keys( def generate_and_write_ssh_key(ssh_key_path: Path) -> None: + """Generate an RSA SSH key and save it to the specified file path. + + Parameters + ---------- + ssh_key_path + The full file path where the private SSH key will be saved. + + """ key = paramiko.RSAKey.generate(4096) key.write_private_key_file(ssh_key_path.as_posix()) def get_remote_server_key(central_host_id: str): - """ - Get the remove server host key for validation before - connection. + """Get the remove server host key for validation before connection. + + Parameters + ---------- + central_host_id + The hostname or IP address of the central host. + """ transport: paramiko.Transport with paramiko.Transport(central_host_id) as transport: @@ -90,6 +129,24 @@ def get_remote_server_key(central_host_id: str): def save_hostkey_locally(key, central_host_id, hostkeys_path) -> None: + """Save the SSH host key locally to the specified hostkeys file. + + The host key uniquely identifies the SSH server to prevent + man-in-the-middle attacks by verifying the server's identity + on future connections. + + Parameters + ---------- + key + The SSH host key to save. + + central_host_id + The hostname or IP address of the central host. + + hostkeys_path + The file path where host keys are stored locally. + + """ client = paramiko.SSHClient() client.get_host_keys().add(central_host_id, key.get_name(), key) client.get_host_keys().save(hostkeys_path.as_posix()) @@ -106,26 +163,28 @@ def setup_ssh_key( cfg: Configs, log: bool = True, ) -> None: - """ - Set up an SSH private / public key pair with - central server. First, a private key is generated - and saved in the .datashuttle config path. - Next a connection requiring input - password made, and the public part of the key + """Set up an SSH private / public key pair with central server. + + First, a private key is generated and saved in the .datashuttle config path. + Next a connection requiring input password made, and the public part of the key added to ~/.ssh/authorized_keys. Parameters - ----------- - - ssh_key_path : path to the ssh private key + ---------- + ssh_key_path + path to the ssh private key - hostkeys_path : path to the ssh host key, once the user + hostkeys_path + path to the ssh host key, once the user has confirmed the key ID this is saved so verification is not required each time. - cfg : datashuttle config UserDict + cfg + datashuttle config UserDict + + log + log if True, logger must already be initialised. - log : log if True, logger must already be initialised. """ if not sys.stdin.isatty(): proceed = input( @@ -171,8 +230,8 @@ def connect_client_with_logging( password: Optional[str] = None, message_on_sucessful_connection: bool = True, ) -> None: - """ - Connect client to central server using paramiko. + """Connect client to central server using paramiko. + Accept either password or path to private key, but not both. Paramiko does not support pathlib. """ @@ -180,7 +239,7 @@ def connect_client_with_logging( connect_client_core(client, cfg, password) if message_on_sucessful_connection: utils.print_message_to_user( - f"Connection to { cfg['central_host_id']} made successfully." + f"Connection to {cfg['central_host_id']} made successfully." ) except Exception: @@ -199,10 +258,28 @@ def connect_client_with_logging( def verify_ssh_central_host( central_host_id: str, hostkeys_path: Path, log: bool = True ) -> bool: - """ - Similar to connecting with other SSH manager e.g. putty, - get the server key and present when connecting - for manual validation. + """Prompt the user to verify and cache the SSH server's host key. + + This function retrieves the SSH server's key and asks the user to + manually validate and accept it. Accepting the key caches it locally + to ensure secure future connections. + + Parameters + ---------- + central_host_id + Hostname or IP address of the SSH server. + + hostkeys_path + Path to the local file where known host keys are stored. + + log + Whether to log the verification messages. + + Returns + ------- + bool + True if the host key was accepted and saved, False otherwise. + """ key = get_remote_server_key(central_host_id) @@ -246,21 +323,30 @@ def search_ssh_central_for_folders( verbose: bool = True, return_full_path: bool = False, ) -> Tuple[List[Any], List[Any]]: - """ - Search for the search prefix in the search path over SSH. - Returns the list of matching folders, files are filtered out. + """Search for the search prefix in the search path over SSH. Parameters - ----------- + ---------- + search_path + Path to search for folders in. + + search_prefix + Search prefix for folder names e.g. "sub-*". + + cfg + See connect_client_with_logging(). - search_path : path to search for folders in + verbose + If `True`, if a search folder cannot be found, a message + will be printed with the un-found path. - search_prefix : search prefix for folder names e.g. "sub-*" + return_full_path + include the search_path in the returned paths - cfg : see connect_client_with_logging() + Returns + ------- + Discovered folders (`all_folder_names`) and files (`all_filenames`). - verbose : If `True`, if a search folder cannot be found, a message - will be printed with the un-found path. """ client: paramiko.SSHClient with paramiko.SSHClient() as client: @@ -288,29 +374,39 @@ def get_list_of_folder_names_over_sftp( verbose: bool = True, return_full_path: bool = False, ) -> Tuple[List[Any], List[Any]]: - """ - Use paramiko's sftp to search a path - over ssh for folders. Return the folder names. + """Use paramiko's sftp to search a path over ssh for folders. + + Return the folder names. Parameters ---------- + sftp + Connected paramiko stfp object + (see search_ssh_central_for_folders()). - stfp : connected paramiko stfp object - (see search_ssh_central_for_folders()) + search_path + Path to search for folders in. - search_path : path to search for folders in - - search_prefix : prefix (can include wildcards) + search_prefix + Prefix (can include wildcards) to search folder names. - verbose : If `True`, if a search folder cannot be found, a message - will be printed with the un-found path. + verbose + If `True`, if a search folder cannot be found, a message + will be printed with the un-found path. + + return_full_path + include the search_path in the returned paths. + + Returns + ------- + Discovered folders (`all_folder_names`) and files (`all_filenames`). + """ all_folder_names = [] all_filenames = [] try: for file_or_folder in sftp.listdir_attr(search_path.as_posix()): - if file_or_folder.st_mode is not None and fnmatch.fnmatch( file_or_folder.filename, search_prefix ): diff --git a/datashuttle/utils/utils.py b/datashuttle/utils/utils.py index 87a39e5c9..7cc9df541 100644 --- a/datashuttle/utils/utils.py +++ b/datashuttle/utils/utils.py @@ -19,28 +19,30 @@ def log(message: str) -> None: - """ - Log the message to the main initialised - logger. - """ + """Log the message to the main initialised logger.""" if ds_logger.logging_is_active(): logger = ds_logger.get_logger() logger.debug(message) def log_and_message(message: str, use_rich: bool = False) -> None: - """ - Log the message and send it to user. - use_rich : is True, use rich's print() function + """Log the message and send it to user. + + Parameters + ---------- + message + Message to log and print to user. + + use_rich + If True, use rich's print() function. + """ log(message) print_message_to_user(message, use_rich) def log_and_raise_error(message: str, exception: Any) -> None: - """ - Log the message before raising the same message as an error. - """ + """Log the message before raising the same message as an error.""" if ds_logger.logging_is_active(): logger = ds_logger.get_logger() logger.error(f"\n\n{' '.join(traceback.format_stack(limit=5))}") @@ -49,7 +51,17 @@ def log_and_raise_error(message: str, exception: Any) -> None: def warn(message: str, log: bool) -> None: - """ """ + """Send a warning. + + Parameters + ---------- + message + Message to warn. + + log + If True, log at WARNING level. + + """ if log and ds_logger.logging_is_active(): logger = ds_logger.get_logger() logger.warning(message) @@ -57,10 +69,10 @@ def warn(message: str, log: bool) -> None: def raise_error(message: str, exception) -> None: - """ - Centralized way to raise an error. The logger is closed - to ensure it is not still running if a function call - raises an exception in a python environment. + """Centralized way to raise an error. + + The logger is closed to ensure it is not still running + if a function call raises an exception in a python environment. """ ds_logger.close_log_filehandler() raise exception(message) @@ -69,9 +81,16 @@ def raise_error(message: str, exception) -> None: def print_message_to_user( message: Union[str, list], use_rich: bool = False ) -> None: - """ - Centralised way to send message. - use_rich : use rich's print() function. + """Centralised way to send message. + + Parameters + ---------- + message + Message to print. + + use_rich + If True, use rich's print() function. + """ if use_rich: rich_print(message) @@ -80,9 +99,7 @@ def print_message_to_user( def get_user_input(message: str) -> str: - """ - Centralised way to get user input - """ + """Centralised way to get user input.""" input_ = input(message) return input_ @@ -93,6 +110,7 @@ def get_user_input(message: str) -> str: def path_starts_with_base_folder(base_folder: Path, path_: Path) -> bool: + """Return a bool indicating whether the path starts with the base folder path.""" return path_.as_posix().startswith(base_folder.as_posix()) @@ -125,20 +143,36 @@ def get_values_from_bids_formatted_name( return_as_int: bool = False, sort: bool = False, ) -> Union[List[int], List[str]]: - """ - Find the values associated with a key from a list of all - BIDS-formatted file / folder names. This is typically used to - find sub / ses values. + """Find the values associated with a key in a BIDS-style name. + + Parameters + ---------- + all_names + A list of names from which to find the value associated with the key. + + key + Key from which to associate the values e.g. "sub") + + return_as_int + If True and the value can be cast to int (e.g. `sub-001`), return as `int`. + + sort + If True, results are sorted before being returned. + + Returns + ------- + all_values + The values of the corresponding `key` extracted from the name. Notes ----- This function does not raise through datashuttle because we don't want to turn off logging, as some times these exceptions are caught and skipped. + """ all_values = [] for name in all_names: - if key not in name: raise NeuroBlueprintError( f"The key {key} is not found in {name}", KeyError @@ -167,6 +201,7 @@ def get_values_from_bids_formatted_name( def sub_or_ses_value_to_int(value: str) -> int: + """Return a subject or session value converted to an integer.""" try: int_value = int(value) except ValueError: @@ -177,11 +212,9 @@ def sub_or_ses_value_to_int(value: str) -> int: def get_value_from_key_regexp(name: str, key: str) -> List[str]: - """ - Find the value related to the key in a - BIDS-style key-value pair name. - e.g. sub-001_ses-312 would find - 312 for key "ses". + """Return the value related to the key in a BIDS-style key-value pair name. + + e.g. sub-001_ses-312 would find 312 for key "ses". """ return re.findall(f"{key}-(.*?)(?=_|$)", name) @@ -192,35 +225,36 @@ def get_value_from_key_regexp(name: str, key: str) -> List[str]: def integers_are_consecutive(list_of_ints: List[int]) -> bool: + """Return a bool indicating whether a list of integers is consecutive.""" diff_between_ints = diff(list_of_ints) return all([diff == 1 for diff in diff_between_ints]) def diff(x: List) -> List: - """ - slow, custom differentiator for small inputs, to avoid - adding numpy as a dependency. + """Return differentiated list of numbers. + + Slow, only to avoid adding numpy as a dependency. """ return [x[i + 1] - x[i] for i in range(len(x) - 1)] -def num_leading_zeros(string: str) -> int: - """int() strips leading zeros""" - if string[:4] in ["sub-", "ses-"]: - string = string[4:] +def num_leading_zeros(name: str) -> int: + """Return the number of leading zeros in a sub- or ses- id. + + e.g. sub-001 has 2 leading zeros. + int() strips leading zeros. + """ + if name[:4] in ["sub-", "ses-"]: + name = name[4:] - return len(string) - len(str(int(string))) + return len(name) - len(str(int(name))) def all_unique(list_: List) -> bool: - """ - Check that all values in a list are different. - """ + """Return bool indicating whether all values in a list are different.""" return len(list_) == len(set(list_)) def all_identical(list_: List) -> bool: - """ - Check that all values in a list are identical. - """ + """Return bool indicating whether all values in a list are identical.""" return len(set(list_)) == 1 diff --git a/datashuttle/utils/validation.py b/datashuttle/utils/validation.py index 28a01f04f..59ad5fbe2 100644 --- a/datashuttle/utils/validation.py +++ b/datashuttle/utils/validation.py @@ -34,6 +34,7 @@ def get_missing_prefix_error(name: str, prefix, path_: Path | None) -> str: + """Return error message when a required prefix is missing from a name.""" return handle_path( f"MISSING_PREFIX: The prefix {prefix} was not found in the name: {name}", path_, @@ -41,6 +42,7 @@ def get_missing_prefix_error(name: str, prefix, path_: Path | None) -> str: def get_bad_value_error(name: str, prefix, path_: Path | None) -> str: + """Return error message when the value for a prefix is not an integer.""" return handle_path( f"BAD_VALUE: The value for prefix {prefix} in name {name} is not an integer.", path_, @@ -48,19 +50,22 @@ def get_bad_value_error(name: str, prefix, path_: Path | None) -> str: def get_duplicate_prefix_error(name: str, prefix, path_: Path | None) -> str: + """Return error message when a name contains multiple instances of the same prefix.""" return handle_path( - f"DUPLICATE_PREFIX: The name: {name} of contains more than one instance of the prefix {prefix}.", + f"DUPLICATE_PREFIX: The name: {name} contains more than one instance of the prefix {prefix}.", path_, ) def get_name_error(name: str, prefix: Prefix, path_: Path | None) -> str: + """Return error message when a name is invalid for a given prefix.""" return handle_path( f"BAD_NAME: The name: {name} of type: {prefix} is not valid.", path_ ) def get_special_char_error(name: str, path_: Path | None) -> str: + """Return error message when a name contains invalid characters.""" return handle_path( f"SPECIAL_CHAR: The name: {name}, contains characters which are not alphanumeric, dash or underscore.", path_, @@ -68,6 +73,7 @@ def get_special_char_error(name: str, path_: Path | None) -> str: def get_name_format_error(name: str, path_: Path | None) -> str: + """Return error message when a name does not follow key-value pair format.""" return handle_path( f"NAME_FORMAT: The name {name} does not consist of key-value pairs separated by underscores.", path_, @@ -75,10 +81,12 @@ def get_name_format_error(name: str, path_: Path | None) -> str: def get_value_length_error(prefix: Prefix) -> str: + """Return error message for inconsistent value lengths for a prefix.""" return f"VALUE_LENGTH: Inconsistent value lengths for the prefix: {prefix} were found in the project." def get_datetime_error(key, name: str, strfmt: str, path_: Path | None) -> str: + """Return error message when a datetime value is not in the expected ISO format.""" return handle_path( f"DATETIME: Name {name} contains an invalid {key}. It should be ISO format: {strfmt}.", path_, @@ -86,9 +94,9 @@ def get_datetime_error(key, name: str, strfmt: str, path_: Path | None) -> str: def get_template_error(name: str, regexp: str, path_: Path | None) -> str: - """ - The missing full-stop at the end is intentional, to avoid - confusion when reading the regexp. + """Return error message when a name does not match a given template. + + The missing full-stop at the end is intentional, to avoid confusion when reading the regexp. """ return handle_path( f"TEMPLATE: The name: {name} does not match the template: {regexp}", @@ -99,6 +107,7 @@ def get_template_error(name: str, regexp: str, path_: Path | None) -> str: def get_missing_top_level_folder_error( path_: Path | None, local_or_central: Literal["local", "central"] ) -> str: + """Return error message when the top level folder is missing from the project.""" return handle_path( f"TOP_LEVEL_FOLDER: The {local_or_central} project must contain a 'rawdata' or 'derivatives' folder.", path_, @@ -108,6 +117,7 @@ def get_missing_top_level_folder_error( def get_duplicate_name_error( new_name: str, exist_name: str, exist_path: Path | None ) -> str: + """Return error message when a new name duplicates an existing name.""" return handle_path( f"DUPLICATE_NAME: The prefix for {new_name} duplicates the name: {exist_name}.", exist_path, @@ -115,12 +125,14 @@ def get_duplicate_name_error( def get_datatype_error(datatype_name: str, path_: Path | None) -> str: + """Return error message when an invalid datatype name is encountered.""" return handle_path( f"DATATYPE: {datatype_name} is not a valid datatype name.", path_ ) def handle_path(message: str, path_: Path | None) -> str: + """Append the file path to the error message if available.""" if path_: message += f" Path: {path_.as_posix()}" return message @@ -137,25 +149,28 @@ def validate_list_of_names( name_templates: Optional[Dict] = None, check_value_lengths: bool = True, ) -> List[str]: - """ - Validate a list of subject or session names, ensuring - they are formatted as per NeuroBlueprint. + """Validate a list of subject or session names against NeuroBlueprint. Parameters ---------- - - path_or_name_list : List[Path] + path_or_name_list A list of pathlib.Path to NeuroBlueprint-formatted folders to validate - prefix: Prefix + prefix Whether these are subject (sub) or session (ses) level names - name_templates: Optional[Dict] + name_templates A `name_template` dictionary to validate against. See `set_name_templates()`. - check_value_lengths : bool + check_value_lengths If `True`, check that the prefix- value lengths are consistent across the passed list. + + Returns + ------- + error_messages + A list of found validation errors. + """ if len(path_or_name_list) == 0: return [] @@ -164,7 +179,6 @@ def validate_list_of_names( # First, just validate each name individually for path_or_name in path_or_name_list: - path_, name = get_path_and_name(path_or_name) error_messages += prefix_is_duplicate_or_has_bad_values( @@ -190,7 +204,6 @@ def validate_list_of_names( ) for path_or_name in stripped_path_or_names_list: - path_, name = get_path_and_name(path_or_name) error_messages += new_name_duplicates_existing( @@ -208,10 +221,26 @@ def validate_list_of_names( def prefix_is_duplicate_or_has_bad_values( name: str, prefix: Prefix, path_: Path | None ) -> List[str]: - """ - Check that the prefix (sub- or ses-) is found only - once in the name and that its value can be converted - to integer. + """Check the sub- or ses- prefix. + + Ensure it is found only once in the name and + that its value can be converted to integer. + + Parameters + ---------- + name + Name to check against template. + + prefix + "sub" or "ses" + + path_ + Path to the folder that is being checked. + + Returns + ------- + A list of validation errors. + """ value = re.findall(f"{prefix}(.*?)(?=_|$)", name) @@ -233,16 +262,29 @@ def new_name_duplicates_existing( existing_path_or_name_list: List[Path] | List[str], prefix: Prefix, ) -> List[str]: - """ - Check that a subject or session value does not duplicate - an existing value. The only case this is allowed is - when the names match exactly. - - For example, if "sub-001" exists, we can pass - "sub-001" as a valid subject name (for example, when making sessions). - However, if "sub-001_another-tag" exists, we should throw an - error, because this shares the same subject id but refers to - a different subject. + """Check that a subject or session value does not duplicate an existing value. + + The only case this is allowed is when the names match exactly. + For example, if "sub-001" exists, we can pass "sub-001" as a valid subject name + (for example, when making sessions). However, if "sub-001_another-tag" exists, + we should throw an error, because this shares the same subject id but refers + to a different subject. + + Parameters + ---------- + new_name + The name to check against all existing names. + + existing_path_or_name_list + A list of existing names to check against. + + prefix + "sub" or "ses" + + Returns + ------- + A list of validation errors. + """ # Make a list of matches between `new_name` and any in `existing_names` new_name_id = utils.get_values_from_bids_formatted_name( @@ -251,7 +293,6 @@ def new_name_duplicates_existing( error_messages = [] for exist_path_or_name in existing_path_or_name_list: - exist_path, exist_name = get_path_and_name(exist_path_or_name) exist_name_id = utils.get_values_from_bids_formatted_name( @@ -274,9 +315,26 @@ def names_dont_match_templates( prefix: Prefix, name_templates: Optional[Dict] = None, ) -> List[str]: - """ - Test a list of subject or session names against - the respective `name_templates`, a regexp template. + """Validate a list of sub/ses names against the respective regexp `name_templates`. + + Parameters + ---------- + name + Name to check against template. + + path_ + Path to the folder that is being checked. + + prefix + "sub" or "ses" + + name_templates + Datashuttle's Name Templates dictionary defining the templates used. + + Returns + ------- + A list of validation errors. + """ if name_templates is None: return [] @@ -298,11 +356,7 @@ def names_dont_match_templates( def get_path_and_name(path_or_name: Path | str) -> Tuple[Optional[Path], str]: - """ - Convenience function to get the folder name - from something that is either a Path (pointing - to the folder) or a str of the folder name itself. - """ + """Return the folder name from a Path (to the folder) or a str of the name itself.""" if isinstance(path_or_name, Path): return path_or_name, path_or_name.name else: @@ -310,15 +364,24 @@ def get_path_and_name(path_or_name: Path | str) -> Tuple[Optional[Path], str]: def replace_tags_in_regexp(regexp: str) -> str: - r""" - Before validation, all tags in the names are converted to - their final values (e.g. @DATE@ -> _date-). We also want to - allow template to be formatted like `sub-\d\d_@DATE@` as it - is convenient for auto-completion in the TUI. - - Therefore, we must replace the tags in the regexp with their - actual regexp equivalent before comparison. + r"""Before validation, all tags in the names are converted to their final values. + + For example, (e.g. @DATE@ -> _date-). + We also want to allow template to be formatted like `sub-\d\d_@DATE@` as it is + convenient for auto-completion in the TUI. Therefore, we must replace the tags + in the regexp with their actual regexp equivalent before comparison. + Note `replace_date_time_tags_in_name()` operates in place on a list. + + Parameters + ---------- + regexp + The name template regexp in which to replace tags. + + Returns + ------- + The regexp with tags (e.g. @DATE@) formatted properly. + """ regexp_list = [regexp] date_regexp = r"\d\d\d\d\d\d\d\d" @@ -336,9 +399,23 @@ def replace_tags_in_regexp(regexp: str) -> str: def name_begins_with_bad_key( name: str, prefix: Prefix, path_: Path | None ) -> List[str]: - """ - Check that a list of NeuroBlueprint names begin - with the required prefix (sub- or ses-). + """Check that a list of NeuroBlueprint names begin with the required prefix (sub- or ses-). + + Parameters + ---------- + name + Name of the folder to validate. + + prefix + "sub" or "ses". + + path_ + Path to the folder that is being validated. + + Returns + ------- + A list of validation errors. + """ if name[:4] != f"{prefix}-": return [get_name_error(name, prefix, path_)] @@ -349,10 +426,22 @@ def name_begins_with_bad_key( def names_include_special_characters( name: str, path_: Path | None ) -> List[str]: - """ - Check that a list of NeuroBlueprint formatted - names do not contain special characters (i.e. characters - that are not integers, letters, dash or underscore). + """Check that a list of NeuroBlueprint formatted names do not contain special characters. + + Special characters are characters that are not integers, letters, dash or underscore. + + Parameters + ---------- + name + Name of the folder to validate + + path_ + Path of the folder that is being validated. + + Returns + ------- + A list of validation errors. + """ if name_has_special_character(name): return [get_special_char_error(name, path_)] @@ -361,17 +450,30 @@ def names_include_special_characters( def name_has_special_character(name: str) -> bool: + """Return a bool indicating if the name contains special characters.""" return not re.match("^[A-Za-z0-9_-]*$", name) def dashes_and_underscore_alternate_incorrectly( name: str, path_: Path | None ) -> List[str]: - """ - Check a list of NeuroBlueprint formatted names - have the "-" and "-" ordered correctly. Names should be - key-value pairs separated by underscores e.g. - sub-001_ses-001. + """Check a list of names for expected Neuroblueprint underscore-dash order. + + Names should have the "-" and "-" ordered correctly. Names should be + key-value pairs separated by underscores e.g. sub-001_ses-001. + + Parameters + ---------- + name + Name of the folder to validate + + path_ + Path of the folder that is being validated. + + Returns + ------- + A list of validation errors. + """ discrim = {"-": 1, "_": -1} @@ -386,7 +488,7 @@ def dashes_and_underscore_alternate_incorrectly( or dashes_underscores[0] != 1 # first must be - or dashes_underscores[-1] != 1 # last must be - or underscore_dash_not_interleaved - or (name[-1] in discrim.keys()) # name cannot end with - or _ + or (name[-1] in discrim) # name cannot end with - or _ ): return [get_name_format_error(name, path_)] else: @@ -397,10 +499,22 @@ def value_lengths_are_inconsistent( path_or_names_list: List[Path] | List[str] | List[Path | str], prefix: Prefix, ) -> List[str]: - """ - Given a list of NeuroBlueprint-formatted subject or session - names, determine if there are inconsistent value lengths for - the sub or ses key. e.g. ["sub-01", "sub-001"] is an error. + """Determine if there are inconsistent value lengths for the sub or ses key in a list of names. + + For example, ["sub-01", "sub-001"] is an error. + + Parameters + ---------- + path_or_names_list + A path of names of folders to validate, or path to folders to validate. + + prefix + "sub" or "ses" + + Returns + ------- + A list of validation errors. + """ names_list = [ path_or_name if isinstance(path_or_name, str) else path_or_name.name @@ -429,8 +543,21 @@ def datetime_are_iso_format( name: str, path_: Path | None, ) -> List[str]: - """ - Check formatting for date-, time-, or datetime- tags. + """Check formatting for date-, time-, or datetime- tags. + + Parameters + ---------- + name + Name of the folder to validate + + path_ + Path of the folder that is being validated. + + Returns + ------- + error_message + A list of validation errors. + """ formats = { "datetime": "%Y%m%dT%H%M%S", @@ -466,9 +593,21 @@ def datetime_are_iso_format( def raise_display_mode( message: str, display_mode: DisplayMode, log: bool ) -> None: - """ - Show a message by raising an error, displaying warning, or printing. + """Show a message by raising an error, displaying warning, or printing. + Optionally log with the current datashuttle logger. + + Parameters + ---------- + message + Message to display. + + display_mode + Mode to display, "error", "warn" or "print". + + log + If `True`, log the message. + """ if display_mode == "error": utils.log_and_raise_error(message, NeuroBlueprintError) @@ -501,39 +640,43 @@ def validate_project( name_templates: Optional[Dict] = None, strict_mode: bool = False, ) -> List[str]: - """ - Validate all subject and session folders within a project. + """Validate all subject and session folders within a project. Parameters - ----------- - - cfg : Configs + ---------- + cfg datashuttle Configs class. - top_level_folder_list: List[TopLevelFolder] + top_level_folder_list The top level folders to validate. - include_central : bool + include_central If `False`, only project folders in the `local_path` will be validated. Otherwise, project folders in both the `local_path` and `central_path` will be validated. - display_mode : DisplayMode + display_mode Determine whether error or warning is raised. - log : bool + log If `True`, errors or warnings are logged to "datashuttle" logger. - name_templates: Optional[Dict] + name_templates A `name_template` dictionary to validate against. See `set_name_templates()`. - strict_mode: bool + strict_mode If `True`, only allow NeuroBlueprint-formatted folders to exist in the project. By default, non-NeuroBlueprint folders (e.g. a folder called 'my_stuff' in the 'rawdata') are allowed, and only folders starting with sub- or ses- prefix are checked. In `Strict Mode`, any folder not prefixed with sub-, ses- or a valid datatype will raise a validation issue. + + Returns + ------- + error_messages + A list of validation errors. + """ error_messages = [] @@ -541,7 +684,6 @@ def validate_project( error_messages += check_high_level_project_structure(cfg, include_central) for top_level_folder in top_level_folder_list: - if strict_mode: error_messages += check_strict_mode( cfg, top_level_folder, include_central @@ -568,7 +710,6 @@ def validate_project( # Check all names as well as duplicates per-subject for ses_paths in folder_paths["ses"].values(): - error_messages += validate_list_of_names( ses_paths, "ses", @@ -604,10 +745,9 @@ def validate_names_against_project( log: bool = True, name_templates: Optional[Dict] = None, ) -> None: - """ - Given a list of subject and (optionally) session names, - check that these names are formatted consistently with the - rest of the project. Used for creating folders. + """Check that sub / ses names are formatted consistently with the rest of the project. + + This is used for creating folders. Unfortunately this is quite fiddly, as it is important to only validate the passed list of subject / session names while ignoring @@ -615,36 +755,36 @@ def validate_names_against_project( Parameters ---------- - - cfg : Configs + cfg datashuttle Configs class. - top_level_folder : TopLevelFolder + top_level_folder The top level folder to validate - sub_names : List[str] + sub_names A list of subject-level names to validate against the subject names that exist in the project. - ses_names : List[str] + ses_names A list of session-level names to validate against the session names that exist in the project. Note that duplicate checks will only be performed for sessions within the passed `sub_names`. - include_central : bool + include_central If `True`, only project folders in the `local_path` will be validated against. Otherwise, project folders in both the `local_path` and `central_path` will be validated against. - display_mode : DisplayMode + display_mode Determine whether error or warning is raised. - log : bool + log If `True`, errors or warnings are logged to "datashuttle" logger. - name_templates: Optional[Dict] + name_templates A `name_template` dictionary to validate against. See `set_name_templates()`. + """ error_messages = [] @@ -662,7 +802,6 @@ def validate_names_against_project( ) if folder_paths["sub"]: - # Strip any totally invalid names which we can't extract # the sub integer value for the following checks valid_sub_names = strip_uncheckable_names(sub_names, "sub") @@ -690,14 +829,12 @@ def validate_names_against_project( # Now we need to check the sessions. if ses_names is not None and any(ses_names): - # First, validate the list of passed session names error_messages += validate_list_of_names( ses_names, "ses", name_templates=name_templates ) if folder_paths["sub"]: - # Next, we need to check that the passed session names # do not duplicate existing session names and # that do not create inconsistent ses- lengths across the project. @@ -708,7 +845,6 @@ def validate_names_against_project( # are allowed across different subjects (but not within a single sub). for new_sub in sub_names: if new_sub in folder_paths["ses"]: - valid_ses_in_sub = strip_uncheckable_names( folder_paths["ses"][new_sub], "ses", @@ -746,11 +882,24 @@ def validate_names_against_project( def check_high_level_project_structure( cfg: Configs, include_central: bool ) -> List[str]: - """ - Perform basic validation checks on the project structure, - that the project folder name is valid, and that the - project folder contains either a "rawdata" or "derivatives" - folder. + """Perform basic validation checks on the project structure. + + This includes checking that the project folder name is valid, and that the + project folder contains either a "rawdata" or "derivatives" folder. + + Parameters + ---------- + cfg + Datashuttle Configs class. + + include_central + If `True`, the central project is also checked. + + Returns + ------- + error_messages + A list of validation errors. + """ # To avoid circular imports from datashuttle.utils.folders import search_for_folders @@ -810,7 +959,8 @@ def check_high_level_project_structure( def check_strict_mode( cfg: Configs, top_level_folder: TopLevelFolder, include_central: bool ) -> List[str]: - """ + """Perform `strict_mode` validation. + `strict_mode` does not allow any non-NeuroBlueprint folder to exist in the project outside the datatype folder. NeuroBlueprint folders are top-level folder or folder with sub-, ses- or datatype. @@ -820,6 +970,23 @@ def check_strict_mode( prefix typos and other issues (e.g. if subs-extra is a folder, or "rat1", how do we know if it is intended to be a subject folder and validate it, or just an auxiliary folder. + + Parameters + ---------- + cfg + Datashuttle Configs class. + + top_level_folder + "rawdata" or "derivatives". + + include_central + If `True`, the central project is also checked. + + Returns + ------- + error_messages + A list of validation errors. + """ # For circular imports from datashuttle.utils import folders @@ -841,7 +1008,6 @@ def check_strict_mode( ) for sub_level_path in sub_level_folder_paths["local"]: - # Check all folders found in a top-level folder are # sub- prefixed folders. sub_level_name = sub_level_path.name @@ -861,7 +1027,6 @@ def check_strict_mode( ) for ses_level_path in ses_level_folder_paths["local"]: - # For each sub- prefixed folder, check that all folders within # the subject folder are ses- prefixed folders. ses_level_name = ses_level_path.name @@ -884,7 +1049,6 @@ def check_strict_mode( canonical_datatypes = canonical_configs.get_datatypes() for datatype_level_path in search_results: - # For each ses- prefixed folder, check that # only valid datatypes are included within it. datatype_level_name = datatype_level_path.name @@ -916,18 +1080,29 @@ def strip_uncheckable_names( path_or_names_list: List[Path] | List[str], prefix: Prefix, ) -> List[Path] | List[str]: - """ - Convenience function to remove any name in which - the `prefix` value (sub or ses typically) cannot be - converted into an integer. This is necessary as some - validation steps (e.g. checking duplicate names) requires - conversion to `int` and will fail for this reason if these - bad names are not removed. + """Remove any name in which the `prefix` value (sub or ses typically) cannot be converted into an integer. + + This is necessary as some validation steps (e.g. checking duplicate names) + requires conversion to `int` and will fail for this reason if + these bad names are not removed. + + Parameters + ---------- + path_or_names_list + A path of names of folders to validate, or path to folders to validate. + + prefix + "sub" or "ses". + + Returns + ------- + List of path or names which the uncheckable ones (e.g. that are too broken + to be validated against) removed. + """ new_list = [] for path_or_name in path_or_names_list: - path_, name = get_path_and_name(path_or_name) try: @@ -953,9 +1128,22 @@ def strip_uncheckable_names( def check_datatypes_are_valid( datatype: Union[List[str], str], allow_all: bool = False ) -> str | None: - """ - Check a datatype of list of datatypes is a valid - NeuroBlueprint datatype. + """Check a datatype of list of datatypes is a valid NeuroBlueprint datatype. + + Parameters + ---------- + datatype + A str or a list of str where str is a NeuroBlueprint datatype. + + allow_all + At this stage, "all" may still be included in the list of datatypes. + This is valid in some cases as it indicates all datatypes. + + Returns + ------- + message + An error message or None if no errors found. + """ datatype_folders = canonical_folders.get_datatype_folders() diff --git a/pyproject.toml b/pyproject.toml index 528693b54..87af8ef45 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -52,7 +52,6 @@ dev = [ "pytest-mock", "coverage", "tox", - "black", "mypy", "pre-commit", "ruff", @@ -92,22 +91,50 @@ exclude = ["tests*", "docs*"] [tool.pytest.ini_options] addopts = "--cov=datashuttle" -[tool.black] -target-version = ['py39', 'py310', 'py311', 'py312'] -skip-string-normalization = false -line-length = 79 - [tool.ruff] line-length = 79 exclude = ["__init__.py","build",".eggs"] fix = true +# Ruff config is not exactly the same as in the movement repo, as +# currently we are only adding linting enforcement to docstrings. +# Other ruff rules that are also present in movement repo can be +# added here in a separate PR after these changes have been merged. + [tool.ruff.lint] -ignore = ["E203","E501","E731","C901","W291","W293","E402","E722"] -select = ["I", "E", "F", "TCH", "TID252"] +# See https://docs.astral.sh/ruff/rules/ -[tool.ruff.lint.per-file-ignores] -"__init__.py" = ["F401"] +ignore = [ + "E203", # whitespace before ':' (conflicts with Black) + "E501", # line too long (handled by Black) + "E731", # do not assign a lambda expression, use a def + "C901", # function is too complex + "W291", # trailing whitespace + "W293", # blank line contains whitespace + "E402", # module level import not at top of file + "E722", # do not use bare 'except' + "D100", # missing docstring in public module + "D203", # 1 blank line required before class docstring (conflicts with D211) + "D213", # multi-line docstring summary should start at the second line (conflicts with D212) +] +select = [ + "I", # isort + "E", # pycodestyle errors + "F", # Pyflakes + "TC", # flake8-type-checking + "TID252", # flake8-tidy-imports relative-imports + "D", # pydocstyle +] +per-file-ignores = { "tests/*" = [ + "D" # ignore docstring formatting in tests for now +], "examples/*" = [ + "D400", # first line should end with a period. + "D415", # first line should end with a period, question mark... + "D205", # missing blank line between summary and description +]} + +[tool.ruff.format] +docstring-code-format = true # Also format code in docstrings [tool.ruff.lint.mccabe] max-complexity = 18 diff --git a/tests/conftest.py b/tests/conftest.py index 2203b1025..50c27d849 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,5 +1,4 @@ -""" -Test configs, used for setting up SSH tests. +"""Test configs, used for setting up SSH tests. Before running these tests, it is necessary to setup an SSH key. This can be done through datashuttle diff --git a/tests/quick_make_project.py b/tests/quick_make_project.py index 250e66507..cbb837a32 100644 --- a/tests/quick_make_project.py +++ b/tests/quick_make_project.py @@ -1,5 +1,5 @@ -base_path = r"C:/Users/Joe/work/git-repos/forks/yxtuix/joe" - from test_utils import quick_create_project +base_path = r"C:/Users/Joe/work/git-repos/forks/yxtuix/joe" + quick_create_project(base_path) diff --git a/tests/ssh_test_utils.py b/tests/ssh_test_utils.py index 0838669f3..a7af1a65c 100644 --- a/tests/ssh_test_utils.py +++ b/tests/ssh_test_utils.py @@ -7,9 +7,8 @@ def setup_project_for_ssh( project, central_path, central_host_id, central_host_username ): - """ - Set up the project configs to use SSH connection - to central + """Set up the project configs to use SSH connection + to central. """ project.update_config_file( central_path=central_path, @@ -26,11 +25,10 @@ def setup_project_for_ssh( def setup_mock_input(input_): - """ - This is very similar to pytest monkeypatch but + """Very similar to pytest monkeypatch but using that was giving me very strange output, monkeypatch.setattr('builtins.input', lambda _: "n") - i.e. pdb went deep into some unrelated code stack + i.e. pdb went deep into some unrelated code stack. """ orig_builtin = copy.deepcopy(builtins.input) builtins.input = lambda _: input_ # type: ignore @@ -38,16 +36,12 @@ def setup_mock_input(input_): def restore_mock_input(orig_builtin): - """ - orig_builtin: the copied, original builtins.input - """ + """orig_builtin: the copied, original builtins.input.""" builtins.input = orig_builtin def setup_hostkeys(project): - """ - Convenience function to verify the server hostkey. - """ + """Convenience function to verify the server hostkey.""" orig_builtin = setup_mock_input(input_="y") ssh.verify_ssh_central_host( project.cfg["central_host_id"], project.cfg.hostkeys_path, log=True diff --git a/tests/test_utils.py b/tests/test_utils.py index 597fc21e3..c70928fe5 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -27,10 +27,8 @@ def setup_project_default_configs( local_path=False, central_path=False, ): - """ - Set up a fresh project to test on - - local_path / central_path: provide the config paths to set + """Set up a fresh project to test on + local_path / central_path: provide the config paths to set. """ delete_project_if_it_exists(project_name) @@ -70,8 +68,7 @@ def make_project_paths(config_dict): def glob_basenames(search_path, recursive=False, exclude=None): - """ - Use glob to search but strip the full path, including + """Use glob to search but strip the full path, including only the base name (lowest level). """ paths_ = glob.glob(search_path, recursive=recursive) @@ -86,7 +83,6 @@ def glob_basenames(search_path, recursive=False, exclude=None): def teardown_project( project, ): # 99% sure these are unnecessary with pytest tmp_path but keep until SSH testing. - """""" delete_all_folders_in_project_path(project, "central") delete_all_folders_in_project_path(project, "local") delete_project_if_it_exists(project.project_name) @@ -99,7 +95,6 @@ def delete_all_folders_in_local_path(project): def delete_all_folders_in_project_path(project, local_or_central): - """""" folder = f"{local_or_central}_path" if project.cfg is None or ( @@ -116,7 +111,6 @@ def delete_all_folders_in_project_path(project, local_or_central): def delete_project_if_it_exists(project_name): - """""" config_path, _ = canonical_folders.get_project_datashuttle_path( project_name ) @@ -126,8 +120,7 @@ def delete_project_if_it_exists(project_name): def setup_project_fixture(tmp_path, test_project_name, project_type="full"): - """ - Set up a project, either in full mode or local-only mode. This is + """Set up a project, either in full mode or local-only mode. This is very similar to the `BaseTest` fixture but is designed for use in other fixtures that require additional boilerplate e.g. logging. """ @@ -154,7 +147,6 @@ def make_test_path(base_path, local_or_central, test_project_name): def create_all_pathtable_files(pathtable): - """ """ for i in range(pathtable.shape[0]): filepath = pathtable["base_folder"][i] / pathtable["path"][i] filepath.parents[0].mkdir(parents=True, exist_ok=True) @@ -177,8 +169,7 @@ def get_test_config_arguments_dict( set_as_defaults=False, required_arguments_only=False, ): - """ - Retrieve configs, either the required configs + """Retrieve configs, either the required configs (for project.make_config_file()), all configs (default) or non-default configs. Note that default configs here are the expected default arguments in project.make_config_file(). @@ -220,8 +211,7 @@ def get_test_config_arguments_dict( def get_all_broad_folders_used(value=True): - """ - The `folders_used` construct tells the tests which + """The `folders_used` construct tells the tests which folders were used (e.g. created or transferred) and which are not. This means the expected datatypes can be checked. @@ -246,8 +236,7 @@ def get_all_broad_folders_used(value=True): def check_folder_tree_is_correct( base_folder, subs, sessions, folder_used, created_folder_dict=None ): - """ - Automated test that folders are made based + """Automated test that folders are made based on the structure specified on project itself. Cycle through all datatypes (defined in @@ -275,7 +264,7 @@ def check_folder_tree_is_correct( key, folder, ) in canonical_folders.get_datatype_folders().items(): - assert key in folder_used.keys(), ( + assert key in folder_used, ( "Key not found in folder_used. " "Update folder used and hard-coded tests: " "test_custom_folder_names(), test_explicitly_session_list()" @@ -309,8 +298,7 @@ def check_folder_tree_is_correct( def check_and_cd_folder(path_): - """ - Check a folder exists and CD to it if it does. + """Check a folder exists and CD to it if it does. Use the pytest -s flag to print all tested paths """ @@ -324,8 +312,7 @@ def check_datatype_sub_ses_uploaded_correctly( subs_to_upload=None, ses_to_upload=None, ): - """ - Iterate through the project (datatype > ses > sub) and + """Iterate through the project (datatype > ses > sub) and check that the folders at each level match those that are expected (passed in datatype / sub / ses to upload). Folders are searched with wildcard glob. @@ -361,8 +348,7 @@ def check_datatype_sub_ses_uploaded_correctly( def make_and_check_local_project_folders( project, top_level_folder, subs, sessions, datatype, datatypes_used=None ): - """ - Make a local project folder tree with the specified datatype, + """Make a local project folder tree with the specified datatype, subs, sessions and check it is made successfully. Since empty folders are not transferred, it is necessary @@ -400,7 +386,6 @@ def make_local_folders_with_files_in( def check_configs(project, kwargs, config_path=None): - """""" if config_path is None: config_path = project._config_path @@ -415,8 +400,7 @@ def check_project_configs( project, *kwargs, ): - """ - Core function for checking the config against + """Core function for checking the config against provided configs (kwargs). Open the config.yaml file and check the config values stored there, and in project.cfg, against the provided configs. @@ -436,8 +420,7 @@ def check_project_configs( def check_config_file(config_path, *kwargs): - """""" - with open(config_path, "r") as config_file: + with open(config_path) as config_file: config_yaml = yaml.full_load(config_file) for name, value in kwargs[0].items(): @@ -452,11 +435,9 @@ def check_config_file(config_path, *kwargs): def get_top_level_folder_path( project, local_or_central="local", folder_name="rawdata" ): - """""" - - assert ( - folder_name in canonical_folders.get_top_level_folders() - ), "folder_name must be canonical e.g. rawdata" + assert folder_name in canonical_folders.get_top_level_folders(), ( + "folder_name must be canonical e.g. rawdata" + ) if local_or_central == "local": base_path = project.cfg["local_path"] @@ -473,8 +454,7 @@ def handle_upload_or_download( top_level_folder=None, swap_last_folder_only=False, ): - """ - To keep things consistent and avoid the pain of writing + """To keep things consistent and avoid the pain of writing files over SSH, to test download just swap the central and local server (so things are still transferred from local machine to central, but using the download function). @@ -500,7 +480,6 @@ def handle_upload_or_download( def get_transfer_func( project, upload_or_download, transfer_method, top_level_folder=None ): - """""" if transfer_method == "top_level_folder": assert top_level_folder is not None, "must pass top-level-folder" assert top_level_folder in [None, "rawdata", "derivatives"] @@ -530,8 +509,7 @@ def get_transfer_func( def swap_local_and_central_paths(project, swap_last_folder_only=False): - """ - When testing upload vs. download, the most convenient way + """When testing upload vs. download, the most convenient way to test download is to swap the paths. In this case, we 'download' from local to central. It much simplifies creating the folders to transfer (which are created locally), and is fully required @@ -577,25 +555,21 @@ def swap_local_and_central_paths(project, swap_last_folder_only=False): def get_default_sub_sessions_to_test(): - """ - Canonical subs / sessions for these tests - """ + """Canonical subs / sessions for these tests.""" subs = ["sub-001", "sub-002", "sub-003"] sessions = ["ses-001_datetime-20220516T135022", "ses-002", "ses-003"] return subs, sessions def move_some_keys_to_end_of_dict(config): - """ - Need to move connection method to the end + """Need to move connection method to the end so ssh opts are already set before it is changed. """ config["connection_method"] = config.pop("connection_method") def clear_capsys(capsys): - """ - read from capsys clears it, so new + """Read from capsys clears it, so new print statements are clearer to read. """ capsys.readouterr() @@ -612,14 +586,13 @@ def write_file(path_, contents="", append=False): def read_file(path_): - with open(path_, "r") as file: + with open(path_) as file: contents = file.readlines() return contents def set_datashuttle_loggers(disable): - """ - Turn off or on datashuttle logs, if these are + """Turn off or on datashuttle logs, if these are on when testing with pytest they will be propagated to pytest's output, making it difficult to read. @@ -636,8 +609,7 @@ def set_datashuttle_loggers(disable): def check_working_top_level_folder_only_exists( folder_name, base_path_to_check, subs, sessions, folders_used=None ): - """ - Check that the folder tree made in the 'folder_name' + """Check that the folder tree made in the 'folder_name' (e.g. 'rawdata') top level folder is correct. Additionally, check that no other top-level folders exist. This is to ensure that folders made / transferred from one top-level folder @@ -665,11 +637,11 @@ def read_log_file(logging_path): log_filepath = list(glob.glob(str(logging_path / "*.log"))) assert len(log_filepath) == 1, ( - f"there should only be one log " f"in log output path {logging_path}" + f"there should only be one log in log output path {logging_path}" ) log_filepath = log_filepath[0] - with open(log_filepath, "r") as file: + with open(log_filepath) as file: log = file.read() return log @@ -677,7 +649,7 @@ def read_log_file(logging_path): def delete_log_files(logging_path): ds_logger.close_log_filehandler() - for log in glob.glob((str(logging_path / "*.log"))): + for log in glob.glob(str(logging_path / "*.log")): os.remove(log) diff --git a/tests/tests_integration/base.py b/tests/tests_integration/base.py index c87d31152..5ce5359c1 100644 --- a/tests/tests_integration/base.py +++ b/tests/tests_integration/base.py @@ -7,11 +7,9 @@ class BaseTest: - @pytest.fixture(scope="function") def no_cfg_project(test): - """ - Fixture that creates an empty project. Ignore the warning + """Fixture that creates an empty project. Ignore the warning that no configs are setup yet. """ test_utils.delete_project_if_it_exists(TEST_PROJECT_NAME) @@ -24,8 +22,7 @@ def no_cfg_project(test): @pytest.fixture(scope="function") def project(self, tmp_path, request): - """ - Set up a project with default configs to use for testing. + """Set up a project with default configs to use for testing. This fixture uses indirect parameterization to test both 'full' and 'local-only' (no `central_path` or `connection_method`). The @@ -60,8 +57,7 @@ def project(self, tmp_path, request): @pytest.fixture(scope="function") def clean_project_name(self): - """ - Create an empty project, but ensure no + """Create an empty project, but ensure no configs already exists, and delete created configs after test. """ diff --git a/tests/tests_integration/test_configs.py b/tests/tests_integration/test_configs.py index a18682636..5156f84b7 100644 --- a/tests/tests_integration/test_configs.py +++ b/tests/tests_integration/test_configs.py @@ -15,25 +15,20 @@ class TestConfigs(BaseTest): @pytest.fixture(scope="function") def non_existent_path(self, tmp_path): - """ - Return a path that does not exist. - """ + """Return a path that does not exist.""" non_existent_path = tmp_path / "does_not_exist" assert not non_existent_path.is_dir() return non_existent_path @pytest.fixture(scope="function") def existent_path(self, tmp_path): - """ - Return a path that exists. - """ + """Return a path that exists.""" existent_path = tmp_path / "exists" os.makedirs(existent_path, exist_ok=True) return existent_path def test_warning_on_startup(self, no_cfg_project): - """ - When no configs have been set, a warning should be shown that + """When no configs have been set, a warning should be shown that the config has not been initialized. Need to download Rclone first to ensure input() is not called. """ @@ -60,10 +55,9 @@ def test_warning_on_startup(self, no_cfg_project): ) @pytest.mark.parametrize("path_type", ["local_path", "central_path"]) def test_bad_path_syntax(self, project, bad_pattern, path_type, tmp_path): - """ - "~", "." and "../" syntax is not supported because + """`~`, `.` and `../` syntax is not supported because it does not work with rclone. Theoretically it - could be supported by checking for "." etc. and + could be supported by checking for `.` etc. and filling in manually, but it does not seem robust. Here check an error is raised when path contains @@ -97,9 +91,8 @@ def test_bad_path_syntax(self, project, bad_pattern, path_type, tmp_path): assert "must contain the full folder path with no " in str(e.value) def test_no_ssh_options_set_on_make_config_file(self, no_cfg_project): - """ - Check that program will assert if not all ssh options - are set on make_config_file + """Check that program will assert if not all ssh options + are set on make_config_file. """ with pytest.raises(ConfigError) as e: no_cfg_project.make_config_file( @@ -118,8 +111,7 @@ def test_no_ssh_options_set_on_make_config_file(self, no_cfg_project): # ------------------------------------------------------------- def test_required_configs(self, no_cfg_project, tmp_path): - """ - Set the required arguments of the config (local_path, central_path, + """Set the required arguments of the config (local_path, central_path, connection_method and check they are set correctly in both the no_cfg_project.cfg dict and config.yaml file. """ @@ -135,8 +127,7 @@ def test_required_configs(self, no_cfg_project, tmp_path): ) def test_config_defaults(self, no_cfg_project, tmp_path): - """ - Check the default configs are set as expected + """Check the default configs are set as expected (see get_test_config_arguments_dict()) for tested defaults. """ required_options = test_utils.get_test_config_arguments_dict( @@ -152,8 +143,7 @@ def test_config_defaults(self, no_cfg_project, tmp_path): test_utils.check_configs(no_cfg_project, default_options) def test_non_default_configs(self, no_cfg_project, tmp_path): - """ - Set the configs to non-default options, make the + """Set the configs to non-default options, make the config file and check file and no_cfg_project.cfg are set correctly. """ changed_configs = test_utils.get_test_config_arguments_dict( @@ -169,8 +159,7 @@ def test_non_default_configs(self, no_cfg_project, tmp_path): # ------------------------------------------------------------- def test_update_config_file__(self, no_cfg_project, tmp_path): - """ - Set the configs as default, and then update them to + """Set the configs as default, and then update them to new configs and check they are updated properly. Then, update only a subset (back to the defaults) and @@ -215,8 +204,7 @@ def test_update_config_file__(self, no_cfg_project, tmp_path): test_utils.check_configs(project, default_configs) def test_existing_projects(self, monkeypatch, tmp_path): - """ - Test existing projects are correctly found based on whether + """Test existing projects are correctly found based on whether they exist in the home directory and contain a config.yaml. By default, datashuttle saves project folders to @@ -225,9 +213,9 @@ def test_existing_projects(self, monkeypatch, tmp_path): function is monkeypatched in order to point to a tmp_path. The tmp_path / "projects" is filled with a mix of project folders - with and without config, and tested against accordingly. The `local_path` - and `central_path` specified in the DataShuttle config are arbitrarily put in - `tmp_path`. + with and without config, and tested against accordingly. The + `local_path` and `central_path` specified in the DataShuttle config are + arbitrarily put in `tmp_path`. """ def patch_get_datashuttle_path(): @@ -263,13 +251,12 @@ def patch_get_datashuttle_path(): (tmp_path / "projects" / "project_3"), ] - # -------------------------------------------------------------------------------------------------------------------- + # ------------------------------------------------------------------------- # Utils - # -------------------------------------------------------------------------------------------------------------------- + # ------------------------------------------------------------------------- def check_config_reopen_and_check_config_again(self, project, *kwargs): - """ - Check the config file and project.cfg against provided kwargs, + """Check the config file and project.cfg against provided kwargs, delete the project and set up the project again, checking everything is loaded correctly. """ diff --git a/tests/tests_integration/test_create_folders.py b/tests/tests_integration/test_create_folders.py index e50623776..cb1de20a3 100644 --- a/tests/tests_integration/test_create_folders.py +++ b/tests/tests_integration/test_create_folders.py @@ -13,11 +13,9 @@ class TestCreateFolders(BaseTest): - @pytest.mark.parametrize("project", ["local", "full"], indirect=True) def test_generate_folders_default_ses(self, project): - """ - Make a subject folders with full tree. Don't specify + """Make a subject folders with full tree. Don't specify session name (it will default to no sessions). Check that the folder tree is created correctly. Pass @@ -37,8 +35,7 @@ def test_generate_folders_default_ses(self, project): @pytest.mark.parametrize("project", ["local", "full"], indirect=True) def test_explicitly_session_list(self, project): - """ - Perform an alternative test where the output is tested explicitly. + """Perform an alternative test where the output is tested explicitly. This is some redundancy to ensure tests are working correctly and make explicit the expected folder tree. @@ -83,8 +80,7 @@ def test_explicitly_session_list(self, project): def test_every_broad_datatype_passed( self, project, behav, ephys, funcimg, anat ): - """ - Check every combination of data type used and ensure only the + """Check every combination of data type used and ensure only the correct ones are made. NOTE: This test could be refactored to reduce code reuse. @@ -121,8 +117,7 @@ def test_every_broad_datatype_passed( @pytest.mark.parametrize("project", ["local", "full"], indirect=True) def test_custom_folder_names(self, project, monkeypatch): - """ - Change folder names to custom (non-default) and + """Change folder names to custom (non-default) and ensure they are made correctly. """ new_name_datafolders = canonical_folders.get_datatype_folders() @@ -178,8 +173,7 @@ def new_name_func(): ) @pytest.mark.parametrize("project", ["local", "full"], indirect=True) def test_datatypes_subsection(self, project, files_to_test): - """ - Check that combinations of datatypes passed to make file folder + """Check that combinations of datatypes passed to make file folder make the correct combination of datatypes. Note this will fail when new top level folders are added, and should be @@ -207,9 +201,8 @@ def test_datatypes_subsection(self, project, files_to_test): @pytest.mark.parametrize("project", ["local", "full"], indirect=True) def test_date_flags_in_session(self, project): - """ - Check that @DATE@ is converted into current date - in generated folder names + """Check that @DATE@ is converted into current date + in generated folder names. """ date, time_ = self.get_formatted_date_and_time() @@ -230,9 +223,8 @@ def test_date_flags_in_session(self, project): @pytest.mark.parametrize("project", ["local", "full"], indirect=True) def test_datetime_flag_in_session(self, project): - """ - Check that @DATETIME@ is converted to datetime - in generated folder names + """Check that @DATETIME@ is converted to datetime + in generated folder names. """ date, time_ = self.get_formatted_date_and_time() @@ -257,8 +249,7 @@ def test_datetime_flag_in_session(self, project): @pytest.mark.parametrize("project", ["local", "full"], indirect=True) def test_created_paths_dict_sub_or_ses_only(self, project): - """ - Test that the `created_folders` dictionary returned by + """Test that the `created_folders` dictionary returned by `create_folders` correctly splits paths when only subject or session is passed. The `datatype` case is tested in `test_utils.check_folder_tree_is_correct()`. @@ -292,8 +283,7 @@ def test_created_paths_dict_sub_or_ses_only(self, project): ) @pytest.mark.parametrize("project", ["local", "full"], indirect=True) def test_all_top_level_folders(self, project, top_level_folder): - """ - Check that when switching the top level folder (e.g. rawdata, derivatives) + """Check that when switching the top level folder (e.g. rawdata, derivatives) new folders are made in the correct folder. """ subs = ["sub-001", "sub-002"] @@ -319,8 +309,7 @@ def test_all_top_level_folders(self, project, top_level_folder): @pytest.mark.parametrize("top_level_folder", ["rawdata", "derivatives"]) @pytest.mark.parametrize("return_with_prefix", [True, False]) def test_get_next_sub(self, project, return_with_prefix, top_level_folder): - """ - Test that the next subject number is suggested correctly. + """Test that the next subject number is suggested correctly. This takes the union of subjects available in the local and central repository. As such test the case where either are empty, or when they have different subjects in. @@ -373,8 +362,7 @@ def test_get_next_sub(self, project, return_with_prefix, top_level_folder): @pytest.mark.parametrize("top_level_folder", ["rawdata", "derivatives"]) @pytest.mark.parametrize("return_with_prefix", [True, False]) def test_get_next_ses(self, project, return_with_prefix, top_level_folder): - """ - Almost identical to test_get_next_sub() but with calls + """Almost identical to test_get_next_sub() but with calls for searching sessions. This could be combined with above but reduces readability, so leave with some duplication. @@ -443,8 +431,7 @@ def test_get_next_ses(self, project, return_with_prefix, top_level_folder): @pytest.mark.parametrize("project", ["local", "full"], indirect=True) def test_get_next_sub_and_ses_name_template(self, project): - """ - In the case where a name template exists, these getters should use the + """In the case where a name template exists, these getters should use the number of digits on the template (even if these are different within the project!). """ diff --git a/tests/tests_integration/test_datatypes.py b/tests/tests_integration/test_datatypes.py index 840fc26ee..688da3e36 100644 --- a/tests/tests_integration/test_datatypes.py +++ b/tests/tests_integration/test_datatypes.py @@ -8,15 +8,13 @@ class TestDatatypes(BaseTest): - """ - Tests for creating folders and transfer (very similar to other tests) + """Tests for creating folders and transfer (very similar to other tests) however which test the creation and transfer of narrow datatypes. Other tests used broad datatypes. """ def test_create_narrow_datatypes(self, project): - """ - Create all narrow datatype folders and check + """Create all narrow datatype folders and check they are created as expected. """ # Make folder tree including all narrow datatypes @@ -41,8 +39,7 @@ def test_create_narrow_datatypes(self, project): ) def get_narrow_only_datatypes_used(self, used=True): - """ - This is similar to test_utils.get_all_broad_folders_used + """Similar to test_utils.get_all_broad_folders_used but for narrow datatypes. """ return { @@ -55,8 +52,7 @@ def test_transfer_datatypes( project, upload_or_download, ): - """ - Create a project with narrow datatypes and check these + """Create a project with narrow datatypes and check these folders are transferred as expected. """ subs, sessions = test_utils.get_default_sub_sessions_to_test() diff --git a/tests/tests_integration/test_filesystem_transfer.py b/tests/tests_integration/test_filesystem_transfer.py index 95e785189..7dbfab4d5 100644 --- a/tests/tests_integration/test_filesystem_transfer.py +++ b/tests/tests_integration/test_filesystem_transfer.py @@ -13,7 +13,6 @@ class TestFileTransfer(BaseTest): - @pytest.mark.parametrize( "top_level_folder", canonical_folders.get_top_level_folders() ) @@ -28,8 +27,7 @@ def test_transfer_empty_folder_structure( upload_or_download, transfer_method, ): - """ - First make a project (folders only) locally. + """First make a project (folders only) locally. Next upload this to the central path and check all folders are uploaded correctly. """ @@ -78,8 +76,7 @@ def test_transfer_across_top_level_folders( upload_or_download, transfer_method, ): - """ - For each possible top level folder (e.g. rawdata, derivatives) + """For each possible top level folder (e.g. rawdata, derivatives) (parametrized) create a folder tree in every top-level folder, then transfer using upload / download and upload_rawdata() / download_rawdata() that only the working top-level folder @@ -130,7 +127,6 @@ def test_transfer_across_top_level_folders( @pytest.mark.parametrize("upload_or_download", ["upload", "download"]) def test_transfer_all_top_level_folders(self, project, upload_or_download): - """ """ subs, sessions = test_utils.get_default_sub_sessions_to_test() for top_level_folder in canonical_folders.get_top_level_folders(): @@ -151,7 +147,6 @@ def test_transfer_all_top_level_folders(self, project, upload_or_download): transfer_function() for top_level_folder in canonical_folders.get_top_level_folders(): - test_utils.check_folder_tree_is_correct( os.path.join(base_path_to_check, top_level_folder), subs, @@ -177,8 +172,7 @@ def test_transfer_all_top_level_folders(self, project, upload_or_download): def test_transfer_empty_folder_specific_data( self, project, upload_or_download, datatype_to_transfer ): - """ - For the combination of datatype folders, make a folder + """For the combination of datatype folders, make a folder tree with all datatype folders then upload select ones, checking only the selected ones are uploaded. """ @@ -215,7 +209,7 @@ def test_transfer_empty_folder_specific_data( ["behav", "ephys", "funcimg", "anat"], ], ) - @pytest.mark.parametrize("upload_or_download", ["upload" "download"]) + @pytest.mark.parametrize("upload_or_download", ["upload", "download"]) def test_transfer_empty_folder_specific_subs( self, project, @@ -223,8 +217,7 @@ def test_transfer_empty_folder_specific_subs( datatype_to_transfer, sub_idx_to_upload, ): - """ - Create a project folder tree with a set of subs, then + """Create a project folder tree with a set of subs, then take a subset of these subs and upload them. Check only the selected subs were uploaded. """ @@ -268,8 +261,7 @@ def test_transfer_empty_folder_specific_ses( sub_idx_to_upload, ses_idx_to_upload, ): - """ - Make a project with set subs and sessions. Then select a subset of the + """Make a project with set subs and sessions. Then select a subset of the sessions to upload. Check only the selected sessions were uploaded. """ subs, sessions = test_utils.get_default_sub_sessions_to_test() @@ -303,8 +295,7 @@ def test_transfer_empty_folder_specific_ses( def test_transfer_with_keyword_parameters( self, project, upload_or_download ): - """ - Test the @TO@ keyword is accepted properly when making a session and + """Test the @TO@ keyword is accepted properly when making a session and transferring it. First pass @TO@-formatted sub and sessions to create_folders. Then transfer the files (upload or download). @@ -350,8 +341,7 @@ def test_transfer_with_keyword_parameters( @pytest.mark.parametrize("upload_or_download", ["upload", "download"]) def test_wildcard_transfer(self, project, upload_or_download): - """ - Transfer a subset of define subject and session + """Transfer a subset of define subject and session and check only the expected folders are there. """ subs = ["sub-389", "sub-989", "sub-445"] @@ -395,8 +385,7 @@ def test_wildcard_transfer(self, project, upload_or_download): ] def test_deep_folder_structure(self, project): - """ - Just a quick test as all other tests only test files directly in the + """Just a quick test as all other tests only test files directly in the datatyp directly. Check that rlcone is setup to transfer multiple levels down from the datatype level. """ @@ -427,8 +416,7 @@ def test_rclone_options( dry_run, capsys, ): - """ - When verbosity is --vv, rclone itself will output + """When verbosity is --vv, rclone itself will output a list of all called arguments. Use this to check rclone is called with the arguments set in configs as expected. verbosity itself is tested in another method. @@ -475,8 +463,7 @@ def test_overwrite_same_size_earlier_to_later( top_level_folder, upload_or_download, ): - """ - Main test to check every parameterization for overwrite settings. + """Main test to check every parameterization for overwrite settings. It is such an important setting it is tested for all top level folder, transfer method, even though it makes for quite a confusing function. @@ -541,8 +528,7 @@ def test_overwrite_same_size_later_to_earlier( top_level_folder, upload_or_download, ): - """ - This functions is extremely similar to + """Extremely similar to `test_overwrite_same_size_later_to_earlier()` but it is much easier to understand individually when they are split. @@ -588,8 +574,7 @@ def test_overwrite_same_size_later_to_earlier( def test_overwrite_different_size_different_times( self, project, overwrite_existing_files ): - """ - Quick additional test to confirm that "if_source_newer" will still + """Quick additional test to confirm that "if_source_newer" will still not transfer even if the older file is larger. This is the expected behaviour from rclone, this is confidence check on understanding. """ @@ -639,7 +624,6 @@ def get_paths_to_a_local_and_central_file(self, project, top_level_folder): def setup_overwrite_file_tests( self, upload_or_download, top_level_folder, project ): - """""" local_file_path, central_file_path = ( self.get_paths_to_a_local_and_central_file( project, top_level_folder @@ -670,8 +654,7 @@ def setup_overwrite_file_tests( def test_dry_run( self, project, top_level_folder, transfer_method, upload_or_download ): - """ - Just do a quick functional test on dry-run that indeed nothing + """Just do a quick functional test on dry-run that indeed nothing is transferred across all top-level-folder / upload-download methods. """ @@ -704,8 +687,7 @@ def test_specific_file_or_folder( transfer_file, upload_or_download, ): - """ - Test upload_specific_folder_or_file() and download_specific_folder_or_file(). + """Test upload_specific_folder_or_file() and download_specific_folder_or_file(). Make a project with two different files (just to ensure non-target files are not transferred). Transfer @@ -749,7 +731,6 @@ def test_specific_file_or_folder( assert transferred_files == to_test_against def setup_specific_file_or_folder_files(self, project, top_level_folder): - """ """ project.create_folders( top_level_folder, ["sub-001", "sub-002"], diff --git a/tests/tests_integration/test_formatting.py b/tests/tests_integration/test_formatting.py index bf62c6a50..0b5db8338 100644 --- a/tests/tests_integration/test_formatting.py +++ b/tests/tests_integration/test_formatting.py @@ -11,8 +11,7 @@ class TestFormatting(BaseTest): "input", [1, {"test": "one"}, 1.0, ["1", "2", ["three"]]] ) def test_format_names_bad_input(self, input, prefix): - """ - Test that names passed in incorrect type + """Test that names passed in incorrect type (not str, list) raise appropriate error. """ with pytest.raises(TypeError) as e: @@ -22,8 +21,7 @@ def test_format_names_bad_input(self, input, prefix): @pytest.mark.parametrize("prefix", ["sub", "ses"]) def test_format_names_duplicate_ele(self, prefix): - """ - Test that appropriate error is raised when duplicate name + """Test that appropriate error is raised when duplicate name is passed to format_names(). """ with pytest.raises(NeuroBlueprintError) as e: @@ -37,8 +35,7 @@ def test_format_names_duplicate_ele(self, prefix): ) def test_format_names_prefix(self): - """ - Check that format_names correctly prefixes input + """Check that format_names correctly prefixes input with default sub or ses prefix. This is less useful now that ses/sub name dash and underscore order is more strictly checked. diff --git a/tests/tests_integration/test_local_only_mode.py b/tests/tests_integration/test_local_only_mode.py index 3de7bdc47..b2853cbad 100644 --- a/tests/tests_integration/test_local_only_mode.py +++ b/tests/tests_integration/test_local_only_mode.py @@ -12,11 +12,9 @@ class TestLocalOnlyProject(BaseTest): - def test_bad_setup(self, tmp_path): - """ - Test setup without providing both central_path and connection - method (distinguishing a full vs local-only project) + """Test setup without providing both central_path and connection + method (distinguishing a full vs local-only project). """ local_path = tmp_path / "test_local" @@ -40,8 +38,7 @@ def test_bad_setup(self, tmp_path): @pytest.mark.parametrize("project", ["local"], indirect=True) def test_full_to_local_project(self, project): - """ - Make a full project a local-only project, and check the transfer + """Make a full project a local-only project, and check the transfer functionality is now restricted. """ project.update_config_file(central_path=None, connection_method=None) @@ -59,8 +56,7 @@ def test_full_to_local_project(self, project): @pytest.mark.parametrize("project", ["local"], indirect=True) def test_local_project_to_full(self, tmp_path, project): - """ - Test updating a local-only project to a full one + """Test updating a local-only project to a full one by adding the required configs (both must be set together) Perform a quick check that data transfer does not error out now that the project is full, and the configs are set as expected. @@ -87,8 +83,7 @@ def test_local_project_to_full(self, tmp_path, project): @pytest.mark.parametrize("project", ["local"], indirect=True) def test_local_to_full_project(self, project): - """ - Change a project from local-only to a normal project by updating + """Change a project from local-only to a normal project by updating the relevant configs. Smoke test that general functionality is maintained and that transfers work correctly. """ @@ -125,8 +120,7 @@ def test_local_to_full_project(self, project): @pytest.mark.parametrize("top_level_folder", ["rawdata", "derivatives"]) @pytest.mark.parametrize("project", ["full"], indirect=True) def test_get_next_sub_and_ses(self, project, top_level_folder): - """ - Make a 'full' project with subject and session > 1 in both local + """Make a 'full' project with subject and session > 1 in both local and central projects. Then, delete the local and run get next sub / ses, and make the project local-only. Call validation requesting to also check central path, which should be ignored as we are in local-only mode. diff --git a/tests/tests_integration/test_logging.py b/tests/tests_integration/test_logging.py index 0ef940400..9a3ec8855 100644 --- a/tests/tests_integration/test_logging.py +++ b/tests/tests_integration/test_logging.py @@ -17,12 +17,9 @@ class TestLogging: - @pytest.fixture(scope="function") def teardown_logger(self): - """ - Ensure the logger is deleted at the end of each test. - """ + """Ensure the logger is deleted at the end of each test.""" yield if "datashuttle" in logging.root.manager.loggerDict: logging.root.manager.loggerDict.pop("datashuttle") @@ -32,14 +29,11 @@ def teardown_logger(self): # ------------------------------------------------------------------------- def test_logger_name(self): - """ - Check the canonical logger name. - """ + """Check the canonical logger name.""" assert ds_logger.get_logger_name() == "datashuttle" def test_start_logging(self, tmp_path, teardown_logger): - """ - Test that the central `start` logging function + """Test that the central `start` logging function starts the named logger with the expected handlers. """ assert ds_logger.logging_is_active() is False @@ -56,9 +50,7 @@ def test_start_logging(self, tmp_path, teardown_logger): assert isinstance(logger.handlers[0], logging.FileHandler) def test_shutdown_logger(self, tmp_path, teardown_logger): - """ - Check the log handler remover indeed removes the handles. - """ + """Check the log handler remover indeed removes the handles.""" assert ds_logger.logging_is_active() is False ds_logger.start(tmp_path, "test-command", variables=[]) @@ -71,9 +63,7 @@ def test_shutdown_logger(self, tmp_path, teardown_logger): assert ds_logger.logging_is_active() is False def test_logging_an_error(self, project, teardown_logger): - """ - Check that errors are caught and logged properly. - """ + """Check that errors are caught and logged properly.""" with pytest.raises(NeuroBlueprintError): project.create_folders("rawdata", "sob-001") @@ -88,8 +78,7 @@ def test_logging_an_error(self, project, teardown_logger): @pytest.fixture(scope="function") def clean_project_name(self): - """ - Create an empty project, but ensure no + """Create an empty project, but ensure no configs already exists, and delete created configs after test. @@ -106,8 +95,7 @@ def clean_project_name(self): @pytest.fixture(scope="function") def project(self, tmp_path, clean_project_name, request): - """ - Set `up a project with default configs to use + """Set `up a project with default configs to use for testing. This fixture is distinct from the base.py fixture as requires additional logging setup / teardown. @@ -136,23 +124,21 @@ def project(self, tmp_path, clean_project_name, request): @pytest.mark.parametrize("project", ["local", "full"], indirect=True) def test_log_filename(self, project): - """ - Check the log filename is formatted correctly, for - `update_config_file`, an arbitrary command + """Check the log filename is formatted correctly, for + `update_config_file`, an arbitrary command. """ project.update_config_file(central_host_id="test_id") log_search = list(project.cfg.logging_path.glob("*.log")) - assert ( - len(log_search) == 1 - ), "should only be 1 log in this test environment." + assert len(log_search) == 1, ( + "should only be 1 log in this test environment." + ) log_filename = log_search[0].name regex = re.compile(r"\d{8}T\d{6}_update-config-file.log") assert re.search(regex, log_filename) is not None def test_logs_make_config_file(self, clean_project_name, tmp_path): - """""" project = test_utils.make_project(clean_project_name) project.make_config_file( @@ -253,8 +239,7 @@ def test_create_folders(self, project): def test_logs_upload_and_download( self, project, upload_or_download, transfer_method ): - """ - Set transfer verbosity and progress settings so + """Set transfer verbosity and progress settings so maximum output is produced to test against. """ subs = ["sub-11"] @@ -310,8 +295,7 @@ def test_logs_upload_and_download( def test_logs_upload_and_download_folder_or_file( self, project, upload_or_download ): - """ - Set transfer verbosity and progress settings so + """Set transfer verbosity and progress settings so maximum output is produced to test against. """ test_utils.make_and_check_local_project_folders( @@ -352,8 +336,7 @@ def test_logs_upload_and_download_folder_or_file( def test_temp_log_folder_moved_make_config_file( self, clean_project_name, tmp_path ): - """ - Check that + """Check that logs are moved to the passed `local_path` when `make_config_file()` is passed. """ @@ -378,8 +361,7 @@ def test_temp_log_folder_moved_make_config_file( assert "make-config-file" in project_path_logs[0] def test_clear_logging_path(self, clean_project_name, tmp_path): - """ - The temporary logging path holds logs which are all + """The temporary logging path holds logs which are all transferred to a new `local_path` when configs are updated. This should only ever be the most recent log action, and not others which may @@ -418,7 +400,6 @@ def test_clear_logging_path(self, clean_project_name, tmp_path): # ---------------------------------------------------------------------------------- def test_logs_check_update_config_error(self, project): - """""" with pytest.raises(ConfigError): project.update_config_file( connection_method="ssh", central_host_username=None @@ -437,7 +418,6 @@ def test_logs_check_update_config_error(self, project): @pytest.mark.parametrize("project", ["local", "full"], indirect=True) def test_logs_bad_create_folders_error(self, project): - """""" project.create_folders("rawdata", "sub-001", datatype="all") test_utils.delete_log_files(project.cfg.logging_path) @@ -454,8 +434,7 @@ def test_logs_bad_create_folders_error(self, project): @pytest.mark.parametrize("project", ["local", "full"], indirect=True) def test_validate_project_logging(self, project): - """ - Test that `validate_project` logs errors + """Test that `validate_project` logs errors and warnings to file. """ # Make conflicting subject folders @@ -488,8 +467,7 @@ def test_validate_project_logging(self, project): @pytest.mark.parametrize("project", ["local", "full"], indirect=True) def test_validate_names_against_project_logging(self, project): - """ - Implicitly test `validate_names_against_project` called when + """Implicitly test `validate_names_against_project` called when `make_project_folders` is called, that it logs errors to file. Warnings are not tested. """ diff --git a/tests/tests_integration/test_settings.py b/tests/tests_integration/test_settings.py index b75f60bbc..bcb9c73ec 100644 --- a/tests/tests_integration/test_settings.py +++ b/tests/tests_integration/test_settings.py @@ -11,11 +11,9 @@ class TestPersistentSettings(BaseTest): - @pytest.mark.parametrize("project", ["local", "full"], indirect=True) def test_persistent_settings_name_templates(self, project): - """ - Test the 'name_templates' option that is stored in persistent + """Test the 'name_templates' option that is stored in persistent settings and adds a regexp to validate subject and session names against. @@ -120,13 +118,11 @@ def test_persistent_settings_name_templates(self, project): @pytest.mark.parametrize("project", ["local", "full"], indirect=True) def test_persistent_settings_tui(self, project): - """ - Test persistent settings for the project that + """Test persistent settings for the project that determine display of the TUI. First check defaults are correct, change every one and save, then check they are correct on re-load. """ - # test all defaults settings = project._load_persistent_settings() tui_settings = settings["tui"] @@ -145,8 +141,7 @@ def test_persistent_settings_tui(self, project): @pytest.mark.parametrize("project", ["local", "full"], indirect=True) def test_bypass_validation(self, project): - """ - Check bypass validation which will allow folder + """Check bypass validation which will allow folder creation even when validation fails. Check it is off by default, turn on, check bad name can be created. Reload, turn off, check for error on attempting to create @@ -161,13 +156,12 @@ def test_bypass_validation(self, project): project.create_folders("rawdata", "sub-@@@") assert ( - "BAD_VALUE: The value for prefix sub in name sub-@@@ is not an integer." - == str(e.value) + str(e.value) + == "BAD_VALUE: The value for prefix sub in name sub-@@@ is not an integer." ) def get_settings_default(self): - """ - Hard-coded default settings that should mirror `canonical_configs` + """Hard-coded default settings that should mirror `canonical_configs` and should be changed whenever the canonical configs are changed. This is to protect against accidentally changing these configs. """ @@ -210,9 +204,7 @@ def get_settings_default(self): return default_settings def get_settings_changed(self): - """ - The default settings with every possible setting changed. - """ + """The default settings with every possible setting changed.""" changed_settings = { "create_checkboxes_on": {}, "transfer_checkboxes_on": { diff --git a/tests/tests_integration/test_ssh_file_transfer.py b/tests/tests_integration/test_ssh_file_transfer.py index 393de076c..4e98894c6 100644 --- a/tests/tests_integration/test_ssh_file_transfer.py +++ b/tests/tests_integration/test_ssh_file_transfer.py @@ -1,5 +1,3 @@ -""" """ - import copy import glob import shutil @@ -29,8 +27,7 @@ class TestFileTransfer: ], ) def pathtable_and_project(self, request, tmpdir_factory): - """ - Create a project for SSH testing. Setup + """Create a project for SSH testing. Setup the project as normal, and switch configs to use SSH connection. @@ -49,7 +46,7 @@ def pathtable_and_project(self, request, tmpdir_factory): items have been transferred. This is achieved by using "class" scope. - NOTES + Notes ----- - Pytest params - The `params` key sets the `params` attribute on the pytest `request` fixture. @@ -68,6 +65,7 @@ def pathtable_and_project(self, request, tmpdir_factory): a few seconds after SSH transfer. This makes the tests run very slowly. We can get rid of this limitation on linux. + """ testing_ssh = request.param tmp_path = tmpdir_factory.mktemp("test") @@ -165,11 +163,10 @@ def test_all_data_transfer_options( datatype, upload_or_download, ): - """ - Parse the arguments to filter the pathtable, getting + """Parse the arguments to filter the pathtable, getting the files expected to be transferred passed on the arguments Note files in sub/ses/datatype folders must be handled - separately to those in non-sub, non-ses, non-datatype folders + separately to those in non-sub, non-ses, non-datatype folders. see test_utils.swap_local_and_central_paths() for the logic on setting up and swapping local / central paths for @@ -245,9 +242,8 @@ def test_all_data_transfer_options( # --------------------------------------------------------------------------------------------------------------- def query_table(self, pathtable, arguments): - """ - Search the table for arguments, return empty - if arguments empty + """Search the table for arguments, return empty + if arguments empty. """ if any(arguments): folders = pathtable.query(" | ".join(arguments)) @@ -256,8 +252,7 @@ def query_table(self, pathtable, arguments): return folders def parse_arguments(self, pathtable, list_of_names, field): - """ - Replicate datashuttle name formatting by parsing + """Replicate datashuttle name formatting by parsing "all" arguments and turning them into a list of all names, (subject or session), taken from the pathtable. """ @@ -276,8 +271,7 @@ def parse_arguments(self, pathtable, list_of_names, field): return list_of_names def make_pathtable_search_filter(self, sub_names, ses_names, datatype): - """ - Create a string of arguments to pass to pd.query() that will + """Create a string of arguments to pass to pd.query() that will create the table of only transferred sub, ses and datatype. Two arguments must be created, one of all sub / ses / datatypes diff --git a/tests/tests_integration/test_ssh_setup.py b/tests/tests_integration/test_ssh_setup.py index 752985a67..31c1fa72e 100644 --- a/tests/tests_integration/test_ssh_setup.py +++ b/tests/tests_integration/test_ssh_setup.py @@ -1,7 +1,6 @@ -""" -SSH configs are set in conftest.py . The password +"""SSH configs are set in conftest.py . The password should be stored in a file called test_ssh_password.txt located -in the same folder as test_ssh.py +in the same folder as test_ssh.py. """ import pytest @@ -16,9 +15,8 @@ class TestSSH: @pytest.fixture(scope="function") def project(test, tmp_path): - """ - Make a project as per usual, but now add - in test ssh configurations + """Make a project as per usual, but now add + in test ssh configurations. """ tmp_path = tmp_path / "test with space" @@ -43,10 +41,9 @@ def project(test, tmp_path): def test_verify_ssh_central_host_do_not_accept( self, capsys, project, input_ ): - """ - Use the main function to test this. Test the sub-function + """Use the main function to test this. Test the sub-function when accepting, because this main function will also - call setup ssh key pairs which we don't want to do yet + call setup ssh key pairs which we don't want to do yet. This should only accept for "y" so try some random strings including "n" and check they all do not make the connection. @@ -62,8 +59,7 @@ def test_verify_ssh_central_host_do_not_accept( assert "Host not accepted. No connection made.\n" in captured.out def test_verify_ssh_central_host_accept(self, capsys, project): - """ - User is asked to accept the server hostkey. Mock this here + """User is asked to accept the server hostkey. Mock this here and check hostkey is successfully accepted and written to configs. """ test_utils.clear_capsys(capsys) @@ -79,20 +75,19 @@ def test_verify_ssh_central_host_accept(self, capsys, project): captured = capsys.readouterr() assert captured.out == "Host accepted.\n" - with open(project.cfg.hostkeys_path, "r") as file: + with open(project.cfg.hostkeys_path) as file: hostkey = file.readlines()[0] assert f"{project.cfg['central_host_id']} ssh-ed25519 " in hostkey def test_generate_and_write_ssh_key(self, project): - """ - Check ssh key for passwordless connection is written - to file + """Check ssh key for passwordless connection is written + to file. """ path_to_save = project.cfg["local_path"] / "test" ssh.generate_and_write_ssh_key(path_to_save) - with open(path_to_save, "r") as file: + with open(path_to_save) as file: first_line = file.readlines()[0] assert first_line == "-----BEGIN RSA PRIVATE KEY-----\n" diff --git a/tests/tests_integration/test_transfer_checks.py b/tests/tests_integration/test_transfer_checks.py index 342ed6407..7a6f631f3 100644 --- a/tests/tests_integration/test_transfer_checks.py +++ b/tests/tests_integration/test_transfer_checks.py @@ -15,8 +15,7 @@ class TestTransferChecks(BaseTest): [["rawdata", "derivatives"], ["rawdata"], ["derivatives"]], ) def test_rclone_check(self, project, top_level_folders): - """ - Test rclone.get_local_and_central_file_differences(). This function + """Test rclone.get_local_and_central_file_differences(). This function returns a dictionary where values are list of paths and keys separate based on differences between local and central projects. diff --git a/tests/tests_integration/test_validation.py b/tests/tests_integration/test_validation.py index 70cf140ae..c8eaa0d3a 100644 --- a/tests/tests_integration/test_validation.py +++ b/tests/tests_integration/test_validation.py @@ -15,7 +15,6 @@ class TestValidation(BaseTest): - @pytest.mark.parametrize( "sub_name", ["sub-001", "sub-999_@DATE@", "sub-001_random-tag_another-tag"], @@ -35,8 +34,7 @@ class TestValidation(BaseTest): def test_warn_on_inconsistent_sub_value_lengths( self, project, sub_name, bad_sub_name ): - """ - This test checks that inconsistent sub value lengths are properly + """Checks that inconsistent sub value lengths are properly detected across the project. This is performed with an assortment of possible filenames and leading zero conflicts. @@ -95,8 +93,7 @@ def test_warn_on_inconsistent_sub_value_lengths( def test_warn_on_inconsistent_ses_value_lengths( self, project, ses_name, bad_ses_name ): - """ - This function is exactly the same as + """Exactly the same as `test_warn_on_inconsistent_sub_value_lengths()` but operates at the session level. This is extreme code duplication, but factoring the main logic out got very messy and hard to follow. @@ -140,8 +137,7 @@ def test_warn_on_inconsistent_ses_value_lengths( @pytest.mark.parametrize("project", ["local", "full"], indirect=True) def test_warn_on_inconsistent_sub_and_ses_value_lengths(self, project): - """ - Test that warning is shown for both subject and session when + """Test that warning is shown for both subject and session when inconsistent zeros are found in both. """ os.makedirs( @@ -160,7 +156,6 @@ def test_warn_on_inconsistent_sub_and_ses_value_lengths(self, project): def check_inconsistent_sub_or_ses_value_length_warning( self, project, warn_idx=0, include_central=True ): - """""" with pytest.warns(UserWarning) as w: project.validate_project( "rawdata", display_mode="warn", include_central=include_central @@ -174,8 +169,7 @@ def check_inconsistent_sub_or_ses_value_length_warning( @pytest.mark.parametrize("project", ["local", "full"], indirect=True) def test_duplicate_ses_or_sub_key_value_pair(self, project): - """ - Test the check that if a duplicate key is attempt to be made + """Test the check that if a duplicate key is attempt to be made when making a folder e.g. sub-001 exists, then make sub-001_id-123. After this check, make a folder that can be made (e.g. sub-003) just to make sure it does not raise error. @@ -208,8 +202,7 @@ def test_duplicate_ses_or_sub_key_value_pair(self, project): @pytest.mark.parametrize("project", ["local", "full"], indirect=True) def test_duplicate_sub_and_ses_num_leading_zeros(self, project): - """ - Very similar to test_duplicate_ses_or_sub_key_value_pair(), + """Very similar to test_duplicate_ses_or_sub_key_value_pair(), but explicitly check that error is raised if the same number is used with different number of leading zeros. """ @@ -229,8 +222,7 @@ def test_duplicate_sub_and_ses_num_leading_zeros(self, project): @pytest.mark.parametrize("project", ["local", "full"], indirect=True) def test_duplicate_sub_when_creating_session(self, project): - """ - Check the unique case that a duplicate subject is + """Check the unique case that a duplicate subject is introduced when the session is made. """ project.create_folders("rawdata", "sub-001") @@ -274,8 +266,7 @@ def test_duplicate_sub_when_creating_session(self, project): assert "DUPLICATE_NAME" in str(e.value) def test_duplicate_ses_across_subjects(self, project): - """ - Quick test that duplicate session folders only raise + """Quick test that duplicate session folders only raise an error when they are in the same subject. """ project.create_folders("rawdata", "sub-001", "ses-001") @@ -294,8 +285,7 @@ def test_duplicate_ses_across_subjects(self, project): @pytest.mark.parametrize("project", ["local", "full"], indirect=True) def test_invalid_sub_and_ses_name(self, project): - """ - This is a slightly weird case, the name is successfully + """Slightly weird case, the name is successfully prefixed as 'sub-sub_100` but when the value if `sub-` is extracted, it is also "sub" and so an error is raised. """ @@ -303,16 +293,16 @@ def test_invalid_sub_and_ses_name(self, project): project.create_folders("rawdata", "sub_100") assert ( - "BAD_VALUE: The value for prefix sub in name sub-sub_100 is not an integer." - == str(e.value) + str(e.value) + == "BAD_VALUE: The value for prefix sub in name sub-sub_100 is not an integer." ) with pytest.raises(NeuroBlueprintError) as e: project.create_folders("rawdata", "sub-001", "ses_100") assert ( - "BAD_VALUE: The value for prefix ses in name ses-ses_100 is not an integer." - == str(e.value) + str(e.value) + == "BAD_VALUE: The value for prefix ses in name ses-ses_100 is not an integer." ) # ------------------------------------------------------------------------- @@ -320,8 +310,7 @@ def test_invalid_sub_and_ses_name(self, project): # ------------------------------------------------------------------------- def test_validate_project(self, project): - """ - Test the `validate_project` function over all it's arguments. + """Test the `validate_project` function over all it's arguments. Note not every validation case is tested exhaustively, these are tested in `test_validation_unit.py` elsewhere here. """ @@ -391,7 +380,6 @@ def test_validate_project(self, project): @pytest.mark.parametrize("prefix", ["sub", "ses"]) def test_validate_project_returned_list(self, project, prefix): - """ """ bad_names = [ f"{prefix}-001", f"{prefix}-001_@DATE@", @@ -423,7 +411,6 @@ def test_validate_project_returned_list(self, project, prefix): assert "VALUE_LENGTH" in concat_error def test_output_paths_are_valid(self, project): - """ """ sub_name = "sub-001x" ses_name = "ses-001x" project.create_folders( @@ -458,8 +445,7 @@ def test_output_paths_are_valid(self, project): def test_validate_names_against_project_with_bad_existing_names( self, project ): - """ - When using `validate_names_against_project()` there are + """When using `validate_names_against_project()` there are three possible classes of error: 1) error in the passed names. 2) an error already exists in the project. @@ -589,8 +575,7 @@ def test_validate_names_against_project_with_bad_existing_names( @pytest.mark.parametrize("project", ["local", "full"], indirect=True) def test_validate_names_against_project_interactions(self, project): - """ - Check that interactions between the list of names and existing + """Check that interactions between the list of names and existing project are caught. This includes duplicate subject / session names as well as inconsistent subject / session value lengths. """ @@ -712,8 +697,7 @@ def test_validate_names_against_project_interactions(self, project): @pytest.mark.parametrize("project", ["local", "full"], indirect=True) def test_tags_in_name_templates_pass_validation(self, project): - """ - It is useful to allow tags in the `name_templates` as it means + """It is useful to allow tags in the `name_templates` as it means auto-completion in the TUI can use tags for automatic name generation. Because all subject and session names are fully formatted (e.g. @DATE@ converted to actual dates) @@ -766,7 +750,6 @@ def test_tags_in_name_templates_pass_validation(self, project): assert "TEMPLATE: The name: ses-001_datex-20241212" in str(e.value) def test_name_templates_validate_project(self, project): - # set up name templates name_templates = { "on": True, @@ -804,7 +787,6 @@ def test_name_templates_validate_project(self, project): # ---------------------------------------------------------------------------------- def test_quick_validation(self, mocker, project): - """ """ project.create_folders("rawdata", "sub-1") os.makedirs(project.cfg["local_path"] / "rawdata" / "sub-02") project.create_folders("derivatives", "sub-1") @@ -842,8 +824,7 @@ def test_quick_validation(self, mocker, project): assert kwargs["name_templates"] == {"on": False} def test_quick_validation_top_level_folder(self, project): - """ - Test that errors are raised as expected on + """Test that errors are raised as expected on bad project path input. """ with pytest.raises(FileNotFoundError) as e: @@ -862,7 +843,6 @@ def test_quick_validation_top_level_folder(self, project): @pytest.mark.parametrize("top_level_folder", ["rawdata", "derivatives"]) def test_strict_mode_validation(self, project, top_level_folder): - """ """ project.create_folders( top_level_folder, ["sub-001", "sub-002"], @@ -931,10 +911,7 @@ def test_strict_mode_validation(self, project, top_level_folder): def test_check_high_level_project_structure( self, project, top_level_folder ): - """ - Check that local and central project names are properly formatted - and that - """ + """Check that local and central project names are properly formatted.""" with pytest.warns(UserWarning) as w: project.validate_project( top_level_folder, "warn", include_central=True diff --git a/tests/tests_regression/test_backwards_compatibility.py b/tests/tests_regression/test_backwards_compatibility.py index 79cefe5a1..7d73ff1d0 100644 --- a/tests/tests_regression/test_backwards_compatibility.py +++ b/tests/tests_regression/test_backwards_compatibility.py @@ -9,11 +9,9 @@ class TestBackwardsCompatibility: - @pytest.fixture(scope="function") def project(self): - """ - Delete the project configs if they exist, + """Delete the project configs if they exist, and tear down after the test has run. """ test_utils.delete_project_if_it_exists(TEST_PROJECT_NAME) @@ -25,8 +23,7 @@ def project(self): test_utils.delete_project_if_it_exists(TEST_PROJECT_NAME) def test_v0_6_0(self, project, tmp_path): - """ - v0.6.0 is the first version with narrow datatypes, and the checkboxes was refactored to + """v0.6.0 is the first version with narrow datatypes, and the checkboxes was refactored to be a {"on": bool, "displayed": bool} dict rather than a bool indicating whether the checkbox is on. However, this version is missing narrow datatypes added later (e.g. "motion"). In the test file, all 'displayed' are turned off except f2pe. @@ -53,8 +50,7 @@ def test_v0_6_0(self, project, tmp_path): assert transfer_checkboxes[key]["displayed"] is (key == "f2pe") def test_v0_5_3(self, project, tmp_path): - """ - This version did not have narrow datatypes, and the persistent checkbox setting was only a + """This version did not have narrow datatypes, and the persistent checkbox setting was only a bool. Therefore, the "displayed" uses the canonical defaults (because they don't exist in the file yet). """ reloaded_ver_configs, reloaded_ver_persistent_settings = ( @@ -82,8 +78,7 @@ def test_v0_5_3(self, project, tmp_path): def load_and_check_old_version_yamls( self, project, tmp_path, datashuttle_version ): - """ - Load an old config file in the current datashuttle version, + """Load an old config file in the current datashuttle version, and check that the new-version ('canonical') configs and persistent settings match the structure of the files loaded from the old datashuttle version. @@ -130,8 +125,7 @@ def load_and_check_old_version_yamls( return reloaded_ver_configs, reloaded_ver_persistent_settings def recursive_test_dictionary(self, dict_canonical, dict_to_test): - """ - A dictionary to check all keys in a nested dictionary + """A dictionary to check all keys in a nested dictionary match and all value types are the same. """ keys_canonical = list(dict_canonical.keys()) @@ -144,7 +138,6 @@ def recursive_test_dictionary(self, dict_canonical, dict_to_test): ) for key in dict_canonical.keys(): - if isinstance(dict_canonical[key], dict): self.recursive_test_dictionary( dict_canonical[key], dict_to_test[key] diff --git a/tests/tests_tui/test_local_only_project.py b/tests/tests_tui/test_local_only_project.py index 0599ea144..8a06ccfa7 100644 --- a/tests/tests_tui/test_local_only_project.py +++ b/tests/tests_tui/test_local_only_project.py @@ -5,14 +5,12 @@ class TestTuiLocalOnlyProject(TuiBase): - @pytest.mark.asyncio async def test_local_only_make_project( self, empty_project_paths, ): - """ - Test a local-only project, where the only set config is `local_path`. + """Test a local-only project, where the only set config is `local_path`. Set up a local project, and check the 'Transfer' tab is disabled and set configs are propagated. """ @@ -21,7 +19,6 @@ async def test_local_only_make_project( app = TuiApp() async with app.run_test(size=self.tui_size()) as pilot: - await self.setup_and_check_local_only_project( pilot, project_name, local_path ) @@ -49,8 +46,7 @@ async def test_local_project_to_full( self, empty_project_paths, ): - """ - It is possible to switch between a 'local-only' project (`local_path` + """It is possible to switch between a 'local-only' project (`local_path` only set) and a full project with all configs set, where transfer is allowed. Here start as a local project then set configs so we become a full project. """ @@ -60,7 +56,6 @@ async def test_local_project_to_full( app = TuiApp() async with app.run_test(size=self.tui_size()) as pilot: - # Set up a local-only project await self.setup_and_check_local_only_project( pilot, project_name, local_path @@ -125,8 +120,7 @@ async def test_full_project_to_local( self, setup_project_paths, ): - """ - Very similar to `test_check_local_only_project_to_full()`, but + """Very similar to `test_check_local_only_project_to_full()`, but going from a full project to a local only. This still requires a refresh so the full transfer tab can be set to a placeholder. """ @@ -135,7 +129,6 @@ async def test_full_project_to_local( app = TuiApp() async with app.run_test(size=self.tui_size()) as pilot: - # Fixture generated a full project, switch to it's # project manager screen here. await self.check_and_click_onto_existing_project( @@ -187,8 +180,7 @@ async def test_full_project_to_local( async def setup_and_check_local_only_project( self, pilot, project_name, local_path ): - """ - Set up a local-only project by filling in the `local_path` and setting + """Set up a local-only project by filling in the `local_path` and setting the radio button to the no-connection option. """ # Move to configs window diff --git a/tests/tests_tui/test_tui_configs.py b/tests/tests_tui/test_tui_configs.py index e8f5d0767..c2e110b65 100644 --- a/tests/tests_tui/test_tui_configs.py +++ b/tests/tests_tui/test_tui_configs.py @@ -15,7 +15,6 @@ class TestTuiConfigs(TuiBase): - # ------------------------------------------------------------------------- # Test New Project Configs # ------------------------------------------------------------------------- @@ -27,8 +26,7 @@ async def test_make_new_project_configs( empty_project_paths, kwargs_set, ): - """ - Check the ConfigsContent when making a new project. This contains + """Check the ConfigsContent when making a new project. This contains many widgets shared with the ConfigsContent on the tab page, however also includes an additional information banner and input for the project name. @@ -68,7 +66,6 @@ async def test_make_new_project_configs( app = TuiApp() async with app.run_test(size=self.tui_size()) as pilot: - # Select a new project, check NewProjectScreen is # displayed correctly. await self.scroll_to_click_pause( @@ -159,8 +156,7 @@ async def test_make_new_project_configs( async def test_update_config_on_project_manager_screen( self, setup_project_paths ): - """ - Test the ConfigsContent on the project manager tab screen. + """Test the ConfigsContent on the project manager tab screen. The project is set up in the fixture, navigate to the project page. Check that the default configs are displayed. Change all the configs, save, and check these are updated on the config file and on the @@ -173,7 +169,6 @@ async def test_update_config_on_project_manager_screen( app = TuiApp() async with app.run_test(size=self.tui_size()) as pilot: - # Navigate to the existing project and click onto the # configs tab. await self.check_and_click_onto_existing_project( @@ -215,7 +210,7 @@ async def test_update_config_on_project_manager_screen( "central_host_username": "random_username", } - for key in new_kwargs.keys(): + for key in new_kwargs: # The purpose is to update to completely new configs assert new_kwargs[key] != project_cfg[key] @@ -268,8 +263,7 @@ async def test_update_config_on_project_manager_screen( @pytest.mark.asyncio async def test_configs_select_path(self, monkeypatch): - """ - Test the 'Select' buttons / DirectoryTree on the ConfigsContent. + """Test the 'Select' buttons / DirectoryTree on the ConfigsContent. These are used to select folders that are filled into the Input. Open the select dialog, select a folder, check the path is filled into the Input. There is one for both local @@ -282,7 +276,6 @@ async def test_configs_select_path(self, monkeypatch): app = TuiApp() async with app.run_test(size=self.tui_size()) as pilot: - # Select the page and ConfigsContent for setting up new project await self.scroll_to_click_pause( pilot, "#mainwindow_new_project_button" @@ -345,10 +338,8 @@ async def test_configs_select_path(self, monkeypatch): @pytest.mark.asyncio async def test_bad_configs_screen_input(self, empty_project_paths): - app = TuiApp() async with app.run_test(size=self.tui_size()) as pilot: - # Select a new project, check NewProjectScreen is displayed correctly. await self.scroll_to_click_pause( pilot, "#mainwindow_new_project_button" @@ -376,11 +367,9 @@ async def test_bad_configs_screen_input(self, empty_project_paths): async def check_configs_widgets_match_configs( self, configs_content, kwargs ): - """ - Check that the widgets of the TUI configs match those found + """Check that the widgets of the TUI configs match those found in `kwargs`. """ - # Local Path ---------------------------------------------------------- assert ( @@ -403,7 +392,6 @@ async def check_configs_widgets_match_configs( ) if kwargs["connection_method"] == "ssh": - # Central Host ID ------------------------------------------------- assert ( @@ -430,11 +418,9 @@ async def check_configs_widgets_match_configs( ) async def set_configs_content_widgets(self, pilot, kwargs): - """ - Given a dict of options that can be set on the configs TUI + """Given a dict of options that can be set on the configs TUI in kwargs, set all configs widgets according to kwargs. """ - # Local Path ---------------------------------------------------------- await self.fill_input( @@ -444,7 +430,6 @@ async def set_configs_content_widgets(self, pilot, kwargs): # Connection Method --------------------------------------------------- if kwargs["connection_method"] == "ssh": - await self.scroll_to_click_pause(pilot, "#configs_ssh_radiobutton") # Central Host ID ------------------------------------------------- @@ -472,8 +457,7 @@ async def set_configs_content_widgets(self, pilot, kwargs): async def check_new_project_configs( self, pilot, project_name, configs_content, kwargs ): - """ - Check the configs displayed on the TUI match those found in `kwargs`. + """Check the configs displayed on the TUI match those found in `kwargs`. Also, check the widgets unique to ConfigsContent on the configs selection for a new project. """ diff --git a/tests/tests_tui/test_tui_create_folders.py b/tests/tests_tui/test_tui_create_folders.py index 0f1779f11..b7d3822ee 100644 --- a/tests/tests_tui/test_tui_create_folders.py +++ b/tests/tests_tui/test_tui_create_folders.py @@ -13,7 +13,6 @@ class TestTuiCreateFolders(TuiBase): - # ------------------------------------------------------------------------- # General test Create Folders # ------------------------------------------------------------------------- @@ -23,9 +22,7 @@ class TestTuiCreateFolders(TuiBase): async def test_create_folders_sub_and_ses( self, setup_project_paths, test_multi_input ): - """ - Basic test that folders are created as expected through the TUI. - """ + """Basic test that folders are created as expected through the TUI.""" # Define folders to create if test_multi_input: sub_text = "sub-001, sub-002" @@ -42,7 +39,6 @@ async def test_create_folders_sub_and_ses( app = TuiApp() async with app.run_test(size=self.tui_size()) as pilot: - # Set up the TUI on the 'create' tab, filling the # input with the subject and session folders to create. await self.check_and_click_onto_existing_project( @@ -87,8 +83,7 @@ async def test_create_folders_sub_and_ses( @pytest.mark.asyncio async def test_create_folders_formatted_names(self, setup_project_paths): - """ - Test preview tooltips and create folders with _@DATE@ formatting. + """Test preview tooltips and create folders with _@DATE@ formatting. The @TO@ key is not tested. """ tmp_config_path, tmp_path, project_name = setup_project_paths.values() @@ -181,15 +176,13 @@ async def test_create_folders_formatted_names(self, setup_project_paths): async def test_create_folders_bad_validation_tooltips( self, setup_project_paths ): - """ - Check that correct tooltips are displayed when + """Check that correct tooltips are displayed when various invalid subject or session names are provided. """ tmp_config_path, tmp_path, project_name = setup_project_paths.values() app = TuiApp() async with app.run_test(size=self.tui_size()) as pilot: - await self.check_and_click_onto_existing_project( pilot, project_name ) @@ -227,11 +220,12 @@ async def test_create_folders_bad_validation_tooltips( ).tooltip == "Formatted names: ['sub-001']" ) + assert ( pilot.app.screen.query_one( "#create_folders_session_input" ).tooltip - == "DUPLICATE_PREFIX: The name: ses-001_ses-001 of contains more than one instance of the prefix ses." + == "DUPLICATE_PREFIX: The name: ses-001_ses-001 contains more than one instance of the prefix ses." ) await self.fill_input( @@ -259,8 +253,7 @@ async def test_create_folders_bad_validation_tooltips( async def test_validation_error_and_bypass_validation( self, setup_project_paths ): - """ - Test validation and bypass validation options by + """Test validation and bypass validation options by first trying to create an invalid folder name, and checking an error displays. Next, turn on 'bypass validation' and check the folders are created despite being invalid. @@ -269,7 +262,6 @@ async def test_validation_error_and_bypass_validation( app = TuiApp() async with app.run_test(size=self.tui_size()) as pilot: - await self.check_and_click_onto_existing_project( pilot, project_name ) @@ -338,15 +330,13 @@ async def test_validation_error_and_bypass_validation( async def test_name_template_next_sub_or_ses_and_validation( self, setup_project_paths ): - """ - Test validation and double-click for next sub / ses + """Test validation and double-click for next sub / ses values when 'name templates' is set in the 'Settings' window. """ tmp_config_path, tmp_path, project_name = setup_project_paths.values() app = TuiApp() async with app.run_test(size=self.tui_size()) as pilot: - await self.check_and_click_onto_existing_project( pilot, project_name ) @@ -474,15 +464,13 @@ async def test_name_template_next_sub_or_ses_and_validation( @pytest.mark.asyncio async def test_get_next_sub_and_ses_no_template(self, setup_project_paths): - """ - Test the double click on Input correctly fills with the + """Test the double click on Input correctly fills with the next sub or ses (or prefix only when CTRL is pressed). """ tmp_config_path, tmp_path, project_name = setup_project_paths.values() app = TuiApp() async with app.run_test(size=self.tui_size()) as pilot: - await self.setup_existing_project_create_tab_filled_sub_and_ses( pilot, project_name, create_folders=True ) @@ -540,8 +528,7 @@ async def test_get_next_sub_and_ses_no_template(self, setup_project_paths): async def test_get_next_sub_and_ses_central_no_template( self, setup_project_paths, mocker ): - """ - Test getting the next subject / session with the include_central option. Check the + """Test getting the next subject / session with the include_central option. Check the checkbox widget that turns the setting on. Trigger a get next subject / session and mock the underlying datashuttle function to ensure include_central is properly called. """ @@ -601,8 +588,7 @@ async def test_get_next_sub_and_ses_central_no_template( @pytest.mark.asyncio async def test_get_next_sub_and_ses_error_popup(self, setup_project_paths): - """ - Test the modal error dialog display on encountering an error + """Test the modal error dialog display on encountering an error while suggesting next sub/ses. Since getting the suggestion happens in a thread, the `dismiss_popup_and_show_modal_error_dialog_from_thread` function which is used to display the modal error dialog from main thread @@ -640,15 +626,13 @@ async def test_get_next_sub_and_ses_error_popup(self, setup_project_paths): async def test_create_folders_settings_top_level_folder( self, setup_project_paths ): - """ - Check the folders are created in the correct top level + """Check the folders are created in the correct top level folder when this is changed in the 'Settings' screen. """ tmp_config_path, tmp_path, project_name = setup_project_paths.values() app = TuiApp() async with app.run_test(size=self.tui_size()) as pilot: - # Open the CreateFoldersSettingsScreen await self.setup_existing_project_create_tab_filled_sub_and_ses( pilot, project_name, create_folders=False @@ -727,7 +711,6 @@ async def iterate_and_check_all_datatype_folders( folder_used = test_utils.get_all_broad_folders_used(value=False) for datatype in canonical_configs.get_broad_datatypes(): - await self.scroll_to_click_pause( pilot, f"#create_{datatype}_checkbox", diff --git a/tests/tests_tui/test_tui_datatypes.py b/tests/tests_tui/test_tui_datatypes.py index 0e9ec8a64..76a438f35 100644 --- a/tests/tests_tui/test_tui_datatypes.py +++ b/tests/tests_tui/test_tui_datatypes.py @@ -7,9 +7,7 @@ class TestDatatypesTUI(TuiBase): - """ - Test the datatype selection screen for the Create and Transfer tab. - """ + """Test the datatype selection screen for the Create and Transfer tab.""" @pytest.mark.asyncio async def test_select_displayed_datatypes_create( @@ -19,7 +17,6 @@ async def test_select_displayed_datatypes_create( app = TuiApp() async with app.run_test(size=self.tui_size()) as pilot: - # Set up the TUI on the 'create' tab, filling the # input with the subject and session folders to create. await self.check_and_click_onto_existing_project( @@ -140,7 +137,6 @@ async def test_select_displayed_datatypes_transfer( app = TuiApp() async with app.run_test(size=self.tui_size()) as pilot: - # Set up the TUI on the 'transfer' tab (custom) and # open the datatype selection screen await self.check_and_click_onto_existing_project( diff --git a/tests/tests_tui/test_tui_directorytree.py b/tests/tests_tui/test_tui_directorytree.py index 1a6e03d8b..2bcd33ec8 100644 --- a/tests/tests_tui/test_tui_directorytree.py +++ b/tests/tests_tui/test_tui_directorytree.py @@ -14,22 +14,19 @@ class TestTuiCreateDirectoryTree(TuiBase): - """ - Test the `Create` tab directory tree. - `Transfer` + """Test the `Create` tab directory tree. + `Transfer`. """ @pytest.mark.asyncio async def test_fill_and_append_next_sub_and_ses(self, setup_project_paths): - """ - Test the CTRL+F and CTRL+A functions on the directorytree + """Test the CTRL+F and CTRL+A functions on the directorytree that fill and append subject / session name to the inputs. """ tmp_config_path, tmp_path, project_name = setup_project_paths.values() app = TuiApp() async with app.run_test(size=self.tui_size()) as pilot: - # Open the create tab and first fill the subject # and session inputs with -001. await self.check_and_click_onto_existing_project( @@ -116,8 +113,7 @@ async def test_fill_and_append_next_sub_and_ses(self, setup_project_paths): async def test_create_folders_directorytree_clipboard( self, setup_project_paths ): - """ - Check that pressing CTRL+Q on the directorytree copies the + """Check that pressing CTRL+Q on the directorytree copies the hovered folder to the clipboard (using pyperclip). """ tmp_config_path, tmp_path, project_name = setup_project_paths.values() @@ -159,7 +155,6 @@ async def test_failed_pyperclip_copy( app = TuiApp() async with app.run_test(size=self.tui_size()) as pilot: - # Set up a project and navigate to the directory tree await self.setup_existing_project_create_tab_filled_sub_and_ses( pilot, project_name, create_folders=True @@ -197,8 +192,7 @@ def mock_copy(_): async def test_create_folders_directorytree_open_filesystem( self, setup_project_paths, monkeypatch ): - """ - Test pressing CTRL+O on the filetree triggers the opening + """Test pressing CTRL+O on the filetree triggers the opening of a folder through the show-in-file-manager package (monkeypatched function). """ @@ -206,7 +200,6 @@ async def test_create_folders_directorytree_open_filesystem( app = TuiApp() async with app.run_test(size=self.tui_size()) as pilot: - # Set up the 'create tab' with loaded nodes await self.setup_existing_project_create_tab_filled_sub_and_ses( pilot, project_name, create_folders=True @@ -256,7 +249,6 @@ async def test_create_folders_directorytree_rename( app = TuiApp() async with app.run_test(size=self.tui_size()) as pilot: - # Set up the 'create tab' with loaded nodes await self.setup_existing_project_create_tab_filled_sub_and_ses( pilot, project_name, create_folders=True diff --git a/tests/tests_tui/test_tui_get_help.py b/tests/tests_tui/test_tui_get_help.py index 55d876f35..aa8986b8c 100644 --- a/tests/tests_tui/test_tui_get_help.py +++ b/tests/tests_tui/test_tui_get_help.py @@ -5,17 +5,14 @@ class TestTuiSettings(TuiBase): - """ - Test that the 'Get Help' page from the main menu. + """Test that the 'Get Help' page from the main menu. Open it, check the expected label is displayed, close it. """ @pytest.mark.asyncio async def test_get_help(self, empty_project_paths): - app = TuiApp() async with app.run_test(size=self.tui_size()) as pilot: - await self.scroll_to_click_pause( pilot, "#mainwindow_get_help_button" ) diff --git a/tests/tests_tui/test_tui_logging.py b/tests/tests_tui/test_tui_logging.py index 543c23073..33aa07978 100644 --- a/tests/tests_tui/test_tui_logging.py +++ b/tests/tests_tui/test_tui_logging.py @@ -7,11 +7,9 @@ class TestTuiLogging(TuiBase): - @pytest.mark.asyncio async def test_logging(self, setup_project_paths): - """ - Test logging by running some commands, checking they + """Test logging by running some commands, checking they are displayed on the logging tree, that the most recent log is correct and that the log screen opens when clicked. """ @@ -19,7 +17,6 @@ async def test_logging(self, setup_project_paths): app = TuiApp() async with app.run_test(size=self.tui_size()) as pilot: - # Update configs and create folders to make some logs project = test_utils.make_project(project_name) @@ -63,7 +60,9 @@ async def test_logging(self, setup_project_paths): ) assert ( "create-folders" in widg.get_node_at_line(2).data.path.stem - ), f"ERROR MESSAGE: {widg.get_node_at_line(0).data.path}-{widg.get_node_at_line(1).data.path}-{widg.get_node_at_line(2).data.path}" + ), ( + f"ERROR MESSAGE: {widg.get_node_at_line(0).data.path}-{widg.get_node_at_line(1).data.path}-{widg.get_node_at_line(2).data.path}" + ) # Check the latest logging path is correct assert ( diff --git a/tests/tests_tui/test_tui_selectdirectorytree.py b/tests/tests_tui/test_tui_selectdirectorytree.py index 3d1a9d362..1885ddc34 100644 --- a/tests/tests_tui/test_tui_selectdirectorytree.py +++ b/tests/tests_tui/test_tui_selectdirectorytree.py @@ -10,11 +10,9 @@ class TestSelectTree(TuiBase): @pytest.mark.asyncio async def test_select_directory_tree(self, monkeypatch): - """ - Test that changing the drive in SelectDirectoryTreeScreen + """Test that changing the drive in SelectDirectoryTreeScreen updates the DirectoryTree path as expected. """ - # Set the Select drives to be these test cases monkeypatch.setattr( SelectDirectoryTreeScreen, @@ -30,7 +28,6 @@ async def test_select_directory_tree(self, monkeypatch): app = TuiApp() async with app.run_test() as pilot: - # Open the select directory tree screen await self.scroll_to_click_pause( pilot, "#mainwindow_new_project_button" diff --git a/tests/tests_tui/test_tui_settings.py b/tests/tests_tui/test_tui_settings.py index 77a65b090..8b9bcaa4d 100644 --- a/tests/tests_tui/test_tui_settings.py +++ b/tests/tests_tui/test_tui_settings.py @@ -5,20 +5,16 @@ class TestTuiSettings(TuiBase): - """ - Test the 'Settings' screen accessible from the Main Menu. - """ + """Test the 'Settings' screen accessible from the Main Menu.""" @pytest.mark.asyncio async def test_light_dark_mode(self): - """ - Check the light / dark mode switch which is stored + """Check the light / dark mode switch which is stored in the global tui settings. Global refers to set across all projects not related to a specific project. """ app = TuiApp() async with app.run_test(size=self.tui_size()) as pilot: - await self.scroll_to_click_pause( pilot, "#mainwindow_settings_button" ) @@ -45,8 +41,7 @@ async def test_light_dark_mode(self): @pytest.mark.asyncio async def test_show_transfer_tree_status(self, setup_project_paths): - """ - Check the 'show transfer tree' option that turns off transfer + """Check the 'show transfer tree' option that turns off transfer tree styling by default has the intended effects. It is difficult to test whether the tree is actually styled, so here all underlying configs + the transfer tree legend @@ -56,7 +51,6 @@ async def test_show_transfer_tree_status(self, setup_project_paths): app = TuiApp() async with app.run_test(size=self.tui_size()) as pilot: - # First check the show transfer tree styling is off # in the project manager tab and legend does not exist. await self.check_and_click_onto_existing_project( diff --git a/tests/tests_tui/test_tui_transfer.py b/tests/tests_tui/test_tui_transfer.py index 3e8c389f1..af26264f6 100644 --- a/tests/tests_tui/test_tui_transfer.py +++ b/tests/tests_tui/test_tui_transfer.py @@ -7,8 +7,7 @@ class TestTuiTransfer(TuiBase): - """ - Test transferring through the TUI (entire project, top + """Test transferring through the TUI (entire project, top level only or custom). This class leverages the underlying test utils that check API transfers. """ @@ -24,7 +23,6 @@ async def test_transfer_entire_project( app = TuiApp() async with app.run_test(size=self.tui_size()) as pilot: - await self.check_and_click_onto_existing_project( pilot, project_name ) @@ -57,8 +55,7 @@ async def test_transfer_entire_project( await pilot.pause() async def check_persistent_settings(self, pilot): - """ - Run transfer with each overwrite setting and check it is propagated + """Run transfer with each overwrite setting and check it is propagated to datashuttle methods. """ await self.set_and_check_persistent_settings(pilot, "never", True) @@ -70,7 +67,6 @@ async def check_persistent_settings(self, pilot): ) async def set_overwrite_checkbox(self, pilot, overwrite_setting): - """""" all_positions = {"never": None, "always": 5, "if_source_newer": 6} position = all_positions[overwrite_setting] @@ -89,8 +85,7 @@ async def set_transfer_tab_dry_run_checkbox(self, pilot, dry_run_setting): async def set_and_check_persistent_settings( self, pilot, overwrite_setting, dry_run_setting ): - """ - Run transfer with an overwrite setting and check it is propagated + """Run transfer with an overwrite setting and check it is propagated to datashuttle methods by checking the logs. """ await self.set_overwrite_checkbox(pilot, overwrite_setting) @@ -111,14 +106,12 @@ async def set_and_check_persistent_settings( async def test_transfer_top_level_folder( self, setup_project_paths, top_level_folder, upload_or_download ): - """""" tmp_config_path, tmp_path, project_name = setup_project_paths.values() subs, sessions = test_utils.get_default_sub_sessions_to_test() app = TuiApp() async with app.run_test(size=self.tui_size()) as pilot: - await self.check_and_click_onto_existing_project( pilot, project_name ) @@ -167,7 +160,6 @@ async def test_transfer_custom( app = TuiApp() async with app.run_test(size=self.tui_size()) as pilot: - await self.check_and_click_onto_existing_project( pilot, project_name ) @@ -228,7 +220,6 @@ async def test_transfer_custom( async def switch_top_level_folder_select( self, pilot, id, top_level_folder ): - if top_level_folder == "rawdata": assert pilot.app.screen.query_one(id).value == "rawdata" else: @@ -236,7 +227,6 @@ async def switch_top_level_folder_select( assert pilot.app.screen.query_one(id).value == "derivatives" async def run_transfer(self, pilot, upload_or_download): - """""" # Check assumed default is correct on the transfer switch assert pilot.app.screen.query_one("#transfer_switch").value is False @@ -253,7 +243,6 @@ def setup_project_for_data_transfer( top_level_folder_list, upload_or_download, ): - """""" for top_level_folder in top_level_folder_list: test_utils.make_and_check_local_project_folders( project, diff --git a/tests/tests_tui/test_tui_validate.py b/tests/tests_tui/test_tui_validate.py index af60c32e3..9bf4f7737 100644 --- a/tests/tests_tui/test_tui_validate.py +++ b/tests/tests_tui/test_tui_validate.py @@ -9,19 +9,15 @@ class TestTuiValidate(TuiBase): - @pytest.mark.asyncio async def test_validate_on_project_manager_output( self, setup_project_paths ): - """ - Check that the validate RichLog is updated as expected. - """ + """Check that the validate RichLog is updated as expected.""" tmp_config_path, tmp_path, project_name = setup_project_paths.values() app = TuiApp() async with app.run_test(size=self.tui_size()) as pilot: - # Go to the validate tab on project manager, make # some badly formatted files. await self.check_and_click_onto_existing_project( @@ -72,15 +68,13 @@ async def test_validate_on_project_manager_output( async def test_validate_on_project_manager_kwargs( self, setup_project_paths, mocker ): - """ - Check options are properly passed through to validate_project + """Check options are properly passed through to validate_project from the project manager validate tab (using mocker). """ tmp_config_path, tmp_path, project_name = setup_project_paths.values() app = TuiApp() async with app.run_test(size=self.tui_size()) as pilot: - # Set up a project and open the validate tab await self.check_and_click_onto_existing_project( pilot, project_name @@ -165,8 +159,7 @@ async def test_validate_on_project_manager_kwargs( @pytest.mark.asyncio async def test_validate_at_path_kwargs(self, setup_project_paths, mocker): - """ - Test kwargs are properly passed through from the TUI to `quick_validate_project` + """Test kwargs are properly passed through from the TUI to `quick_validate_project` with mocker. Note that the 'Select' button / directorytree is not tested here, as the screen is tested elsewhere and it's non-critical feature here. """ @@ -174,7 +167,6 @@ async def test_validate_at_path_kwargs(self, setup_project_paths, mocker): app = TuiApp() async with app.run_test(size=self.tui_size()) as pilot: - # Open the validation window and input path to project project_path = (tmp_path / "local" / project_name).as_posix() diff --git a/tests/tests_tui/test_tui_widgets_and_defaults.py b/tests/tests_tui/test_tui_widgets_and_defaults.py index de0efd009..e37117f31 100644 --- a/tests/tests_tui/test_tui_widgets_and_defaults.py +++ b/tests/tests_tui/test_tui_widgets_and_defaults.py @@ -14,8 +14,7 @@ class TestTuiWidgets(TuiBase): - """ - This class performs fundamental checks on the default display + """Performs fundamental checks on the default display of widgets and that changing widgets properly change underlying configs. This does not perform any functional tests e.g. creation of configs of new files. @@ -27,12 +26,9 @@ class TestTuiWidgets(TuiBase): @pytest.mark.asyncio async def test_new_project_configs(self, empty_project_paths): - """ - Test all widgets display as expected on the New Project configs page. - """ + """Test all widgets display as expected on the New Project configs page.""" app = TuiApp() async with app.run_test(size=self.tui_size()) as pilot: - # Select a new project, check NewProjectScreen is displayed correctly. await self.scroll_to_click_pause( pilot, "#mainwindow_new_project_button" @@ -127,7 +123,6 @@ async def test_new_project_configs(self, empty_project_paths): == "" ) if platform.system() == "Windows": - assert ( configs_content.query_one( "#configs_central_path_input" @@ -213,7 +208,6 @@ async def test_new_project_configs(self, empty_project_paths): async def check_new_project_ssh_widgets( self, configs_content, ssh_on, save_pressed=False ): - """""" assert configs_content.query_one( "#configs_setup_ssh_connection_button" ).visible is ( @@ -240,8 +234,7 @@ async def check_new_project_ssh_widgets( @pytest.mark.asyncio async def test_existing_project_configs(self, setup_project_paths): - """ - Because the underlying screen is shared between new and existing + """Because the underlying screen is shared between new and existing project configs, in the existing project configs just check widgets are hidden as expected. """ @@ -249,7 +242,6 @@ async def test_existing_project_configs(self, setup_project_paths): app = TuiApp() async with app.run_test(size=self.tui_size()) as pilot: - # Navigate to the existing project and click onto the # configs tab. await self.check_and_click_onto_existing_project( @@ -278,15 +270,13 @@ async def test_existing_project_configs(self, setup_project_paths): @pytest.mark.asyncio async def test_create_folders_widgets_display(self, setup_project_paths): - """ - Test all widgets on the 'Create' tab of the project manager screen + """Test all widgets on the 'Create' tab of the project manager screen are displayed as expected. """ tmp_config_path, tmp_path, project_name = setup_project_paths.values() app = TuiApp() async with app.run_test(size=self.tui_size()) as pilot: - await self.check_and_click_onto_existing_project( pilot, project_name ) @@ -368,15 +358,13 @@ async def test_create_folders_widgets_display(self, setup_project_paths): @pytest.mark.asyncio async def test_create_folder_settings_widgets(self, setup_project_paths): - """ - Test the widgets in the 'Settings' menu of the project + """Test the widgets in the 'Settings' menu of the project manager's 'Create' tab. """ tmp_config_path, tmp_path, project_name = setup_project_paths.values() app = TuiApp() async with app.run_test(size=self.tui_size()) as pilot: - await self.setup_existing_project_create_tab_filled_sub_and_ses( pilot, project_name, create_folders=False ) @@ -501,8 +489,7 @@ async def test_create_folder_settings_widgets(self, setup_project_paths): async def test_name_templates_widgets_and_settings( self, setup_project_paths ): - """ - Check the 'Name Templates' section of the 'Create' tab 'Settings + """Check the 'Name Templates' section of the 'Create' tab 'Settings page. Here both subject and session configs share the same input, so ensure these are mapped correctly by the radiobutton setting, and that the underlying configs are set correctly. @@ -514,7 +501,6 @@ async def test_name_templates_widgets_and_settings( app = TuiApp() async with app.run_test(size=self.tui_size()) as pilot: - await self.setup_existing_project_create_tab_filled_sub_and_ses( pilot, project_name, create_folders=False ) @@ -664,15 +650,13 @@ async def test_name_templates_widgets_and_settings( @pytest.mark.asyncio async def test_bypass_validation_settings(self, setup_project_paths): - """ - Test all configs that underly the 'bypass validation' + """Test all configs that underly the 'bypass validation' setting are updated correctly by the widget. """ tmp_config_path, tmp_path, project_name = setup_project_paths.values() app = TuiApp() async with app.run_test(size=self.tui_size()) as pilot: - await self.setup_existing_project_create_tab_filled_sub_and_ses( pilot, project_name, create_folders=False ) @@ -731,15 +715,13 @@ async def test_bypass_validation_settings(self, setup_project_paths): @pytest.mark.asyncio async def test_all_top_level_folder_selects(self, setup_project_paths): - """ - Test all 'top level folder' selects (in Create and Transfer tabs) + """Test all 'top level folder' selects (in Create and Transfer tabs) update the underlying configs correctly. """ tmp_config_path, tmp_path, project_name = setup_project_paths.values() app = TuiApp() async with app.run_test(size=self.tui_size()) as pilot: - # Open project, check top level folder are correct await self.setup_existing_project_create_tab_filled_sub_and_ses( pilot, project_name, create_folders=False @@ -904,9 +886,7 @@ async def check_top_folder_select( expected_val, move_to_position: Union[bool, int] = False, ): - """ - If move to position is not False, must be int specifying position - """ + """If move to position is not False, must be int specifying position.""" if move_to_position: await self.move_select_to_position(pilot, id, move_to_position) @@ -929,8 +909,7 @@ async def check_top_folder_select( async def test_search_central_for_suggestion_settings( self, setup_project_paths ): - """ - Check the settings for the checkbox that selects include_central when + """Check the settings for the checkbox that selects include_central when getting the next subject or session in the 'Create' tab and ensure that the underlying settings are changed. """ @@ -938,7 +917,6 @@ async def test_search_central_for_suggestion_settings( app = TuiApp() async with app.run_test(size=self.tui_size()) as pilot: - await self.setup_existing_project_create_tab_filled_sub_and_ses( pilot, project_name, create_folders=False ) @@ -1008,8 +986,7 @@ async def test_search_central_for_suggestion_settings( @pytest.mark.asyncio async def test_all_checkboxes(self, setup_project_paths): - """ - Check all datatype checkboxes (Create and Transfer tab) + """Check all datatype checkboxes (Create and Transfer tab) correctly update the underlying configs. These are tested together to ensure there are no strange interaction between these as they both share stored in the project's 'tui' @@ -1019,7 +996,6 @@ async def test_all_checkboxes(self, setup_project_paths): app = TuiApp() async with app.run_test(size=self.tui_size()) as pilot: - await self.check_and_click_onto_existing_project( pilot, project_name ) @@ -1108,7 +1084,6 @@ async def test_all_checkboxes(self, setup_project_paths): await pilot.pause() def check_datatype_checkboxes(self, pilot, tab, expected_on): - """""" assert tab in ["create", "transfer"] if tab == "create": id = "#create_folders_datatype_checkboxes" @@ -1136,7 +1111,6 @@ async def test_all_transfer_widgets(self, setup_project_paths): app = TuiApp() async with app.run_test(size=self.tui_size()) as pilot: - # Navigate to the existing project and click onto the # configs tab. await self.check_and_click_onto_existing_project( @@ -1318,12 +1292,10 @@ async def test_all_transfer_widgets(self, setup_project_paths): @pytest.mark.asyncio async def test_overwrite_existing_files(self, setup_project_paths): - """ """ tmp_config_path, tmp_path, project_name = setup_project_paths.values() app = TuiApp() async with app.run_test(size=self.tui_size()) as pilot: - # Navigate to the existing project and click onto the # configs tab. await self.check_and_click_onto_existing_project( @@ -1375,8 +1347,7 @@ async def test_overwrite_existing_files(self, setup_project_paths): @pytest.mark.asyncio async def test_dry_run(self, setup_project_paths): - """ - Test the dry run setting. This is very similar in structure + """Test the dry run setting. This is very similar in structure to `test_overwrite_existing_files()`, merge if more persistent settings added. """ @@ -1384,7 +1355,6 @@ async def test_dry_run(self, setup_project_paths): app = TuiApp() async with app.run_test(size=self.tui_size()) as pilot: - # Navigate to the existing project and click onto the # configs tab. await self.check_and_click_onto_existing_project( @@ -1427,7 +1397,6 @@ def check_dry_run(self, pilot, project_name, value): def check_overwrite_existing_files_configs( self, pilot, project_name, value ): - """""" assert ( pilot.app.screen.query_one("#transfer_tab_overwrite_select").value == value diff --git a/tests/tests_tui/tui_base.py b/tests/tests_tui/tui_base.py index 510b943b2..b5b7e88d8 100644 --- a/tests/tests_tui/tui_base.py +++ b/tests/tests_tui/tui_base.py @@ -8,13 +8,10 @@ class TuiBase: - """ - Contains fixtuers and helper functions for TUI tests. - """ + """Contains fixtuers and helper functions for TUI tests.""" def tui_size(self): - """ - If the TUI screen in the test environment is not + """If the TUI screen in the test environment is not large enough, often the error `textual.pilot.OutOfBounds: Target offset is outside of currently-visible screen region.` @@ -27,8 +24,7 @@ def tui_size(self): @pytest_asyncio.fixture(scope="function") async def empty_project_paths(self, tmp_path_factory, monkeypatch): - """ - Get the paths and project name for a non-existent (i.e. not + """Get the paths and project name for a non-existent (i.e. not yet setup) project. """ project_name = "my-test-project" @@ -48,9 +44,7 @@ async def empty_project_paths(self, tmp_path_factory, monkeypatch): @pytest_asyncio.fixture(scope="function") async def setup_project_paths(self, empty_project_paths): - """ - Get the paths and project name for a setup project. - """ + """Get the paths and project name for a setup project.""" test_utils.setup_project_fixture( empty_project_paths["tmp_path"], empty_project_paths["project_name"], @@ -59,8 +53,7 @@ async def setup_project_paths(self, empty_project_paths): return empty_project_paths def monkeypatch_get_datashuttle_path(self, tmp_config_path, _monkeypatch): - """ - For these tests, store the datashuttle configs (usually stored in + """For these tests, store the datashuttle configs (usually stored in Path.home()) in the `tmp_path` provided by pytest, as it simplifies testing here. @@ -79,8 +72,7 @@ def mock_get_datashuttle_path(): ) def monkeypatch_print(self, _monkeypatch): - """ - Calls to `print` in datashuttle crash the TUI in the + """Calls to `print` in datashuttle crash the TUI in the test environment. I am not sure why. Get around this in tests by monkeypatching the datashuttle print method. """ @@ -93,9 +85,7 @@ def return_none(arg1, arg2=None): ) async def fill_input(self, pilot, id, value): - """ - Fill and input of `id` with `value`. - """ + """Fill and input of `id` with `value`.""" await self.scroll_to_click_pause(pilot, id) pilot.app.screen.query_one(id).value = "" await pilot.press(*value) @@ -104,8 +94,7 @@ async def fill_input(self, pilot, id, value): async def setup_existing_project_create_tab_filled_sub_and_ses( self, pilot, project_name, create_folders=False ): - """ - Set up an existing project and switch to the 'Create' tab + """Set up an existing project and switch to the 'Create' tab on the project manager screen. """ await self.check_and_click_onto_existing_project(pilot, project_name) @@ -123,16 +112,14 @@ async def setup_existing_project_create_tab_filled_sub_and_ses( ) async def double_click(self, pilot, id, control=False): - """ - Double-click on a widget of `id`, if `control` is `True` the + """Double-click on a widget of `id`, if `control` is `True` the control modifier key will be used. """ for _ in range(2): await self.scroll_to_click_pause(pilot, id, control=control) async def reload_tree_nodes(self, pilot, id, num_nodes): - """ - For some reason, for TUI tree nodes to register in the + """For some reason, for TUI tree nodes to register in the test environment all need to have `reload_node` called on the node. """ @@ -143,41 +130,32 @@ async def reload_tree_nodes(self, pilot, id, num_nodes): await pilot.pause() async def hover_and_press_tree(self, pilot, id, hover_line, press_string): - """ - Hover over a directorytree at a node-line and press a specific string - """ + """Hover over a directorytree at a node-line and press a specific string.""" await pilot.pause() pilot.app.screen.query_one(id).hover_line = hover_line await pilot.pause() await self.press_tree(pilot, id, press_string) async def press_tree(self, pilot, id, press_string): - """ - Click on a tree to give it focus and press buttons - """ + """Click on a tree to give it focus and press buttons.""" await self.scroll_to_click_pause(pilot, id) await pilot.press(press_string) await pilot.pause() async def scroll_to_and_pause(self, pilot, id): - """ - Scroll to a widget and pause. - """ + """Scroll to a widget and pause.""" widget = pilot.app.screen.query_one(id) widget.scroll_visible(animate=False) await pilot.pause() async def scroll_to_click_pause(self, pilot, id, control=False): - """ - Scroll to a widget, click it and call pause. - """ + """Scroll to a widget, click it and call pause.""" await self.scroll_to_and_pause(pilot, id) await pilot.click(id, control=control) await pilot.pause() async def check_and_click_onto_existing_project(self, pilot, project_name): - """ - From the main menu, go onto the select project page and + """From the main menu, go onto the select project page and select the project created in the test environment. Perform general TUI checks during the navigation. """ @@ -204,13 +182,12 @@ async def change_checkbox(self, pilot, id): async def switch_tab(self, pilot, tab): assert tab in ["create", "transfer", "configs", "logging", "validate"] + content_tab = ContentTab.add_prefix(f"tabscreen_{tab}_tab") await self.scroll_to_click_pause(pilot, f"Tab#{content_tab}") async def turn_off_all_datatype_checkboxes(self, pilot, tab="create"): - """ - Make sure all checkboxes are off to start - """ + """Make sure all checkboxes are off to start.""" assert tab in ["create", "transfer"] checkbox_names = canonical_configs.get_broad_datatypes() @@ -234,8 +211,7 @@ async def turn_off_all_datatype_checkboxes(self, pilot, tab="create"): async def exit_to_main_menu_and_reeneter_project_manager( self, pilot, project_name ): - """ - Exist from the project manager screen, then re-enter back + """Exist from the project manager screen, then re-enter back into the project. This refreshes the screen and is important in testing state is preserved across re-loading. """ @@ -244,15 +220,12 @@ async def exit_to_main_menu_and_reeneter_project_manager( await self.check_and_click_onto_existing_project(pilot, project_name) async def close_messagebox(self, pilot): - """ - Close the modal_dialogs.Messagebox - """ + """Close the modal_dialogs.Messagebox.""" pilot.app.screen.on_button_pressed() await pilot.pause() async def move_select_to_position(self, pilot, id, position): - """ - Move a select widget to a specific position (e.g. "rawdata" + """Move a select widget to a specific position (e.g. "rawdata" or "derivatives" select). The position can be determined by trial and error. """ diff --git a/tests/tests_unit/test_links.py b/tests/tests_unit/test_links.py index 5da0e04c2..86adc8b9f 100644 --- a/tests/tests_unit/test_links.py +++ b/tests/tests_unit/test_links.py @@ -4,8 +4,7 @@ def test_links(): - """ - Test canonical links are working. Unfortunately Zulip links cannot + """Test canonical links are working. Unfortunately Zulip links cannot be validated. """ assert validators.url(links.get_docs_link()) diff --git a/tests/tests_unit/test_unit.py b/tests/tests_unit/test_unit.py index 99b661693..ec2cb875f 100644 --- a/tests/tests_unit/test_unit.py +++ b/tests/tests_unit/test_unit.py @@ -7,9 +7,7 @@ class TestUnit: - """ - Currently contains misc. unit tests. - """ + """Currently contains misc. unit tests.""" @pytest.mark.parametrize( "underscore_position", ["left", "right", "both", "none"] @@ -18,13 +16,12 @@ class TestUnit: "key", [tags("date"), tags("time"), tags("datetime")] ) def test_datetime_string_replacement(self, key, underscore_position): - r""" - Test the function that replaces @DATE, @TIME@ or @DATETIME@ - keywords with the date / time / datetime. Also, it will - pre/append underscores to the tags if they are not - already there (e.g if user input "sub-001@DATE"). - Note cannot use regex \d{8} format because we are in an - f-string. + r"""Test the function that replaces @DATE, @TIME@ or @DATETIME@ + keywords with the date / time / datetime. + + Also, it will pre/append underscores to the tags if they are not + already there (e.g if user input "sub-001@DATE"). Note cannot use + regex \d{8} format because we are in an f-string. """ start = "sub-001" end = "other-tag" @@ -42,13 +39,12 @@ def test_datetime_string_replacement(self, key, underscore_position): name_list = [name] formatting.update_names_with_datetime(name_list) - assert ( - re.search(regex, name_list[0]) is not None - ), "datetime formatting is incorrect." + assert re.search(regex, name_list[0]) is not None, ( + "datetime formatting is incorrect." + ) @pytest.mark.parametrize("prefix", ["sub", "ses"]) def test_process_to_keyword_in_sub_input(self, prefix): - """ """ results = formatting.update_names_with_range_to_flag( [f"{prefix}-001", f"{prefix}-01{tags('to')}123"], prefix ) @@ -109,8 +105,7 @@ def test_process_to_keyword_bad_input_raises_error( ) def test_get_value_from_bids_name_regexp(self): - """ - Test the regexp that finds the value from a BIDS-name + """Test the regexp that finds the value from a BIDS-name key-value pair. """ bids_name = "sub-0123125_ses-11312_datetime-5345323_id-3asd@523" @@ -128,8 +123,7 @@ def test_get_value_from_bids_name_regexp(self): assert id == "3asd@523" def test_num_leading_zeros(self): - """ - Check num_leading_zeros handles prefixed and non-prefixed + """Check num_leading_zeros handles prefixed and non-prefixed case from -1 to -(101x 0)1. """ for i in range(101): @@ -145,10 +139,9 @@ def test_num_leading_zeros(self): def test_get_max_sub_or_ses_num_and_value_length_empty( self, prefix, default_num_value_digits ): - """ - When the list of sub or ses names is empty, the returned max number + """When the list of sub or ses names is empty, the returned max number should be zero and the `default_num_value_digits` - be set to the passed default + be set to the passed default. """ ( max_value, @@ -162,8 +155,7 @@ def test_get_max_sub_or_ses_num_and_value_length_empty( @pytest.mark.parametrize("prefix", ["sub", "ses"]) def test_get_max_sub_or_ses_num_and_value_length_error(self, prefix): - """ - An error will be shown if the sub or ses value digits are inconsistent, + """An error will be shown if the sub or ses value digits are inconsistent, because it is not possible to return the number of values required. A warning should be shown in that the number of value digits are @@ -213,8 +205,7 @@ def test_get_max_sub_or_ses_num_and_value_length_error(self, prefix): def test_get_max_sub_or_ses_num_and_value_length( self, prefix, test_max_num, test_num_digits ): - """ - Test many combinations of subject names + """Test many combinations of subject names and number of digits for a project, e.g. `names = ["sub-001", ... "sub-101"]`. """ @@ -235,9 +226,7 @@ def test_get_max_sub_or_ses_num_and_value_length( @pytest.mark.parametrize("prefix", ["sub", "ses"]) def test_get_max_sub_or_ses_num_and_value_length_edge_case(self, prefix): - """ - Test the edge case where the subject number does not start at 1. - """ + """Test the edge case where the subject number does not start at 1.""" names = [f"{prefix}-09", f"{prefix}-10", f"{prefix}-11"] max_num, num_digits = getters.get_max_sub_or_ses_num_and_value_length( @@ -252,8 +241,7 @@ def test_get_max_sub_or_ses_num_and_value_length_edge_case(self, prefix): # ------------------------------------------------------------------------- def make_name(self, key, underscore_position, start, end): - """ - Make name with / without underscore to test every + """Make name with / without underscore to test every possibility. """ if underscore_position == "left": diff --git a/tests/tests_unit/test_validation_unit.py b/tests/tests_unit/test_validation_unit.py index 473a8a638..262a4741d 100644 --- a/tests/tests_unit/test_validation_unit.py +++ b/tests/tests_unit/test_validation_unit.py @@ -6,8 +6,7 @@ class TestValidationUnit: @pytest.mark.parametrize("prefix", ["sub", "ses"]) def test_more_than_one_instance(self, prefix): - """ - Check that any duplicate sub or ses values are caught + """Check that any duplicate sub or ses values are caught in `validate_list_of_names()`. """ error_message = validation.validate_list_of_names( @@ -16,7 +15,7 @@ def test_more_than_one_instance(self, prefix): assert len(error_message) == 1 assert ( - f"DUPLICATE_PREFIX: The name: {prefix}-99_date-20231214_{prefix}-98 of " + f"DUPLICATE_PREFIX: The name: {prefix}-99_date-20231214_{prefix}-98 " f"contains more than one instance of the prefix {prefix}." == error_message[0] ) @@ -36,8 +35,7 @@ def test_more_than_one_instance(self, prefix): ], ) def test_name_does_not_begin_with_prefix(self, prefix_and_names): - """ - Check validation that names passed to `validate_list_of_names()` + """Check validation that names passed to `validate_list_of_names()` start with the prefix prefix (sub or ses). """ prefix, names = prefix_and_names @@ -48,9 +46,8 @@ def test_name_does_not_begin_with_prefix(self, prefix_and_names): @pytest.mark.parametrize("prefix", ["sub", "ses"]) def test_special_characters_in_format_names(self, prefix): - """ - Check `validate_list_of_names()` catches - spaces in passed names (not all names are bad + """Check `validate_list_of_names()` catches + spaces in passed names (not all names are bad. """ error_messages = validation.validate_list_of_names( [ @@ -72,7 +69,6 @@ def test_special_characters_in_format_names(self, prefix): ], ) def test_prefix_is_not_an_integer(self, prefix_and_names): - """ """ prefix, names = prefix_and_names error_messages = validation.validate_list_of_names(names, prefix) @@ -88,11 +84,9 @@ def test_prefix_is_not_an_integer(self, prefix_and_names): def test_formatting_dashes_and_underscore_alternate_incorrectly( self, prefix ): - """ - Check `validate_list_of_names()` catches "-" and "_" that + """Check `validate_list_of_names()` catches "-" and "_" that are not in the correct order. """ - # Test a large range of bad names. Do not use # parametrize so we can use f"{prefix}". # There should always be two validation errors per list. @@ -144,8 +138,7 @@ def test_formatting_dashes_and_underscore_alternate_incorrectly( @pytest.mark.parametrize("prefix", ["sub", "ses"]) def test_inconsistent_value_lengths_in_list_of_names(self, prefix): - """ - Ensure a list of sub / ses names that contain inconsistent + """Ensure a list of sub / ses names that contain inconsistent leading zeros (e.g. ["sub-001", "sub-02"]) leads to an error. """ for names in [ @@ -163,8 +156,7 @@ def test_inconsistent_value_lengths_in_list_of_names(self, prefix): @pytest.mark.parametrize("prefix", ["sub", "ses"]) def test_duplicate_ids_in_list_of_names(self, prefix): - """ - Ensure a list of sub / ses names that contain duplicate sub / ses + """Ensure a list of sub / ses names that contain duplicate sub / ses ids (e.g. ["sub-001", "sub-001_@DATE@"]) leads to an error. """ names = [ @@ -183,12 +175,10 @@ def test_duplicate_ids_in_list_of_names(self, prefix): @pytest.mark.parametrize("prefix", ["sub", "ses"]) def test_new_name_duplicates_existing(self, prefix): - """ - Test the function `new_name_duplicates_existing()` + """Test the function `new_name_duplicates_existing()` that will throw an error if a sub / ses name matches an existing name (unless it matches exactly). """ - # Check an exactly matching case that should not raise and error new_name = f"{prefix}-002" existing_names = [f"{prefix}-001", f"{prefix}-002", f"{prefix}-003"] @@ -231,8 +221,7 @@ def test_new_name_duplicates_existing(self, prefix): ) def test_tags_autoreplace_in_regexp(self): - """ - Check the validation function `replace_tags_in_regexp()` + """Check the validation function `replace_tags_in_regexp()` correctly replaces tags in a regexp with their regexp equivalent. Test date, time and datetime with some random regexp that @@ -259,7 +248,6 @@ def test_tags_autoreplace_in_regexp(self): ) def test_handle_path(self): - output = validation.handle_path("message", None) assert output == "message" @@ -271,7 +259,6 @@ def test_handle_path(self): @pytest.mark.parametrize("prefix", ["sub", "ses"]) def test_datetime_iso_format(self, prefix): - # Test dates error_messages = validation.validate_list_of_names( [