diff --git a/CHANGELOG.md b/CHANGELOG.md index 42c0180..7ef590a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,21 @@ and this project adheres to [Semantic Versioning]. ## [Unreleased] +## [0.0.24] - 2025-06-20 + +### Added in 0.0.24 + +- sz_update_project can now upgrade V3 -> V4 and V4 -> V4 projects + +### Changed in 0.0.24 + +- Update tools from get_configs() to get_config_registry() (SDK change) +- sz_command now uses the setting and command `scroll` instead of `page` + +### Fixed in 0.0.24 + +- sz_create_project shouldn't copy sz_update_project to a new project + ## [0.0.23] - 2025-06-12 ### Fixed in 0.0.23 diff --git a/pyproject.toml b/pyproject.toml index c6a00bd..23fe881 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,7 +41,9 @@ disable = [ "consider-using-f-string", "line-too-long", "too-many-branches", - "too-many-locals" + "too-many-instance-attributes", + "too-many-locals", + "too-many-statements" ] good-names = [ "template-python" diff --git a/requirements.txt b/requirements.txt index 6146007..a757902 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,2 @@ -senzing==0.2.2 +senzing==0.2.16 senzing-core==0.3.15 diff --git a/setup.cfg b/setup.cfg index c5ecb03..c3e44b5 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = python-tools -version = 0.0.23 +version = 0.0.24 author = senzing author_email = support@senzing.com description = Python Tools @@ -21,8 +21,8 @@ package_dir = packages = find: python_requires = >=3.9 install_requires = - senzing >= 0.2.12 - senzing-core >= 0.3.11 + senzing >= 0.2.16 + senzing-core >= 0.3.15 [options.packages.find] where = src diff --git a/sz_tools/_project_helpers.py b/sz_tools/_project_helpers.py new file mode 100644 index 0000000..9296d5b --- /dev/null +++ b/sz_tools/_project_helpers.py @@ -0,0 +1,188 @@ +"""Helpers for creating and updating projects""" + +import json +import shutil +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +from packaging import version as p_version + +V3_BACKUP_PATH = "v3_to_v4_upgrade_backups" +V4_BUILD = "szBuildVersion.json" +SZ_SYS_PATH = Path("/opt/senzing") +V4_SYS_PATH = SZ_SYS_PATH / "er" +V4_DATA_PATH = SZ_SYS_PATH / "data" +V4_SYS_BUILD = V4_SYS_PATH / V4_BUILD + +COPY_TO_PROJ = { + "er/": {"files": ["*"], "excludes": ["sz_create_project", "sz_update_project", "_project_helpers.py"]}, + "data": {"files": ["*"], "excludes": []}, +} + +PERMISSIONS = { + ".": { + "dir_pint": 0, + "file_pint": 0o660, + "files": ["LICENSE", "NOTICES", "README.1ST", "szBuildVersion.json"], + "excludes": ["setupEnv"], + "recursive": False, + }, + "setupEnv": {"dir_pint": 0, "file_pint": 0o770, "files": [], "excludes": [], "recursive": False}, + "bin": { + "dir_pint": 0o770, + "file_pint": 0o770, + "files": ["*"], + "excludes": ["__pycache__", "_sz_database.py", "_tool_helpers.py"], + "recursive": False, + }, + "data": {"dir_pint": 0o770, "file_pint": 0o660, "files": ["*"], "excludes": [], "recursive": True}, + "lib": {"dir_pint": 0o770, "file_pint": 0o660, "files": ["*"], "excludes": [], "recursive": False}, + "resources": {"dir_pint": 0o770, "file_pint": 0o660, "files": ["*"], "excludes": ["setupEnv"], "recursive": True}, + "resources/templates/setupEnv": { + "dir_pint": 0, + "file_pint": 0o770, + "files": [], + "excludes": [], + "recursive": False, + }, + "sdk": {"dir_pint": 0o770, "file_pint": 0o660, "files": ["*"], "excludes": [], "recursive": True}, + V3_BACKUP_PATH: {"dir_pint": 0o770, "file_pint": 0, "files": [], "excludes": [], "recursive": False}, +} + +PERMISSIONS_2 = { + "bin": { + "dir_pint": 0o770, + "file_pint": 0o660, + "files": ["_sz_database.py", "_tool_helpers.py"], + "excludes": [], + "recursive": False, + } +} + + +@dataclass() +class SzBuildDetails: + """Build information for a project or Senzing SDK system install""" + + platform: str + version: str + build_version: str + build_number: str + major: int = field(init=False) + minor: int = field(init=False) + micro: int = field(init=False) + + def __post_init__(self) -> None: + self.version_parsed = p_version.parse(self.version) + self.build_version_parsed = p_version.parse(self.build_version) + self.major = self.version_parsed.major + self.minor = self.version_parsed.minor + self.micro = self.version_parsed.micro + + +def get_build_details(path: Path) -> SzBuildDetails: + """Return dataclass with the details from a build file.""" + try: + with open(path, "r", encoding="utf-8") as f: + # Ignore DATA_VERSION, V3 build files had it V4 doesn't + version_dict = {k.lower(): v for k, v in json.load(f).items() if k != "DATA_VERSION"} + except (OSError, json.JSONDecodeError) as err: + raise OSError(f"Couldn't get the build information from {path}: {err}") from err + + return SzBuildDetails(**version_dict) + + +def copy_files_dirs(to_copy: dict[str, Any], source_dir: Path, target_dir: Path) -> None: + """Copy files/directories within and to a project""" + for c_path, c_dict in to_copy.items(): + excludes = c_dict["excludes"] + files = c_dict["files"] + source = source_dir / c_path + + try: + if source.is_dir(): + # If the key in to_copy ends with / copy everything in source dir to target_dir + # er/ as the key copies everything from /opt/senzing/er to target_dir + # + # If the key in to_copy doesn't end with / copy source dir and everything in it to target_dir + # data as the key copies /opt/senzing/data to target_dir/data + target = target_dir if c_path.endswith("/") else target_dir / c_path + + # Copy entire contents of the source directory + if not files or (files and files[0] == "*"): + shutil.copytree(source, target, ignore=shutil.ignore_patterns(*excludes), dirs_exist_ok=True) + + # Create the source directory in the target and only copy listed files + if files and files[0] != "*": + target.mkdir(exist_ok=True, parents=True) + for source_file in [source / f for f in files]: + shutil.copy(source_file, target / source_file.name) + + if source.is_file(): + # Single file copy always copies only the file, if the key to to_copy is er/szBuildVersion.json + # szBuildVersion.json is copied to target_dir and not target_dir/er/szBuildVersion.json + target = target_dir / source.name + target_dir.mkdir(exist_ok=True, parents=True) + shutil.copy(source, target) + except OSError as err: + raise OSError(f"Couldn't copy a file or directory: {err}") from err + + +def setup_env(proj_path: Path) -> None: + """Create a new setupEnv and replace place holders with paths for the project""" + try: + shutil.copy(proj_path / "resources/templates/setupEnv", proj_path) + setup_path = proj_path / "setupEnv" + + with open(setup_path, "r", encoding="utf-8") as in_: + data = in_.read() + + data = data.replace("${SENZING_DIR}", str(proj_path)).replace("${SENZING_CONFIG_PATH}", str(proj_path / "etc")) + + with open(setup_path, "w", encoding="utf-8") as out: + out.write(data) + except OSError as err: + raise OSError(f"Couldn't create a new setupEnv file: {err}") from err + + +def set_permissions(proj_path: Path, permissions: dict[str, dict[str, Any]]) -> None: + """Set permissions for files/dirs copied to the project, or dirs removed and replaced completely e.g., data/""" + try: + for p_path, p_dict in permissions.items(): + dir_pint = p_dict["dir_pint"] + file_pint = p_dict["file_pint"] + files = p_dict["files"] + recursive = p_dict["recursive"] + target = proj_path if p_path.startswith(".") else proj_path / p_path + excludes = [target / e for e in p_dict["excludes"]] + + if target.is_dir(): + if dir_pint != 0: + target.chmod(dir_pint) + d_chmods = ( + [d for d in target.glob("*") if d.is_dir() and not d.is_symlink() and d not in excludes] + if not recursive + else [d for d in target.rglob("*") if d.is_dir() and not d.is_symlink() and d not in excludes] + ) + for dir_ in d_chmods: + dir_.chmod(dir_pint) + + if files and files[0] == "*": + f_chmods = ( + [f for f in target.glob("*") if f.is_file() and not f.is_symlink() and f not in excludes] + if not recursive + else [f for f in target.rglob("*") if f.is_file() and not f.is_symlink() and f not in excludes] + ) + + for file in f_chmods: + Path(target / file).chmod(file_pint) + + if files and files[0] != "*": + for file in files: + Path(target / file).chmod(file_pint) + + if target.is_file(): + target.chmod(file_pint) + except OSError as err: + raise OSError(f"Couldn't set a permission: {err}") from err diff --git a/sz_tools/_tool_helpers.py b/sz_tools/_tool_helpers.py index aca2668..3a7afbf 100644 --- a/sz_tools/_tool_helpers.py +++ b/sz_tools/_tool_helpers.py @@ -1,7 +1,3 @@ -""" -# TODO -""" - from __future__ import annotations import cmd @@ -70,7 +66,7 @@ class TimedOut(Exception): - """# TODO""" + """Timeout""" # ------------------------------------------------------------------------- @@ -78,16 +74,12 @@ class TimedOut(Exception): # ------------------------------------------------------------------------- -@dataclass class Colors: - """# TODO""" - AVAILABLE_THEMES = ["DEFAULT", "LIGHT", "DARK", "TERMINAL"] @classmethod def apply(cls, to_color: Union[int, str], colors_list: str = "") -> Union[int, str]: """apply list of colors to a string""" - # TODO colors_list is a string with multiple entries separated by , if colors_list: prefix = "".join([getattr(cls, i.strip().upper()) for i in colors_list.split(",")]) return f"{prefix}{to_color}{cls.RESET}" @@ -96,7 +88,6 @@ def apply(cls, to_color: Union[int, str], colors_list: str = "") -> Union[int, s @classmethod def set_theme(cls, theme: str) -> None: - """# TODO""" theme = theme.upper() # best for dark backgrounds if theme == "DEFAULT": @@ -290,13 +281,12 @@ def set_theme(cls, theme: str) -> None: def check_environment() -> None: - """# TODO""" # Error if can't locate a sz_engine_config.ini or SENZING_ENGINE_CONFIGURATION_JSON if "SENZING_ETC_PATH" not in os.environ and "SENZING_ROOT" not in os.environ: # Check if set or not and that it's not set to null secj = os.environ.get("SENZING_ENGINE_CONFIGURATION_JSON") if not secj or (secj and len(secj) == 0): - # TODO V4 doc links + # TODO - V4 doc links print( textwrap.dedent( """\n\ @@ -317,7 +307,6 @@ def check_environment() -> None: def get_g2module_path() -> Path: - """# TODO""" file_paths = [] msg_args = f"Use command line argument -c (--inifile) to specify the path & filename for {CONFIG_FILE}\n" @@ -361,7 +350,6 @@ def get_g2module_path() -> Path: def print_config_locations(locations: List[Path]) -> None: - """# TODO""" _ = [print(f"\t{loc}") for loc in locations] print() @@ -396,8 +384,6 @@ def get_ini_as_json_str(ini_file: Path) -> str: def get_engine_config(ini_file_name: Union[str, None] = None) -> str: - """# TODO""" - # Initial check to determine is environment variables expected are set check_environment() @@ -431,9 +417,11 @@ def combine_engine_flags(flags: Union[List[TSzEngineFlags], List[str]]) -> int: return result -def get_engine_flag_names() -> List[str]: - """# TODO""" - return list(SzEngineFlags.__members__.keys()) +def get_engine_flag_names(filter_: List[str] = None) -> List[str]: # type: ignore[assignment] + """Return flag names and optionally filter""" + filter_ = [] if filter_ is None else filter_ + + return [f for f in SzEngineFlags.__members__ if f not in filter_] def get_engine_flags_as_int(flags: List[str]) -> int: @@ -446,8 +434,6 @@ def get_engine_flags_as_int(flags: List[str]) -> int: return int(flags[0]) # Named engine flag(s) used, combine and return the int value - # TODO - # return SzEngineFlags.combine_flags(flags) return combine_engine_flags(flags) @@ -482,8 +468,6 @@ def check_path_exists(path: Union[Path, str]) -> bool: def check_file_exists(file_name: Union[Path, str]) -> bool: - """# TODO""" - if isinstance(file_name, str): file_name = Path(file_name) @@ -494,8 +478,6 @@ def check_file_exists(file_name: Union[Path, str]) -> bool: # def check_file_readable(file_name: Union[Path, str]) -> bool: -# """# TODO""" - # if isinstance(file_name, str): # file_name = Path(file_name) @@ -512,10 +494,7 @@ def check_file_exists(file_name: Union[Path, str]) -> bool: # ------------------------------------------------------------------------- -# TODO - This can be merged into colorize_output def colorize_str(string: str, colors_list: str = "", color_disabled: bool = False) -> str: - """# TODO""" - if color_disabled: return string @@ -523,7 +502,6 @@ def colorize_str(string: str, colors_list: str = "", color_disabled: bool = Fals def colorize_json(json_str: str, color_disabled: bool = False) -> str: - """# TODO""" if color_disabled: return json_str @@ -538,13 +516,11 @@ def colorize_json(json_str: str, color_disabled: bool = False) -> str: return json_color -# TODO - Move into Colors and add the missing values? def colorize_output( output: Union[Exception, int, str], color_or_type: str, output_color: bool = True, ) -> str: - """# TODO""" if not output: return "" @@ -590,21 +566,18 @@ def colorize_cmd_prompt(prompt: str, color_or_type: str, color_prompt: bool = Tr def print_debug(msg: str, end_str: str = "\n\n", output_color: bool = True) -> None: - """# TODO""" print(f"\n{colorize_output('DEBUG:', 'debug', output_color)} {msg}", end=end_str) def print_error( msg: Union[Exception, str], end_str: str = "\n\n", output_color: bool = True, exit_: bool = False ) -> None: - """# TODO""" print(f"\n{colorize_output('ERROR:', 'error', output_color)} {msg}", end=end_str) if exit_: sys.exit(1) def print_info(msg: Union[Exception, str], end_str: str = "\n\n", output_color: bool = True, info_prefix=True) -> None: - """# TODO""" if info_prefix: print(f"\n{colorize_output('INFO:', 'info', output_color)} {msg}", end=end_str) else: @@ -612,7 +585,6 @@ def print_info(msg: Union[Exception, str], end_str: str = "\n\n", output_color: def print_warning(msg: Union[Exception, str], end_str: str = "\n\n", output_color: bool = True) -> None: - """# TODO""" # Warnings may be multiline strings, if they are don't add WARNING: before the msg to color if isinstance(msg, str) and "\n" in msg: print(f"\n{colorize_output(msg, 'warning', output_color)}", end=end_str) @@ -627,7 +599,6 @@ def print_response( color_output: bool = True, color: str = "", ) -> str: - """# TODO""" strip_colors = True if not response: @@ -684,12 +655,10 @@ def print_response( def do_shell(self: Union[SzCmdShell, SzCfgShell], line: str) -> None: # pylint: disable=unused-argument - """# TODO""" print(os.popen(line).read()) def do_help(self: Union[SzCmdShell, SzCfgShell], help_topic: str) -> None: - """# TODO""" if not help_topic or help_topic == "overview": self.help_overview() return @@ -751,7 +720,6 @@ def do_help(self: Union[SzCmdShell, SzCfgShell], help_topic: str) -> None: def do_history() -> None: - """# TODO""" print() for i in range(readline.get_current_history_length()): print(readline.get_history_item(i + 1)) @@ -796,12 +764,10 @@ def history_setup(module_name: str) -> Union[None, Path]: def history_write_file(file: Path) -> None: - """# TODO""" readline.write_history_file(file) def history_disabled(file: Path) -> None: - """# TODO""" # Save current session history history_now = [readline.get_history_item(i) for i in range(1, readline.get_current_history_length() + 1)] @@ -820,7 +786,6 @@ def history_disabled(file: Path) -> None: def response_to_clipboard(last_response: str) -> None: - """# TODO""" if not PYCLIP_AVAIL: print_info( "- To send the last response to the clipboard the Python module pyclip needs to be installed\n" @@ -841,7 +806,6 @@ def response_to_clipboard(last_response: str) -> None: def response_to_file( file_path: str, append_to_file: bool, add_last_command: bool, last_command: str, last_response: str ) -> None: - """# TODO""" try: mode = "a" if append_to_file else "w" with open(file_path, mode, encoding="utf-8") as response_out: @@ -859,7 +823,6 @@ def response_to_file( def response_reformat_json(last_response: str, color_json: bool) -> str: - """# TODO""" if not last_response.startswith("{"): print_warning("The last response isn't JSON") return "" @@ -874,8 +837,7 @@ def response_reformat_json(last_response: str, color_json: bool) -> str: def get_max_futures_workers() -> int: - """# TODO""" - # Test the max number of workers ThreadPoolExecutor allocates to use in sizing actual workers to request + """Test the max number of workers ThreadPoolExecutor allocates to use in sizing actual workers to request""" with concurrent.futures.ThreadPoolExecutor() as test: return test._max_workers # pylint: disable=protected-access @@ -886,7 +848,6 @@ def get_max_futures_workers() -> int: def human_readable_bytes(bytes_: int) -> str: - """# TODO""" if bytes_ == 0: return "0" @@ -906,9 +867,7 @@ def human_readable_bytes(bytes_: int) -> str: def case_combinations(strings: Iterable[str]) -> List[str]: - """# TODO""" combos = [] - try: for string in strings: combos.extend(list({"".join(sc) for sc in product(*zip(string.upper(), string.lower()))})) @@ -921,8 +880,6 @@ def case_combinations(strings: Iterable[str]) -> List[str]: def prompt_confirm(msg: str, confirm_values: Iterable[str] = ("y", "yes")) -> bool: - """# TODO""" - case_combos = case_combinations(confirm_values) response = input(msg).strip() if response not in case_combos: @@ -932,7 +889,6 @@ def prompt_confirm(msg: str, confirm_values: Iterable[str] = ("y", "yes")) -> bo def get_char() -> str: - """# TODO""" file_desc = sys.stdin.fileno() orig = termios.tcgetattr(file_desc) @@ -944,7 +900,6 @@ def get_char() -> str: def get_char_with_prompt(prompt, valid_responses=None): - """# TODO""" print(prompt, end="", flush=True) response = "" while True: @@ -970,8 +925,6 @@ def get_char_with_prompt(prompt, valid_responses=None): def get_char_with_timeout(time_out: int) -> str: - """# TODO""" - def handler(*_): # type: ignore[no-untyped-def] raise TimedOut diff --git a/sz_tools/sz_command b/sz_tools/sz_command index 828858a..c2f1b02 100755 --- a/sz_tools/sz_command +++ b/sz_tools/sz_command @@ -1,7 +1,5 @@ #! /usr/bin/env python3 -"""# TODO""" - import argparse import cmd import functools @@ -47,15 +45,14 @@ _WrappedFunc = TypeVar("_WrappedFunc", bound=Callable[..., Any]) MODULE_NAME = pathlib.Path(__file__).stem -# Per command settings -PER_CMD_SETTINGS = ["json", "jsonl", "color", "colour", "nocolor", "nocolour", "debug", "timer", "page"] +PER_CMD_SETTINGS = ["json", "jsonl", "color", "colour", "nocolor", "nocolour", "debug", "timer", "scroll"] DEFAULT_CONFIG = { "format_json": True, "color_output": True, "timer": False, "theme": "TERMINAL", - "page_output": False, + "scroll_output": False, "debug_sdk_call": False, "debug_sz_engine": False, "history_file": True, @@ -73,7 +70,7 @@ SETTINGS_TO_CONFIG_MAP = { "debug": ["debug_sdk_call", True], "debug_sdk": ["debug_sdk_call", True], "timer": ["timer", True], - "page": ["page_output", True], + "scroll": ["scroll_output", True], } # Don't overwrite last command if we still need to use it @@ -101,7 +98,7 @@ CONFIG_SETTINGS: Dict[str, Dict[str, Any]] = { "values": [c.lower() for c in Colors.AVAILABLE_THEMES], "description": "Set color scheme to use", }, - "page_output": { + "scroll_output": { "values": ["off", "on"], "description": "Use a pager for content larger than the terminal", }, @@ -128,7 +125,7 @@ CONFIG_SETTINGS: Dict[str, Dict[str, Any]] = { def do_methods_decorator(do_method: _WrappedFunc) -> _WrappedFunc: @functools.wraps(do_method) - def wrapper(self, *args, **kwargs) -> Any: + def wrapper(self, *args: Any, **kwargs: Any) -> Any: # Remove do_ from wrapped method sdk_method_name = do_method.__name__[3:] @@ -313,7 +310,7 @@ class SzCmdShell(cmd.Cmd): self.attrs = self.get_config_attr_codes() # Get Senzing engine flag names for use in auto completion - self.engine_flags_list = get_engine_flag_names() + self.engine_flags_list = get_engine_flag_names(["_SZ_WITHOUT_INFO"]) except SzError as err: print_error(err) sys.exit(1) @@ -568,7 +565,6 @@ class SzCmdShell(cmd.Cmd): set_theme_parser.add_argument("theme", choices=self.themes, nargs=1) def get_config_attr_codes(self) -> List[str]: - # TODO - Work in progress for JSON autocomplete try: config_id = self.sz_engine.get_active_config_id() sz_config = self.sz_configmgr.create_config_from_config_id(config_id) @@ -585,7 +581,7 @@ class SzCmdShell(cmd.Cmd): formatted_response: str = print_response( response, self.per_cmd_config["format_json"], # type: ignore[arg-type] - self.per_cmd_config["page_output"], # type: ignore[arg-type] + self.per_cmd_config["scroll_output"], # type: ignore[arg-type] self.per_cmd_config["color_output"], # type: ignore[arg-type] color=color, ) @@ -663,7 +659,7 @@ class SzCmdShell(cmd.Cmd): if self.debug_reinit["prior_instance_reinitialized"]: print_response("Senzing engines were reinitialized due to a configuration change", color="success") else: - print_info(f"Welcome to {MODULE_NAME}. Type help or ? for help", info_prefix=False) + print_info(f"Type help or ? for help", info_prefix=False) # ------------------------------------------------------------------------- # Non-interactive input methods @@ -784,7 +780,7 @@ class SzCmdShell(cmd.Cmd): - nocolor / nocolour - debug - timer - - page + - scroll {colorize_str('- Examples:', 'dim')} - get_entity_by_entity_id 1001 jsonl @@ -848,21 +844,21 @@ class SzCmdShell(cmd.Cmd): Notes: - Retrieve the active configuration identifier with get_active_config_id - - Retrieve a list of configurations and identifiers with get_configs""" + - Retrieve a list of configurations and identifiers with get_config_registry""" sz_config = self.sz_configmgr.create_config_from_config_id(kwargs["parsed_args"].config_id) response = sz_config.export() self.last_response = self.output_response(response) @do_methods_decorator - def do_get_configs(self) -> None: + def do_get_config_registry(self) -> None: """ - Get a list of current configurations + Get details of current registered configurations Syntax: - get_configs""" + get_config_registry""" - response = self.sz_configmgr.get_configs() + response = self.sz_configmgr.get_config_registry() self.last_response = self.output_response(response) @do_methods_decorator @@ -892,7 +888,7 @@ class SzCmdShell(cmd.Cmd): new_default_config_id = Configuration identifier Notes: - - Retrieve a list of configurations and identifiers with get_configs""" + - Retrieve a list of configurations and identifiers with get_config_registry""" self.sz_configmgr.replace_default_config_id( kwargs["parsed_args"].current_default_config_id, @@ -2300,7 +2296,7 @@ class SzCmdShell(cmd.Cmd): return kwargs["cmd_settings"] @cmd_settings_decorator - def complete_get_configs(self, text: str, line: str, begidx: int, endidx: int, **kwargs) -> List[str]: + def complete_get_config_registry(self, text: str, line: str, begidx: int, endidx: int, **kwargs) -> List[str]: return kwargs["cmd_settings"] @cmd_settings_decorator diff --git a/sz_tools/sz_configtool b/sz_tools/sz_configtool index 6294676..36ad78b 100755 --- a/sz_tools/sz_configtool +++ b/sz_tools/sz_configtool @@ -40,7 +40,6 @@ MODULE_NAME = pathlib.Path(__file__).stem def parse_cli_args() -> argparse.Namespace: - """# TODO""" arg_parser = argparse.ArgumentParser( allow_abbrev=False, description="Utility to view and manipulate the Senzing configuration", @@ -627,7 +626,7 @@ class SzCfgShell(cmd.Cmd): def preloop(self): self.load_config() colorize_msg( - "Welcome to the Senzing configuration tool. Type help or ? for help", + "Type help or ? for help", "highlight2", ) @@ -755,7 +754,7 @@ class SzCfgShell(cmd.Cmd): except SzError as err: colorize_msg(err, "error") - def do_getConfigList(self, arg): + def do_getConfigRegistry(self, arg): """ Returns the list of all known configurations @@ -764,7 +763,7 @@ class SzCfgShell(cmd.Cmd): """ arg = self.check_arg_for_output_format("record", arg) try: - config_list = self.sz_configmgr.get_configs() + config_list = self.sz_configmgr.get_config_registry() self.print_json_record(json.loads(config_list)["CONFIGS"]) except SzError as err: colorize_msg(err, "error") @@ -1256,7 +1255,6 @@ class SzCfgShell(cmd.Cmd): self.print_json_lines(json_lines) - # TODO auto complete doesn't work if there is a - in dsrc def do_deleteDataSource(self, arg): """ Delete an existing data source diff --git a/sz_tools/sz_create_project b/sz_tools/sz_create_project index 748ccfc..9e3baf4 100755 --- a/sz_tools/sz_create_project +++ b/sz_tools/sz_create_project @@ -1,216 +1,103 @@ #! /usr/bin/env python3 - -"""# TODO""" +"""Create a Senzing SDK project""" import argparse -import json import sys from pathlib import Path -from shutil import copyfile, copytree, ignore_patterns -from typing import List, Union +from _project_helpers import ( + COPY_TO_PROJ, + PERMISSIONS, + PERMISSIONS_2, + SZ_SYS_PATH, + V4_SYS_BUILD, + copy_files_dirs, + get_build_details, + set_permissions, + setup_env, +) -def parse_cli_args() -> argparse.Namespace: - """# TODO""" +COPY_TO_ETC = {"er/resources/templates/": {"files": ["*"], "excludes": ["G2C.*", "setupEnv", "g2config.json"]}} +COPY_TO_VAR: dict[str, dict[str, list[str]]] = {"er/resources/templates/G2C.db": {"files": [], "excludes": []}} + +def parse_cli_args() -> argparse.Namespace: + """Parse the CLI arguments""" arg_parser = argparse.ArgumentParser( allow_abbrev=False, - description="Utility to create a new instance of a Senzing project in a path", + description=" Create a new instance of a Senzing project", formatter_class=argparse.RawTextHelpFormatter, ) arg_parser.add_argument( - "path", - help="path to create new Senzing project in, it must not already exist", - metavar="PATH", + "project_path", + metavar="path", + help="path to create new Senzing project in, it must not exist", ) return arg_parser.parse_args() -def get_version_details(sz_root_path: Path) -> List[str]: - """Return version details of Senzing installation""" - try: - sz_root_path = sz_root_path.joinpath("szBuildVersion.json") - with open(sz_root_path, encoding="utf-8") as file_version: - version_details = json.load(file_version) - except IOError as err: - print(f"\nERROR: Unable to read {sz_root_path} to retrieve version details - {err}") +def pre_check(project_path: Path) -> None: + """Check not trying to overwrite the V4 Senzing system install and the path doesn't exist""" + if str(project_path).startswith(str(SZ_SYS_PATH)): + print(f"\nProject cannot be created in {SZ_SYS_PATH}") sys.exit(1) - except json.JSONDecodeError as err: - print(f"\nERROR: {err}") - details: List[str] = [] - details.append(version_details.get("BUILD_VERSION", "")) - - if not all(details): - print(f"\nERROR: Problem reading values from version details, missing value(s) - {details}") + if project_path.exists(): + print(f"\n{project_path} exists, specify a different path") sys.exit(1) - return details - - -def replace_in_file(filename: Path, old_string: str, new_string: str) -> None: - """Replace strings in new project files""" +def update_sz_engine_config(config_file: Path, project_path: Path) -> None: + """Update sz_engine_config.ini with project paths""" try: - with open(filename, encoding="utf-8") as fr: - data = fr.read() - with open(filename, "w", encoding="utf-8") as fw: - fw.write(data.replace(old_string, new_string)) - except IOError as err: - raise err - - -def set_folder_permissions(path: Path, permissions: int, folders_to_ignore: Union[List[str], None] = None) -> None: - """Set permissions recursively on a folder, optionally ignore specific folders""" - if folders_to_ignore is None: - folders_to_ignore = [] - - path.chmod(permissions) - - dirs: List[Path] = [d for d in path.rglob("*") if d.is_dir() and not d.is_symlink() and d not in folders_to_ignore] - for dir_ in dirs: - dir_.chmod(permissions) - - -def set_file_permissions( - path: Path, - permissions: int, - files_to_ignore: Union[List[str], None] = None, - recursive: bool = False, -) -> None: - """Set permissions on files in a folder, optionally do recursively""" - if files_to_ignore is None: - files_to_ignore = [] - - files: List[Path] = [] - if recursive: - files = [f for f in path.rglob("*") if f.is_file() and f not in files_to_ignore] - else: - files = [f for f in path.iterdir() if f.is_file() and f not in files_to_ignore] + with open(config_file, "r", encoding="utf-8") as in_: + data = in_.read() + + data = ( + data.replace("${SENZING_DATA_DIR}", str(project_path / "data")) + .replace("${SENZING_CONFIG_PATH}", str(project_path / "etc")) + .replace("${SENZING_RESOURCES_DIR}", str(project_path / "resources")) + .replace("${SENZING_VAR_DIR}", str(project_path / "var")) + ) - for file in files: - file.chmod(permissions) + with open(config_file, "w", encoding="utf-8") as out: + out.write(data) + except OSError as err: + raise OSError(f"Couldn't update new {config_file}: {err}") from err def main() -> None: """main""" - cli_args = parse_cli_args() + proj_path = Path(cli_args.project_path).expanduser().resolve() + pre_check(proj_path) - # sz_path on normal rpm/deb install = /opt/senzing/g2 - # sz_install_root would then = /opt/senzing - # TODO Put back when in API package - sz_path = Path(__file__).resolve().parents[1] - # sz_path = Path("/opt/senzing/er").resolve() - sz_path_root = Path(__file__).resolve().parents[2] - project_path = Path(cli_args.path).expanduser().resolve() - - bin_path = project_path.joinpath("bin") - data_path = project_path.joinpath("data") - etc_path = project_path.joinpath("etc") - lib_path = project_path.joinpath("lib") - resources_path = project_path.joinpath("resources") - sdk_path = project_path.joinpath("sdk") - var_path = project_path.joinpath("var") - - if project_path.exists() and project_path.samefile(sz_path_root): - print(f"\nProject cannot be created in {sz_path_root}. Please specify a different path.") - sys.exit(1) - - if project_path.exists(): - print(f"\n{project_path} exists, please specify a different path.") - sys.exit(1) - - version_details = get_version_details(sz_path) - print(f"\nSenzing version: {version_details[0]}\n") - - ignore_files = ["sz_create_project"] - # Example: ignore_paths = [sz_path.joinpath('python')] - ignore_paths: List[str] = [] - excludes = ignore_files + ignore_paths - - # Copy sz_path to new project path - copytree(sz_path, project_path, ignore=ignore_patterns(*excludes), symlinks=True) - - # Copy resources/templates to etc - ignore_files = ["G2C.db", "setupEnv", "*.template", "g2config.json"] - copytree( - sz_path.joinpath("resources", "templates"), - etc_path, - ignore=ignore_patterns(*ignore_files), - ) - - # Copy setupEnv - copyfile( - sz_path.joinpath("resources", "templates", "setupEnv"), - project_path.joinpath("setupEnv"), - ) + try: + print(f"\nSenzing version: {get_build_details(V4_SYS_BUILD).version}\n") - # Copy G2C.db to runtime location - Path.mkdir(project_path.joinpath("var", "sqlite"), parents=True) - copyfile( - sz_path.joinpath("resources", "templates", "G2C.db"), - var_path.joinpath("sqlite", "G2C.db"), - ) + # Create project and copy main files + copy_files_dirs(COPY_TO_PROJ, SZ_SYS_PATH, proj_path) - # Copy data - copytree( - sz_path_root.joinpath("data"), - data_path, - ignore=ignore_patterns(*excludes), - symlinks=True, - ) + # Create and copy to proj_path/etc + copy_files_dirs(COPY_TO_ETC, SZ_SYS_PATH, proj_path / "etc") - # Files & strings to modify - update_files = [ - project_path.joinpath("setupEnv"), - etc_path.joinpath("sz_engine_config.ini"), - ] - - path_subs = [ - ("${SENZING_DIR}", project_path), - ("${SENZING_CONFIG_PATH}", etc_path), - ("${SENZING_DATA_DIR}", data_path), - ("${SENZING_RESOURCES_DIR}", resources_path), - ("${SENZING_VAR_DIR}", var_path), - ] - - for file in update_files: - for path in path_subs: - replace_in_file(file, path[0], str(path[1])) - - # Folder permissions - set_folder_permissions(project_path, 0o770) - - # Root of project - set_file_permissions(project_path, 0o660) - project_path.joinpath("setupEnv").chmod(0o770) - - # bin - set_file_permissions(bin_path, 0o770, recursive=True) - - # etc - set_file_permissions(etc_path, 0o660) - - # lib - set_file_permissions( - lib_path, - 0o660, - files_to_ignore=["g2.jar"], - ) + # Copy and modify setupEnv + setup_env(proj_path) - # resources - set_file_permissions(resources_path, 0o660, recursive=True) - resources_path.joinpath("templates", "setupEnv").chmod(0o770) + # Create proj_path/var/sqlite and copy G2C.db + copy_files_dirs(COPY_TO_VAR, SZ_SYS_PATH, proj_path / "var" / "sqlite") - # sdk - set_file_permissions(sdk_path, 0o664, recursive=True) + # Modify sz_engine_config.ini + update_sz_engine_config(proj_path / "etc" / "sz_engine_config.ini", proj_path) - # var - set_file_permissions(var_path, 0o660, recursive=True) - - print("Successfully created.") + # Set permissions on the project + set_permissions(proj_path, PERMISSIONS) + set_permissions(proj_path, PERMISSIONS_2) + except OSError as err: + print(f"\nERROR: {err}") + else: + print("Successfully created") if __name__ == "__main__": diff --git a/sz_tools/sz_explorer b/sz_tools/sz_explorer index 3b9488c..d60bc9c 100755 --- a/sz_tools/sz_explorer +++ b/sz_tools/sz_explorer @@ -3027,7 +3027,6 @@ class EdaCmd(cmd.Cmd): f"\nSenzing Support Request: {colorize('https://senzing.zendesk.com/hc/en-us/requests/new', 'highlight2, underline')}\n" ) - # TODO - Ant - Use from helpers def histCheck(self): file_name = f".{MODULE_NAME}_history" @@ -3067,7 +3066,6 @@ class EdaCmd(cmd.Cmd): self.histFileError = None self.histAvail = True - # TODO - Ant - Use from helpers def do_history(self, arg): if self.histAvail: @@ -3078,7 +3076,6 @@ class EdaCmd(cmd.Cmd): else: print_message("History isn't available in this session", "warning") - # TODO - Ant - Use from helpers def do_shell(self, line): """\nRun OS shell commands: !\n""" if line: diff --git a/sz_tools/sz_export b/sz_tools/sz_export index 3a5ba3a..4f8c5ad 100755 --- a/sz_tools/sz_export +++ b/sz_tools/sz_export @@ -257,9 +257,9 @@ if __name__ == "__main__": help=textwrap.dedent( """\ - Path and file name to send output to. + Path and file name to send output to. - """ + """ ), ) @@ -272,9 +272,9 @@ if __name__ == "__main__": help=textwrap.dedent( """\ - Path and file name of optional sz_engine_config.ini to use. + Path and file name of optional sz_engine_config.ini to use. - """ + """ ), ) @@ -286,16 +286,11 @@ if __name__ == "__main__": help=textwrap.dedent( """\ - Space separated list of export flags to apply to the export. - - Valid flags: + Space separated list of export flags to apply to the export. - ... - ... + Default: %(default)s - Default: %(default)s - - """ + """ ), ) @@ -308,11 +303,11 @@ if __name__ == "__main__": help=textwrap.dedent( """\ - Data format to export to, JSON or CSV. + Data format to export to, JSON or CSV. - Default: %(default)s + Default: %(default)s - """ + """ ), ) @@ -324,20 +319,20 @@ if __name__ == "__main__": help=textwrap.dedent( """\ - Return extended details, adds RESOLVED_ENTITY_NAME & JSON_DATA. + Return extended details, adds RESOLVED_ENTITY_NAME & JSON_DATA. - Adding JSON_DATA significantly increases the size of the output and execution time. + Adding JSON_DATA significantly increases the size of the output and execution time. - When used with CSV output, JSON_DATA isn\'t included for the related entities - (RELATED_ENTITY_ID) for each resolved entity (RESOLVED_ENTITY_ID). This reduces - the size of a CSV export by preventing repeating data for related entities. JSON_DATA - for the related entities is still included in the CSV export and is located in the - export record where the RELATED_ENTITY_ID = RESOLVED_ENTITY_ID. + When used with CSV output, JSON_DATA isn\'t included for the related entities + (RELATED_ENTITY_ID) for each resolved entity (RESOLVED_ENTITY_ID). This reduces + the size of a CSV export by preventing repeating data for related entities. JSON_DATA + for the related entities is still included in the CSV export and is located in the + export record where the RELATED_ENTITY_ID = RESOLVED_ENTITY_ID. - WARNING: This is not recommended! To include the JSON_DATA for every CSV record see the - --extendCSVRelates (-xcr) argument. + WARNING: This is not recommended! To include the JSON_DATA for every CSV record see the + --extendCSVRelates (-xcr) argument. - """ + """ ), ) @@ -349,11 +344,11 @@ if __name__ == "__main__": help=textwrap.dedent( """\ - Frequency of export output statistics. + Frequency of export output statistics. - Default: %(default)s + Default: %(default)s - """ + """ ), ) @@ -368,14 +363,14 @@ if __name__ == "__main__": help=textwrap.dedent( f"""\ - Compress output file with gzip. Compression level can be optionally specified. + Compress output file with gzip. Compression level can be optionally specified. - If output file is specified as - (for stdout), use shell redirection instead to compress: - {MODULE_NAME} -o - | gzip -v > export.csv.gz + If output file is specified as - (for stdout), use shell redirection instead to compress: + {MODULE_NAME} -o - | gzip -v > export.csv.gz - Default: %(const)s + Default: %(const)s - """ + """ ), ) @@ -387,13 +382,13 @@ if __name__ == "__main__": help=textwrap.dedent( """\ - WARNING: Use of this argument is not recommended! + WARNING: Use of this argument is not recommended! - Used in addition to --extend (-x), it will include JSON_DATA in CSV output for related entities. + Used in addition to --extend (-x), it will include JSON_DATA in CSV output for related entities. - Only valid for CSV output format. + Only valid for CSV output format. - """ + """ ), ) @@ -489,9 +484,6 @@ if __name__ == "__main__": print(", ".join(invalid_string_flags)) valid_flags = [flag for flag in flags if flag not in invalid_string_flags] - - # TODO - # final_flags = SzEngineFlags.combine_flags(valid_flags) final_flags = combine_engine_flags(valid_flags) # Initialize the export diff --git a/sz_tools/sz_file_loader b/sz_tools/sz_file_loader index 551c970..7ec699d 100755 --- a/sz_tools/sz_file_loader +++ b/sz_tools/sz_file_loader @@ -123,7 +123,7 @@ shutdown = Event() def files_clean_up(errors_file: Path, with_info_file: Path) -> None: - """TODO""" + """Remove error and with_info files if they weren't used""" if check_file_exists(errors_file) and errors_file.stat().st_size == 0: errors_file.unlink() @@ -132,7 +132,7 @@ def files_clean_up(errors_file: Path, with_info_file: Path) -> None: def parse_cli_args() -> argparse.Namespace: - """TODO""" + """Parse command line arguments""" arg_parser = argparse.ArgumentParser( allow_abbrev=False, description="Utility to load Senzing mapped JSON records and process redo records", @@ -374,7 +374,7 @@ def parse_cli_args() -> argparse.Namespace: def check_ingest_files(files: list[str]): - """TODO""" + """Basic test files to load to catch early JSON issues""" for file in files: json_errors = 0 json_good = 0 @@ -392,19 +392,6 @@ def check_ingest_files(files: list[str]): json_good += 1 except JSONDecodeError: json_errors += 1 - # TODO - # try: - # csv_sample = file_.read(5000) - # csv_header = csv.Sniffer().has_header(csv_sample) - # csv_dialect = csv.Sniffer().sniff(csv_sample) - # print(f"\n{csv_header = }") - # print(f"\n{csv_dialect.__dict__ = }") - # reader = csv.reader(file_, csv_dialect) - # for _ in range(500): - # row = next(reader) - # print(row) - # except Exception as err: - # print(f"{err = }") except OSError as err: logger.info("") logger.error(err) @@ -419,12 +406,8 @@ def check_ingest_files(files: list[str]): def docker_redirects( files_list: list[str], redirects, - # redirect_path: str, - # errors_path: str, - # withinfo_path: str, - # shuffle_path: str, ) -> tuple[Any, Any, Any]: - """TODO""" + """Modify paths in docker""" errors_path = redirects.errors_path withinfo_path = redirects.withinfo_path shuffle_path = redirects.shuffle_path @@ -464,7 +447,7 @@ def docker_redirects( # Have a file but not another redirect if files_list and not any(redirects): - file_parent = str(Path(files_list[0]).parent.resolve()) + file_parent = str(Path(files_list[0]).parent.expanduser().resolve()) logger.info( "Setting missing error, with info, shuffled files output paths to ingest file path, any of -ep, -wp, -sp, -rp not specified" ) @@ -480,8 +463,8 @@ def docker_redirects( return (errors_path, withinfo_path, shuffle_path) -def check_redirect_paths(paths: Iterable[str]): - """TODO""" +def check_redirect_paths(paths: Iterable[str]) -> None: + """Check redirect paths are writable""" try: for p in paths: if not p: @@ -553,9 +536,6 @@ def shuffle_ingest_file( logger.info("Shuffling to: %s", shuff_file) shuf_cmd = "gshuf" if sys.platform == "darwin" else "shuf" cmd = f"{shuf_cmd} {ingest_file} > {shuff_file}" - # TODO - # if sourceDict["FILE_FORMAT"] not in ("JSON", "UMF"): - # cmd = f"head -n1 {file_path} > {shuf_file_path} && tail -n+2 {file_path} | shuf >> {shuf_file_path}" try: _ = subprocess.run([cmd], capture_output=True, check=True, shell=True) @@ -578,7 +558,7 @@ def shuffle_ingest_file( def get_sz_engines( sz_factory: SzAbstractFactoryCore, ) -> Tuple[SzEngine, SzDiagnostic, SzProduct, SzConfigManager]: - """TODO""" + """Create required engines""" try: sz_engine = sz_factory.create_engine() sz_diag = sz_factory.create_diagnostic() @@ -591,7 +571,7 @@ def get_sz_engines( def prime_sz_engine(sz_engine: SzEngine) -> None: - """TODO""" + """Prime sz_engine""" logger.info("") logger.info("Priming Senzing engine...") try: @@ -624,7 +604,7 @@ def startup_info( try: lic_info = _json_loads(product.get_license()) ver_info = _json_loads(product.get_version()) - config_list = _json_loads(configmgr.get_configs()) + config_list = _json_loads(configmgr.get_config_registry()) active_cfg_id = engine.get_active_config_id() ds_info = _json_loads(diag.get_datastore_info()) except SzError as err: @@ -913,19 +893,6 @@ def load_and_redo( for f in done: try: result = f.result() - # TODO - Test on OS & SG load with redo when engine fixed for process_redo_record() - # TODO - Collect and retry later? - # # If caught a retryable error resubmit the record to try again - # except SzRetryableError as err: - # logger.info("") - # logger.info( - # "Retrying record due to: %s - Operation: %s - Record: %s", - # err, - # mode_text[mode.__name__]["except_msg"], - # futures[f][0].strip(), - # ) - # logger.info("") - # more_recs = add_new_future(futures[f][0].strip()) except ( SzError, JSONDecodeError, @@ -963,7 +930,7 @@ def load_and_redo( finally: if add_future and not shutdown.is_set(): more_recs = add_new_future() - + del futures[f] # Early errors check to catch mapping errors, missing dsrc_code, etc @@ -1027,7 +994,7 @@ def load_and_redo( def per_result(cli_args: argparse.Namespace, result: dict[str, Any]) -> None: - """TODO""" + """Results for each ingested file""" logger.info("") logger.info("Results") logger.info("-------") @@ -1077,7 +1044,7 @@ def summary_results( cli_args: argparse.Namespace, overall_results: dict[str, dict[str, Any]], ) -> None: - """TODO""" + """Overall results""" elapsed_time_total = 0 load_blank_lines_total = 0 load_error_total = 0 diff --git a/sz_tools/sz_json_analyzer b/sz_tools/sz_json_analyzer index ea34766..db02109 100755 --- a/sz_tools/sz_json_analyzer +++ b/sz_tools/sz_json_analyzer @@ -26,8 +26,6 @@ MODULE_NAME = pathlib.Path(__file__).stem class JsonlReader: - """#TODO""" - def __init__(self, file_handle): self.file_handle = file_handle @@ -39,7 +37,6 @@ class JsonlReader: def get_environment_config(settings: str) -> Dict[str, Any]: - """# TODO""" try: sz_factory = SzAbstractFactoryCore(MODULE_NAME, settings) sz_configmgr = sz_factory.create_configmanager() @@ -54,7 +51,6 @@ def get_environment_config(settings: str) -> Dict[str, Any]: def get_file_config(file: str) -> Dict[str, Any]: - """#TODO""" try: with open(file, "r", encoding="utf-8") as config_file: config_json: Dict[str, Any] = json.loads(config_file.read()) @@ -65,8 +61,6 @@ def get_file_config(file: str) -> Dict[str, Any]: class SzJsonAnalyzer: - """#TODO""" - def __init__(self, config_data: Dict[str, Any]): self.attribute_lookup = {} @@ -107,11 +101,10 @@ class SzJsonAnalyzer: for record in config_data["G2_CONFIG"]["CFG_FTYPE"]: self.feature_lookup[record["FTYPE_CODE"]] = record - # TODO hack until 4.0 to move record_type higher + # NOTE - hack until 4.0 to move record_type higher self.feature_order["RECORD_TYPE"] = 1004 def register_attribute(self, attr_name: str) -> None: - """#TODO""" attr_data = {} if attr_name in self.attribute_lookup: attr_data = self.attribute_lookup[attr_name] @@ -143,8 +136,6 @@ class SzJsonAnalyzer: attr_name: str, attr_value: str, ) -> None: - """#TODO""" - if isinstance(attr_value, (list, dict)): errors.append(f"Expected integer or string for {attr_name}") else: @@ -157,7 +148,6 @@ class SzJsonAnalyzer: features[feature_key].append(attr_data) def update_feature_stats(self, feature: str, attribute: str, value: str) -> None: - """#TODO""" if attribute in self.feature_stats[feature]["attributes"]: self.feature_stats[feature]["attributes"][attribute]["count"] += 1 else: @@ -174,7 +164,6 @@ class SzJsonAnalyzer: self.feature_stats[feature]["attributes"][attribute]["values"][value] = 1 def update_unmapped_stats(self, attr_name: str, attr_value: str) -> None: - """#TODO""" if attr_name in self.unmapped_stats: self.unmapped_stats[attr_name]["count"] += 1 else: @@ -185,7 +174,6 @@ class SzJsonAnalyzer: self.unmapped_stats[attr_name]["values"][attr_value] = 1 def update_message_stats(self, cat: str, stat: str, row_num: Union[int, str] = "n/a") -> None: - """#TODO""" row_num = f"row {row_num}" if isinstance(row_num, int) else row_num if stat not in self.message_stats[cat]: self.message_stats[cat][stat] = {"count": 1, "rows": [row_num]} @@ -195,7 +183,6 @@ class SzJsonAnalyzer: self.message_stats[cat][stat]["rows"].append(row_num) def analyze_json(self, input_data: Dict[str, Any], input_row_num: Union[int, None]) -> None: - """#TODO""" self.record_count += 1 # print('-'*50) @@ -397,7 +384,6 @@ class SzJsonAnalyzer: self.update_message_stats(message[0], message[1], input_row_num) def get_report(self) -> List[List[Union[int, str]]]: - """#TODO""" table_headers = [ "Category", "Attribute", @@ -558,7 +544,6 @@ class SzJsonAnalyzer: # ---------------------------------------- def format_pretty_table(table_rows): - """#TODO""" table_object = prettytable.PrettyTable() table_object.horizontal_char = "\u2500" table_object.vertical_char = "\u2502" @@ -631,7 +616,6 @@ def format_pretty_table(table_rows): def format_csv_table(table_rows): - """#TODO""" output = io.StringIO() writer = csv.writer(output) writer.writerows(table_rows) @@ -639,7 +623,6 @@ def format_csv_table(table_rows): def print_report(report_string: str): - """#TODO""" less = subprocess.Popen(["less", "-FMXSR"], stdin=subprocess.PIPE) try: less.stdin.write(report_string.encode("utf-8")) diff --git a/sz_tools/sz_setup_config b/sz_tools/sz_setup_config index f6459dd..79bbcf0 100755 --- a/sz_tools/sz_setup_config +++ b/sz_tools/sz_setup_config @@ -1,6 +1,4 @@ #! /usr/bin/env python3 -"""# TODO""" - import argparse import pathlib import sys @@ -13,7 +11,6 @@ MODULE_NAME = pathlib.Path(__file__).stem def parse_cli_args() -> argparse.Namespace: - """# TODO""" arg_parser = argparse.ArgumentParser() arg_parser.add_argument( "-c", @@ -29,7 +26,7 @@ def parse_cli_args() -> argparse.Namespace: def main() -> None: - """# TODO""" + """main""" cli_args = parse_cli_args() # Check an engine configuration can be located diff --git a/sz_tools/sz_update_project b/sz_tools/sz_update_project index bba8813..1aaa47b 100755 --- a/sz_tools/sz_update_project +++ b/sz_tools/sz_update_project @@ -1,8 +1,7 @@ #! /usr/bin/env python3 -"""Upgrade a V3 or V4 Senzing SDK project""" +"""Upgrade a V3 or V4 Senzing SDK project to V4.x.x""" import argparse -import json import shutil import sys from contextlib import suppress @@ -10,33 +9,66 @@ from pathlib import Path from time import sleep from typing import Any -INPUT_CONFS = ("y", "Y", "yes", "YES") -SZ_SYS_PATH = Path("/opt/senzing/er") +from _project_helpers import ( + PERMISSIONS, + PERMISSIONS_2, + SZ_SYS_PATH, + V3_BACKUP_PATH, + V4_BUILD, + V4_SYS_BUILD, + V4_SYS_PATH, + SzBuildDetails, + get_build_details, + set_permissions, + setup_env, +) + +INPUT_CONFS = ("y", "Y", "yEs", "yES", "YES", "yes", "YEs", "yeS", "Yes", "YeS") MODULE_NAME = Path(__file__).stem -PROJ_BUILD = "g2BuildVersion.json" -SYS_BUILD = SZ_SYS_PATH / "szBuildVersion.json" -V3_BACKUP_PATH = "v3_upgrade_backups" +V3_BUILD = "g2BuildVersion.json" +V3_SYS_PATH = SZ_SYS_PATH / "g2" +V3_SYS_BUILD = V3_SYS_PATH / V3_BUILD UPGRADE_URL = "https://www.senzing.com/docs/4_beta/index.html" V3_BACKUP_PROJ = { "bin": {"files": ["g2dbencrypt", "g2saltadm", "g2ssadm"], "excludes": []}, "etc": {"files": ["senzing_governor.py"], "excludes": []}, "lib": {"files": ["libSpaceTimeBoxStandardizer.so", "libG2Hasher.so", "libG2SSAdm.so"], "excludes": []}, - "python": {"files": [], "excludes": []}, - "resources/templates": {"files": ["setupEnv"], "excludes": []}, + "python": { + "files": ["*"], + "excludes": [ + "CompressedFile.py", + "DumpStack.py", + "G2Audit.py", + "G2Command.py", + "G2ConfigTables.py", + "G2ConfigTool.py", + "G2Database.py", + "G2Explorer.py", + "G2Export.py", + "G2IniParams.py", + "G2Loader.py", + "G2Paths.py", + "G2Project.py", + "G2S3.py", + "G2SetupConfig.py", + "G2Snapshot.py", + "SenzingGo.py", + "senzing", + ], + }, "setupEnv": {"files": [], "excludes": []}, "g2BuildVersion.json": {"files": [], "excludes": []}, } V3_REMOVE_FROM_PROJ = { - "bin": {"files": ["*"], "excludes": ["bin"]}, + "bin": {"files": ["g2configupgrade", "g2dbencrypt", "g2dbupgrade", "g2saltadm", "g2ssadm"], "excludes": ["bin"]}, "data": {"files": [], "excludes": []}, "etc": {"files": ["senzing_governor.py"], "excludes": []}, - # "g2BuildVersion.json": {"files": [], "excludes": []}, "lib": { "files": [ "g2.jar", - "libG2.so", + "libG2*.so", "libG2Hasher.so", "libG2SSAdm.so", "libg2CompJavaScoreSet.so", @@ -56,6 +88,7 @@ V3_REMOVE_FROM_PROJ = { "files": [ "G2C.db*", "G2Module.ini", + "cfgVariant.json", "custom*.txt", "defaultGNRCP.config", "g2config.json", @@ -65,70 +98,32 @@ V3_REMOVE_FROM_PROJ = { ], "excludes": [], }, - "sdk": {"files": ["*"], "excludes": ["sdk"]}, + "sdk": {"files": ["*"], "excludes": []}, "setupEnv": {"files": [], "excludes": []}, } -V3_COPY_TO_PROJ = { - "LICENSE": {"files": [], "excludes": []}, - "NOTICES": {"files": [], "excludes": []}, - "README.1ST": {"files": [], "excludes": []}, - "bin": {"files": ["*"], "excludes": ["sz_create_project", "sz_update_project"]}, - "/opt/senzing/data": {"files": ["*"], "excludes": []}, - "lib": {"files": ["*"], "excludes": []}, - "resources": {"files": ["*"], "excludes": []}, - "sdk": {"files": ["*"], "excludes": []}, - "szBuildVersion.json": {"files": [], "excludes": []}, +COPY_TO_PROJ = { + "er/": {"files": ["*"], "excludes": ["sz_create_project", "sz_update_project"]}, + "data": {"files": ["*"], "excludes": []}, } - V3_RENAME_IN_PROJ = { "etc": {"from": "G2Module.ini", "to": "sz_engine_config.ini"}, } - -V3_RESET_PERMISSIONS = { - ".": { - "dir_pint": 0, - "file_pint": 0o660, - "files": ["LICENSE", "NOTICES", "README.1ST", "szBuildVersion.json"], - "excludes": ["setupEnv"], - "recursive": False, - }, - "setupEnv": {"dir_pint": 0, "file_pint": 0o770, "files": [], "excludes": [], "recursive": False}, - "bin": { - "dir_pint": 0o770, - "file_pint": 0o770, - "files": ["*"], - "excludes": ["__pycache__", "_sz_database.py", "_tool_helpers.py"], - "recursive": False, - }, - "data": {"dir_pint": 0o770, "file_pint": 0o660, "files": ["*"], "excludes": [], "recursive": True}, - "lib": {"dir_pint": 0o770, "file_pint": 0o660, "files": ["*"], "excludes": [], "recursive": False}, - "resources": {"dir_pint": 0o770, "file_pint": 0o660, "files": ["*"], "excludes": ["setupEnv"], "recursive": True}, - "resources/templates/setupEnv": { - "dir_pint": 0, - "file_pint": 0o770, - "files": [], - "excludes": [], - "recursive": False, - }, - "sdk": {"dir_pint": 0o770, "file_pint": 0o660, "files": ["*"], "excludes": [], "recursive": True}, - "v3_upgrade_backups": {"dir_pint": 0o770, "file_pint": 0, "files": [], "excludes": [], "recursive": False}, -} - # pylint: disable=W0106 def parse_cli_args() -> argparse.Namespace: """Parse the CLI arguments""" arg_parser = argparse.ArgumentParser( - description="Update an existing Senzing project to the system installed version of Senzing." + allow_abbrev=False, + description="Update an existing V3 or V4 Senzing project to the system installed V4 of Senzing.", ) arg_parser.add_argument( "project_path", metavar="path", - help="path of the project to update", + help="Path of the project to update", ) arg_parser.add_argument( "-f", @@ -136,7 +131,7 @@ def parse_cli_args() -> argparse.Namespace: dest="force_mode", default=False, action="store_true", - help="upgrade without prompts. WARNING: Use with caution, ensure you have a backup of the project", + help="Upgrade without prompts. WARNING: Use with caution, ensure you have a backup of the project", ) return arg_parser.parse_args() @@ -154,53 +149,53 @@ def dir_listing(path: Path) -> list[Path]: return listing -def pre_check(path: Path, proj_build_file: Path, sys_build_file: Path, listing: list[Path]) -> tuple[str, str]: - """Check not trying to overwrite the Senzing system install, that path is a project and versions are correct""" - if not SZ_SYS_PATH.exists(): - print(f"\nERROR: Couldn't locate Senzing SDK system install at {SZ_SYS_PATH}") +def pre_check(project_path: Path) -> tuple[SzBuildDetails, SzBuildDetails]: + """Check not trying to overwrite the V4 Senzing system install, that path is a project, and versions are correct""" + if not V4_SYS_PATH.exists(): + print(f"\nERROR: Couldn't locate Senzing SDK V4 system install at {V4_SYS_PATH}") sys.exit(1) - if path.samefile(SZ_SYS_PATH): - print(f"\nERROR: {path} is the Senzing system installation path and not a Senzing project") + if str(project_path).startswith(str(SZ_SYS_PATH)): + print(f"\nERROR: {project_path} is the Senzing system installation, not a Senzing project") sys.exit(1) - if proj_build_file not in listing: - print(f"\nERROR: {path} isn't a Senzing project, expected it to contain the file {proj_build_file.name}") + v3_project_build_file = project_path / V3_BUILD + v4_project_build_file = project_path / V4_BUILD + proj_listing = dir_listing(project_path) + if v3_project_build_file not in proj_listing and v4_project_build_file not in proj_listing: + print(f"\nERROR: {project_path} isn't a Senzing project, expected it to contain either:") + print(f"\tExisting V3 project: {v3_project_build_file}") + print(f"\tExisting V4 project: {v4_project_build_file}") sys.exit(1) try: - proj_version = get_build_version(proj_build_file) - sys_version = get_build_version(sys_build_file) - except OSError as err: - print(f"\nERROR: Trying to read {proj_build_file} and {sys_build_file}: {err}") + proj_build_file = v3_project_build_file if v3_project_build_file in proj_listing else v4_project_build_file + proj_build_details = get_build_details(proj_build_file) + sys_build_details = get_build_details(V4_SYS_BUILD) + except (OSError, TypeError) as err: + print(f"\nERROR: Trying to read {proj_build_file} or {V4_SYS_BUILD} to collect version information: {err}") sys.exit(1) - if int(proj_version[:1]) not in (3, 4) or sys_version[:1] != "4": - print( - f"\nERROR: {MODULE_NAME} updates from V3, or V4.x to V4.y, project version: {proj_version} installed Senzing SDK version: {sys_version}" - ) + if proj_build_details.major not in (3, 4) or sys_build_details.major != 4: + print(f"\nERROR: {MODULE_NAME} updates a V3 project to V4, or V4.n.n to a newer release") + print(f"\tProject version: {proj_build_details.version}") + print(f"\tSystem install version: {sys_build_details.version}") sys.exit(1) - return (proj_version, sys_version) + if sys_build_details.version_parsed < proj_build_details.version_parsed: + print("\nNo update required, project is newer than the system install") + print(f"\tProject version: {proj_build_details.version}") + print(f"\tSystem install version: {sys_build_details.version}") + sys.exit(0) + if sys_build_details.version_parsed == proj_build_details.version_parsed: + print(f"\nNo update required, project and system install are the same version - {proj_build_details.version}") + sys.exit(0) -def get_build_version(path: Path) -> str: - """Retrieve the build version from a build file""" - try: - with open(path, "r", encoding="utf-8") as f: - data = json.load(f) - version = data["VERSION"] - except (OSError, json.JSONDecodeError) as err: - print(f"\nERROR: Couldn't get the version information from {path}: {err}") - sys.exit(1) - except KeyError as err: - print(f"\nERROR: Couldn't retrieve {err} from {path}") - sys.exit(1) + return (proj_build_details, sys_build_details) - return version - -def remove_dir(dir_: Path, excludes: list[Path]): +def remove_dir(dir_: Path, excludes: list[Path]) -> None: """Recursively remove a directory""" try: for path in dir_.iterdir(): @@ -212,10 +207,10 @@ def remove_dir(dir_: Path, excludes: list[Path]): with suppress(FileNotFoundError): dir_.rmdir() except OSError as err: - raise err + raise OSError(f"Couldn't remove directory: {err}") from err -def remove_files_dirs(to_remove: dict[str, Any], target_dir: Path): +def remove_files_dirs(to_remove: dict[str, Any], target_dir: Path) -> None: """Remove files/directories that are no longer required""" for r_path, r_dict in to_remove.items(): target: Path = target_dir / r_path @@ -223,57 +218,60 @@ def remove_files_dirs(to_remove: dict[str, Any], target_dir: Path): excludes = [target / e for e in r_dict["excludes"]] try: - if target.is_file() or target.is_symlink(): - target.unlink(missing_ok=True) + if target.is_dir(): + if not files or (files and files[0] == "*"): + remove_dir(target, excludes) - if target.is_dir() and not files or (files and files[0] == "*"): - remove_dir(target, excludes) + if files and files[0] != "*": + target_files = [] + for f in files: + target_files.append(target / f) if "*" not in f else target_files.extend(list(target.glob(f))) - if target.is_dir() and files and files[0] != "*": - target_files = [] - for f in files: - target_files.append(target / f) if "*" not in f else target_files.extend(list(target.glob(f))) + for target_file in target_files: + target_file.unlink(missing_ok=True) - for target_file in target_files: - target_file.unlink(missing_ok=True) + if target.is_file() or target.is_symlink(): + target.unlink(missing_ok=True) except OSError as err: - raise OSError(f"ERROR: Couldn't delete a file or directory: {err}") from err + raise OSError(f"Couldn't delete a file or directory: {err}") from err -def copy_files_dirs(to_copy: dict[str, Any], source_dir: Path, target_dir: Path): - """Copy files/directories within and to tne project""" +def copy_files_dirs(to_copy: dict[str, Any], source_dir: Path, target_dir: Path) -> None: + """Copy files/directories within and to a project""" for c_path, c_dict in to_copy.items(): excludes = c_dict["excludes"] files = c_dict["files"] - source: Path = source_dir / c_path - target: Path = target_dir / c_path - - if c_path.startswith("/"): - source = Path(c_path) - try: - target = target_dir / source.relative_to(SZ_SYS_PATH) - except ValueError: - target = target_dir / target.name + source = source_dir / c_path try: + if source.is_dir(): + # If the key in to_copy ends with / copy everything in source dir to target_dir + # er/ as the key copies everything from /opt/senzing/er to target_dir + # + # If the key in to_copy doesn't end with / copy source dir and everything in it to target_dir + # data as the key copies /opt/senzing/data to target_dir/data + target = target_dir if c_path.endswith("/") else target_dir / c_path + + # Copy entire contents of the source directory + if not files or (files and files[0] == "*"): + shutil.copytree(source, target, ignore=shutil.ignore_patterns(*excludes), dirs_exist_ok=True) + + # Create the source directory in the target and only copy listed files + if files and files[0] != "*": + target.mkdir(exist_ok=True, parents=True) + for source_file in [source / f for f in files]: + shutil.copy(source_file, target / source_file.name) + if source.is_file(): - shutil.copy( - source, - target, - ) - - if source.is_dir() and not files or (files and files[0] == "*"): - shutil.copytree(source, target, ignore=shutil.ignore_patterns(*excludes), dirs_exist_ok=True) - - if source.is_dir() and files and files[0] != "*": - target.mkdir(exist_ok=True, parents=True) - for source_file in [source / f for f in files]: - shutil.copy(source_file, target / source_file.name) + # Single file copy always copies only the file, if the key to to_copy is er/szBuildVersion.json + # szBuildVersion.json is copied to target_dir and not target_dir/er/szBuildVersion.json + target = target_dir / source.name + shutil.copy(source, target) except OSError as err: - raise OSError(f"ERROR: Couldn't copy a file or directory: {err}") from err + raise OSError(f"Couldn't copy a file or directory: {err}") from err -def rename_files(to_rename: dict[str, dict[str, str]], target_dir: Path): +def rename_files(to_rename: dict[str, dict[str, str]], target_dir: Path) -> None: """Rename existing project files that had a name change""" try: @@ -283,108 +281,52 @@ def rename_files(to_rename: dict[str, dict[str, str]], target_dir: Path): with suppress(FileNotFoundError): current.rename(new) except OSError as err: - raise OSError(f"ERROR: Couldn't rename a file or directory: {err}") from err - - -def setup_env(proj_path: Path): - """Create a new setupEnv and replace place holders with paths for the project""" - try: - shutil.copy(proj_path / "resources/templates/setupEnv", proj_path) - setup_path = proj_path / "setupEnv" - - with open(setup_path, "r", encoding="utf-8") as in_: - data = in_.read() - - data = data.replace("${SENZING_DIR}", str(proj_path)).replace("${SENZING_CONFIG_PATH}", str(proj_path / "etc")) - - with open(setup_path, "w", encoding="utf-8") as out: - out.write(data) - except OSError as err: - raise OSError(f"ERROR: Couldn't create a new setupEnv file: {err}") from err - - -def set_permissions(proj_path: Path, permissions: dict[str, dict[str, Any]]): - """ - Reset permissions for files and dirs copied to the projector, or dirs removed and replaced - completely e.g., data/ - """ - try: - for p_path, p_dict in permissions.items(): - dir_pint = p_dict["dir_pint"] - file_pint = p_dict["file_pint"] - files = p_dict["files"] - recursive = p_dict["recursive"] - target = proj_path if p_path.startswith(".") else proj_path / p_path - excludes = [target / e for e in p_dict["excludes"]] - - if target.is_file(): - target.chmod(file_pint) - - if target.is_dir() and dir_pint != 0: - target.chmod(dir_pint) - d_chmods = ( - [d for d in target.glob("*") if d.is_dir() and not d.is_symlink() and d not in excludes] - if not recursive - else [d for d in target.rglob("*") if d.is_dir() and not d.is_symlink() and d not in excludes] - ) - for dir_ in d_chmods: - dir_.chmod(dir_pint) - - if target.is_dir() and (files and files[0] == "*"): - f_chmods = ( - [f for f in target.glob("*") if f.is_file() and not f.is_symlink() and f not in excludes] - if not recursive - else [f for f in target.rglob("*") if f.is_file() and not f.is_symlink() and f not in excludes] - ) - for file in f_chmods: - Path(target / file).chmod(file_pint) - - if target.is_dir() and files and files[0] != "*": - for file in files: - Path(target / file).chmod(file_pint) - except OSError as err: - raise OSError(f"ERROR: Couldn't set a permission: {err}") from err + raise OSError(f"Couldn't rename a file or directory: {err}") from err def main() -> None: """main""" cli_args = parse_cli_args() - proj_path = Path(cli_args.project_path).resolve() - proj_ver, sys_ver = pre_check(proj_path, proj_path / PROJ_BUILD, SZ_SYS_PATH / SYS_BUILD, dir_listing(proj_path)) + proj_path = Path(cli_args.project_path).expanduser().resolve() + proj_build, sys_build = pre_check(proj_path) + proj_is_v3 = bool(proj_build.major == 3) if not cli_args.force_mode: - print(f"\nWARNING: If you don't have a backup of the project ({proj_path}), create one before completing this!") + print("\nWARNING: If you don't have a backup of the project, create one before continuing!") sleep(3) - if input(f"\nContinue updating the project from version {proj_ver} to {sys_ver}? (y/n) ") not in INPUT_CONFS: + if ( + input(f"\nUpdate the project from version {proj_build.build_version} to {sys_build.build_version}? (y/n) ") + not in INPUT_CONFS + ): sys.exit(0) print("\nUpdating...") else: - print(f"\nUpdating project from version {proj_ver} to {sys_ver}...") + print(f"\nUpdating project from version {proj_build.build_version} to {sys_build.build_version}...") try: - # Backup some of the V3 project files - if proj_ver[:1] == "3": + if proj_is_v3: v3_backup_path = proj_path / V3_BACKUP_PATH v3_backup_path.mkdir(exist_ok=True) copy_files_dirs(V3_BACKUP_PROJ, proj_path, v3_backup_path) - - rename_files(V3_RENAME_IN_PROJ, proj_path) - remove_files_dirs(V3_REMOVE_FROM_PROJ, proj_path) - copy_files_dirs(V3_COPY_TO_PROJ, SZ_SYS_PATH, proj_path) - setup_env(proj_path) - set_permissions(proj_path, V3_RESET_PERMISSIONS) - except OSError as err: - shutil.copytree(v3_backup_path, proj_path, dirs_exist_ok=True) - print(f"\n{err}") - print( - "\nIf the error is file or directory permission related, run again with a user with appropriate privileges" - ) + rename_files(V3_RENAME_IN_PROJ, proj_path) + remove_files_dirs(V3_REMOVE_FROM_PROJ, proj_path) + copy_files_dirs(COPY_TO_PROJ, SZ_SYS_PATH, proj_path) + if proj_is_v3: + setup_env(proj_path) + set_permissions(proj_path, PERMISSIONS) + set_permissions(proj_path, PERMISSIONS_2) + except (OSError, TypeError) as err: + if proj_is_v3: + shutil.copytree(v3_backup_path, proj_path, dirs_exist_ok=True) + print(f"\nERROR: {err}") + print("\nIf the error is file or directory permission related, run again with appropriate privileges") print( "\nIf the error is missing file or directory, check senzingsdk-setup, senzingsdk-tools, and senzingsdk-poc are installed" ) else: - # Remove if no errors so re-running can find the file - proj_path.joinpath(PROJ_BUILD).unlink(missing_ok=True) + if proj_is_v3: # Only delete if no errors so re-running can find the file again + proj_path.joinpath(V3_BUILD).unlink(missing_ok=True) + print(f"\nProject successfully updated. Refer to {UPGRADE_URL} for additional upgrade instructions")