From f7ca5da4ad7877fbf7a2893d85d9532a072a407d Mon Sep 17 00:00:00 2001 From: Ant Date: Thu, 19 Jun 2025 15:44:42 +0100 Subject: [PATCH 1/8] #188 - Save point --- pyproject.toml | 1 + sz_tools/_project_helpers.py | 188 +++++++++++ sz_tools/_tool_helpers.py | 56 +--- sz_tools/sz_command | 3 - sz_tools/sz_configtool | 2 - sz_tools/sz_create_project | 239 ++++---------- sz_tools/sz_create_project_prior1 | 268 ++++++++++++++++ sz_tools/sz_explorer | 3 - sz_tools/sz_export | 2 - sz_tools/sz_file_loader | 33 +- sz_tools/sz_json_analyzer | 19 +- sz_tools/sz_setup_config | 5 +- sz_tools/sz_update_project | 362 +++++++++------------ sz_tools/sz_update_project_prior1 | 483 ++++++++++++++++++++++++++++ sz_tools/sz_update_project_prior2 | 492 ++++++++++++++++++++++++++++ sz_tools/sz_update_project_prior3 | 478 +++++++++++++++++++++++++++ sz_tools/sz_update_project_prior4 | 456 ++++++++++++++++++++++++++ sz_tools/sz_update_project_prior5 | 515 ++++++++++++++++++++++++++++++ sz_tools/sz_update_project_prior6 | 465 +++++++++++++++++++++++++++ sz_tools/sz_update_project_prior7 | 480 ++++++++++++++++++++++++++++ sz_tools/sz_update_project_prior8 | 463 +++++++++++++++++++++++++++ sz_tools/sz_update_project_prior9 | 496 ++++++++++++++++++++++++++++ 22 files changed, 5007 insertions(+), 502 deletions(-) create mode 100644 sz_tools/_project_helpers.py create mode 100755 sz_tools/sz_create_project_prior1 create mode 100755 sz_tools/sz_update_project_prior1 create mode 100755 sz_tools/sz_update_project_prior2 create mode 100755 sz_tools/sz_update_project_prior3 create mode 100755 sz_tools/sz_update_project_prior4 create mode 100755 sz_tools/sz_update_project_prior5 create mode 100755 sz_tools/sz_update_project_prior6 create mode 100755 sz_tools/sz_update_project_prior7 create mode 100755 sz_tools/sz_update_project_prior8 create mode 100755 sz_tools/sz_update_project_prior9 diff --git a/pyproject.toml b/pyproject.toml index c6a00bd..ce839b1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,6 +41,7 @@ disable = [ "consider-using-f-string", "line-too-long", "too-many-branches", + "too-many-instance-attributes", "too-many-locals" ] good-names = [ diff --git a/sz_tools/_project_helpers.py b/sz_tools/_project_helpers.py new file mode 100644 index 0000000..9296d5b --- /dev/null +++ b/sz_tools/_project_helpers.py @@ -0,0 +1,188 @@ +"""Helpers for creating and updating projects""" + +import json +import shutil +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +from packaging import version as p_version + +V3_BACKUP_PATH = "v3_to_v4_upgrade_backups" +V4_BUILD = "szBuildVersion.json" +SZ_SYS_PATH = Path("/opt/senzing") +V4_SYS_PATH = SZ_SYS_PATH / "er" +V4_DATA_PATH = SZ_SYS_PATH / "data" +V4_SYS_BUILD = V4_SYS_PATH / V4_BUILD + +COPY_TO_PROJ = { + "er/": {"files": ["*"], "excludes": ["sz_create_project", "sz_update_project", "_project_helpers.py"]}, + "data": {"files": ["*"], "excludes": []}, +} + +PERMISSIONS = { + ".": { + "dir_pint": 0, + "file_pint": 0o660, + "files": ["LICENSE", "NOTICES", "README.1ST", "szBuildVersion.json"], + "excludes": ["setupEnv"], + "recursive": False, + }, + "setupEnv": {"dir_pint": 0, "file_pint": 0o770, "files": [], "excludes": [], "recursive": False}, + "bin": { + "dir_pint": 0o770, + "file_pint": 0o770, + "files": ["*"], + "excludes": ["__pycache__", "_sz_database.py", "_tool_helpers.py"], + "recursive": False, + }, + "data": {"dir_pint": 0o770, "file_pint": 0o660, "files": ["*"], "excludes": [], "recursive": True}, + "lib": {"dir_pint": 0o770, "file_pint": 0o660, "files": ["*"], "excludes": [], "recursive": False}, + "resources": {"dir_pint": 0o770, "file_pint": 0o660, "files": ["*"], "excludes": ["setupEnv"], "recursive": True}, + "resources/templates/setupEnv": { + "dir_pint": 0, + "file_pint": 0o770, + "files": [], + "excludes": [], + "recursive": False, + }, + "sdk": {"dir_pint": 0o770, "file_pint": 0o660, "files": ["*"], "excludes": [], "recursive": True}, + V3_BACKUP_PATH: {"dir_pint": 0o770, "file_pint": 0, "files": [], "excludes": [], "recursive": False}, +} + +PERMISSIONS_2 = { + "bin": { + "dir_pint": 0o770, + "file_pint": 0o660, + "files": ["_sz_database.py", "_tool_helpers.py"], + "excludes": [], + "recursive": False, + } +} + + +@dataclass() +class SzBuildDetails: + """Build information for a project or Senzing SDK system install""" + + platform: str + version: str + build_version: str + build_number: str + major: int = field(init=False) + minor: int = field(init=False) + micro: int = field(init=False) + + def __post_init__(self) -> None: + self.version_parsed = p_version.parse(self.version) + self.build_version_parsed = p_version.parse(self.build_version) + self.major = self.version_parsed.major + self.minor = self.version_parsed.minor + self.micro = self.version_parsed.micro + + +def get_build_details(path: Path) -> SzBuildDetails: + """Return dataclass with the details from a build file.""" + try: + with open(path, "r", encoding="utf-8") as f: + # Ignore DATA_VERSION, V3 build files had it V4 doesn't + version_dict = {k.lower(): v for k, v in json.load(f).items() if k != "DATA_VERSION"} + except (OSError, json.JSONDecodeError) as err: + raise OSError(f"Couldn't get the build information from {path}: {err}") from err + + return SzBuildDetails(**version_dict) + + +def copy_files_dirs(to_copy: dict[str, Any], source_dir: Path, target_dir: Path) -> None: + """Copy files/directories within and to a project""" + for c_path, c_dict in to_copy.items(): + excludes = c_dict["excludes"] + files = c_dict["files"] + source = source_dir / c_path + + try: + if source.is_dir(): + # If the key in to_copy ends with / copy everything in source dir to target_dir + # er/ as the key copies everything from /opt/senzing/er to target_dir + # + # If the key in to_copy doesn't end with / copy source dir and everything in it to target_dir + # data as the key copies /opt/senzing/data to target_dir/data + target = target_dir if c_path.endswith("/") else target_dir / c_path + + # Copy entire contents of the source directory + if not files or (files and files[0] == "*"): + shutil.copytree(source, target, ignore=shutil.ignore_patterns(*excludes), dirs_exist_ok=True) + + # Create the source directory in the target and only copy listed files + if files and files[0] != "*": + target.mkdir(exist_ok=True, parents=True) + for source_file in [source / f for f in files]: + shutil.copy(source_file, target / source_file.name) + + if source.is_file(): + # Single file copy always copies only the file, if the key to to_copy is er/szBuildVersion.json + # szBuildVersion.json is copied to target_dir and not target_dir/er/szBuildVersion.json + target = target_dir / source.name + target_dir.mkdir(exist_ok=True, parents=True) + shutil.copy(source, target) + except OSError as err: + raise OSError(f"Couldn't copy a file or directory: {err}") from err + + +def setup_env(proj_path: Path) -> None: + """Create a new setupEnv and replace place holders with paths for the project""" + try: + shutil.copy(proj_path / "resources/templates/setupEnv", proj_path) + setup_path = proj_path / "setupEnv" + + with open(setup_path, "r", encoding="utf-8") as in_: + data = in_.read() + + data = data.replace("${SENZING_DIR}", str(proj_path)).replace("${SENZING_CONFIG_PATH}", str(proj_path / "etc")) + + with open(setup_path, "w", encoding="utf-8") as out: + out.write(data) + except OSError as err: + raise OSError(f"Couldn't create a new setupEnv file: {err}") from err + + +def set_permissions(proj_path: Path, permissions: dict[str, dict[str, Any]]) -> None: + """Set permissions for files/dirs copied to the project, or dirs removed and replaced completely e.g., data/""" + try: + for p_path, p_dict in permissions.items(): + dir_pint = p_dict["dir_pint"] + file_pint = p_dict["file_pint"] + files = p_dict["files"] + recursive = p_dict["recursive"] + target = proj_path if p_path.startswith(".") else proj_path / p_path + excludes = [target / e for e in p_dict["excludes"]] + + if target.is_dir(): + if dir_pint != 0: + target.chmod(dir_pint) + d_chmods = ( + [d for d in target.glob("*") if d.is_dir() and not d.is_symlink() and d not in excludes] + if not recursive + else [d for d in target.rglob("*") if d.is_dir() and not d.is_symlink() and d not in excludes] + ) + for dir_ in d_chmods: + dir_.chmod(dir_pint) + + if files and files[0] == "*": + f_chmods = ( + [f for f in target.glob("*") if f.is_file() and not f.is_symlink() and f not in excludes] + if not recursive + else [f for f in target.rglob("*") if f.is_file() and not f.is_symlink() and f not in excludes] + ) + + for file in f_chmods: + Path(target / file).chmod(file_pint) + + if files and files[0] != "*": + for file in files: + Path(target / file).chmod(file_pint) + + if target.is_file(): + target.chmod(file_pint) + except OSError as err: + raise OSError(f"Couldn't set a permission: {err}") from err diff --git a/sz_tools/_tool_helpers.py b/sz_tools/_tool_helpers.py index aca2668..63a8297 100644 --- a/sz_tools/_tool_helpers.py +++ b/sz_tools/_tool_helpers.py @@ -1,7 +1,3 @@ -""" -# TODO -""" - from __future__ import annotations import cmd @@ -70,7 +66,7 @@ class TimedOut(Exception): - """# TODO""" + """Timeout""" # ------------------------------------------------------------------------- @@ -78,16 +74,12 @@ class TimedOut(Exception): # ------------------------------------------------------------------------- -@dataclass class Colors: - """# TODO""" - AVAILABLE_THEMES = ["DEFAULT", "LIGHT", "DARK", "TERMINAL"] @classmethod def apply(cls, to_color: Union[int, str], colors_list: str = "") -> Union[int, str]: """apply list of colors to a string""" - # TODO colors_list is a string with multiple entries separated by , if colors_list: prefix = "".join([getattr(cls, i.strip().upper()) for i in colors_list.split(",")]) return f"{prefix}{to_color}{cls.RESET}" @@ -96,7 +88,6 @@ def apply(cls, to_color: Union[int, str], colors_list: str = "") -> Union[int, s @classmethod def set_theme(cls, theme: str) -> None: - """# TODO""" theme = theme.upper() # best for dark backgrounds if theme == "DEFAULT": @@ -290,13 +281,12 @@ def set_theme(cls, theme: str) -> None: def check_environment() -> None: - """# TODO""" # Error if can't locate a sz_engine_config.ini or SENZING_ENGINE_CONFIGURATION_JSON if "SENZING_ETC_PATH" not in os.environ and "SENZING_ROOT" not in os.environ: # Check if set or not and that it's not set to null secj = os.environ.get("SENZING_ENGINE_CONFIGURATION_JSON") if not secj or (secj and len(secj) == 0): - # TODO V4 doc links + # TODO - V4 doc links print( textwrap.dedent( """\n\ @@ -317,7 +307,6 @@ def check_environment() -> None: def get_g2module_path() -> Path: - """# TODO""" file_paths = [] msg_args = f"Use command line argument -c (--inifile) to specify the path & filename for {CONFIG_FILE}\n" @@ -361,7 +350,6 @@ def get_g2module_path() -> Path: def print_config_locations(locations: List[Path]) -> None: - """# TODO""" _ = [print(f"\t{loc}") for loc in locations] print() @@ -396,8 +384,6 @@ def get_ini_as_json_str(ini_file: Path) -> str: def get_engine_config(ini_file_name: Union[str, None] = None) -> str: - """# TODO""" - # Initial check to determine is environment variables expected are set check_environment() @@ -432,7 +418,6 @@ def combine_engine_flags(flags: Union[List[TSzEngineFlags], List[str]]) -> int: def get_engine_flag_names() -> List[str]: - """# TODO""" return list(SzEngineFlags.__members__.keys()) @@ -446,8 +431,6 @@ def get_engine_flags_as_int(flags: List[str]) -> int: return int(flags[0]) # Named engine flag(s) used, combine and return the int value - # TODO - # return SzEngineFlags.combine_flags(flags) return combine_engine_flags(flags) @@ -482,8 +465,6 @@ def check_path_exists(path: Union[Path, str]) -> bool: def check_file_exists(file_name: Union[Path, str]) -> bool: - """# TODO""" - if isinstance(file_name, str): file_name = Path(file_name) @@ -494,8 +475,6 @@ def check_file_exists(file_name: Union[Path, str]) -> bool: # def check_file_readable(file_name: Union[Path, str]) -> bool: -# """# TODO""" - # if isinstance(file_name, str): # file_name = Path(file_name) @@ -512,10 +491,7 @@ def check_file_exists(file_name: Union[Path, str]) -> bool: # ------------------------------------------------------------------------- -# TODO - This can be merged into colorize_output def colorize_str(string: str, colors_list: str = "", color_disabled: bool = False) -> str: - """# TODO""" - if color_disabled: return string @@ -523,7 +499,6 @@ def colorize_str(string: str, colors_list: str = "", color_disabled: bool = Fals def colorize_json(json_str: str, color_disabled: bool = False) -> str: - """# TODO""" if color_disabled: return json_str @@ -538,13 +513,11 @@ def colorize_json(json_str: str, color_disabled: bool = False) -> str: return json_color -# TODO - Move into Colors and add the missing values? def colorize_output( output: Union[Exception, int, str], color_or_type: str, output_color: bool = True, ) -> str: - """# TODO""" if not output: return "" @@ -590,21 +563,18 @@ def colorize_cmd_prompt(prompt: str, color_or_type: str, color_prompt: bool = Tr def print_debug(msg: str, end_str: str = "\n\n", output_color: bool = True) -> None: - """# TODO""" print(f"\n{colorize_output('DEBUG:', 'debug', output_color)} {msg}", end=end_str) def print_error( msg: Union[Exception, str], end_str: str = "\n\n", output_color: bool = True, exit_: bool = False ) -> None: - """# TODO""" print(f"\n{colorize_output('ERROR:', 'error', output_color)} {msg}", end=end_str) if exit_: sys.exit(1) def print_info(msg: Union[Exception, str], end_str: str = "\n\n", output_color: bool = True, info_prefix=True) -> None: - """# TODO""" if info_prefix: print(f"\n{colorize_output('INFO:', 'info', output_color)} {msg}", end=end_str) else: @@ -612,7 +582,6 @@ def print_info(msg: Union[Exception, str], end_str: str = "\n\n", output_color: def print_warning(msg: Union[Exception, str], end_str: str = "\n\n", output_color: bool = True) -> None: - """# TODO""" # Warnings may be multiline strings, if they are don't add WARNING: before the msg to color if isinstance(msg, str) and "\n" in msg: print(f"\n{colorize_output(msg, 'warning', output_color)}", end=end_str) @@ -627,7 +596,6 @@ def print_response( color_output: bool = True, color: str = "", ) -> str: - """# TODO""" strip_colors = True if not response: @@ -684,12 +652,10 @@ def print_response( def do_shell(self: Union[SzCmdShell, SzCfgShell], line: str) -> None: # pylint: disable=unused-argument - """# TODO""" print(os.popen(line).read()) def do_help(self: Union[SzCmdShell, SzCfgShell], help_topic: str) -> None: - """# TODO""" if not help_topic or help_topic == "overview": self.help_overview() return @@ -751,7 +717,6 @@ def do_help(self: Union[SzCmdShell, SzCfgShell], help_topic: str) -> None: def do_history() -> None: - """# TODO""" print() for i in range(readline.get_current_history_length()): print(readline.get_history_item(i + 1)) @@ -796,12 +761,10 @@ def history_setup(module_name: str) -> Union[None, Path]: def history_write_file(file: Path) -> None: - """# TODO""" readline.write_history_file(file) def history_disabled(file: Path) -> None: - """# TODO""" # Save current session history history_now = [readline.get_history_item(i) for i in range(1, readline.get_current_history_length() + 1)] @@ -820,7 +783,6 @@ def history_disabled(file: Path) -> None: def response_to_clipboard(last_response: str) -> None: - """# TODO""" if not PYCLIP_AVAIL: print_info( "- To send the last response to the clipboard the Python module pyclip needs to be installed\n" @@ -841,7 +803,6 @@ def response_to_clipboard(last_response: str) -> None: def response_to_file( file_path: str, append_to_file: bool, add_last_command: bool, last_command: str, last_response: str ) -> None: - """# TODO""" try: mode = "a" if append_to_file else "w" with open(file_path, mode, encoding="utf-8") as response_out: @@ -859,7 +820,6 @@ def response_to_file( def response_reformat_json(last_response: str, color_json: bool) -> str: - """# TODO""" if not last_response.startswith("{"): print_warning("The last response isn't JSON") return "" @@ -874,8 +834,7 @@ def response_reformat_json(last_response: str, color_json: bool) -> str: def get_max_futures_workers() -> int: - """# TODO""" - # Test the max number of workers ThreadPoolExecutor allocates to use in sizing actual workers to request + """Test the max number of workers ThreadPoolExecutor allocates to use in sizing actual workers to request""" with concurrent.futures.ThreadPoolExecutor() as test: return test._max_workers # pylint: disable=protected-access @@ -886,7 +845,6 @@ def get_max_futures_workers() -> int: def human_readable_bytes(bytes_: int) -> str: - """# TODO""" if bytes_ == 0: return "0" @@ -906,9 +864,7 @@ def human_readable_bytes(bytes_: int) -> str: def case_combinations(strings: Iterable[str]) -> List[str]: - """# TODO""" combos = [] - try: for string in strings: combos.extend(list({"".join(sc) for sc in product(*zip(string.upper(), string.lower()))})) @@ -921,8 +877,6 @@ def case_combinations(strings: Iterable[str]) -> List[str]: def prompt_confirm(msg: str, confirm_values: Iterable[str] = ("y", "yes")) -> bool: - """# TODO""" - case_combos = case_combinations(confirm_values) response = input(msg).strip() if response not in case_combos: @@ -932,7 +886,6 @@ def prompt_confirm(msg: str, confirm_values: Iterable[str] = ("y", "yes")) -> bo def get_char() -> str: - """# TODO""" file_desc = sys.stdin.fileno() orig = termios.tcgetattr(file_desc) @@ -944,7 +897,6 @@ def get_char() -> str: def get_char_with_prompt(prompt, valid_responses=None): - """# TODO""" print(prompt, end="", flush=True) response = "" while True: @@ -970,8 +922,6 @@ def get_char_with_prompt(prompt, valid_responses=None): def get_char_with_timeout(time_out: int) -> str: - """# TODO""" - def handler(*_): # type: ignore[no-untyped-def] raise TimedOut diff --git a/sz_tools/sz_command b/sz_tools/sz_command index 828858a..53c08c0 100755 --- a/sz_tools/sz_command +++ b/sz_tools/sz_command @@ -1,7 +1,5 @@ #! /usr/bin/env python3 -"""# TODO""" - import argparse import cmd import functools @@ -568,7 +566,6 @@ class SzCmdShell(cmd.Cmd): set_theme_parser.add_argument("theme", choices=self.themes, nargs=1) def get_config_attr_codes(self) -> List[str]: - # TODO - Work in progress for JSON autocomplete try: config_id = self.sz_engine.get_active_config_id() sz_config = self.sz_configmgr.create_config_from_config_id(config_id) diff --git a/sz_tools/sz_configtool b/sz_tools/sz_configtool index 036ba0c..11afad4 100755 --- a/sz_tools/sz_configtool +++ b/sz_tools/sz_configtool @@ -40,7 +40,6 @@ MODULE_NAME = pathlib.Path(__file__).stem def parse_cli_args() -> argparse.Namespace: - """# TODO""" arg_parser = argparse.ArgumentParser( allow_abbrev=False, description="Utility to view and manipulate the Senzing configuration", @@ -1256,7 +1255,6 @@ class SzCfgShell(cmd.Cmd): self.print_json_lines(json_lines) - # TODO auto complete doesn't work if there is a - in dsrc def do_deleteDataSource(self, arg): """ Delete an existing data source diff --git a/sz_tools/sz_create_project b/sz_tools/sz_create_project index 748ccfc..9e3baf4 100755 --- a/sz_tools/sz_create_project +++ b/sz_tools/sz_create_project @@ -1,216 +1,103 @@ #! /usr/bin/env python3 - -"""# TODO""" +"""Create a Senzing SDK project""" import argparse -import json import sys from pathlib import Path -from shutil import copyfile, copytree, ignore_patterns -from typing import List, Union +from _project_helpers import ( + COPY_TO_PROJ, + PERMISSIONS, + PERMISSIONS_2, + SZ_SYS_PATH, + V4_SYS_BUILD, + copy_files_dirs, + get_build_details, + set_permissions, + setup_env, +) -def parse_cli_args() -> argparse.Namespace: - """# TODO""" +COPY_TO_ETC = {"er/resources/templates/": {"files": ["*"], "excludes": ["G2C.*", "setupEnv", "g2config.json"]}} +COPY_TO_VAR: dict[str, dict[str, list[str]]] = {"er/resources/templates/G2C.db": {"files": [], "excludes": []}} + +def parse_cli_args() -> argparse.Namespace: + """Parse the CLI arguments""" arg_parser = argparse.ArgumentParser( allow_abbrev=False, - description="Utility to create a new instance of a Senzing project in a path", + description=" Create a new instance of a Senzing project", formatter_class=argparse.RawTextHelpFormatter, ) arg_parser.add_argument( - "path", - help="path to create new Senzing project in, it must not already exist", - metavar="PATH", + "project_path", + metavar="path", + help="path to create new Senzing project in, it must not exist", ) return arg_parser.parse_args() -def get_version_details(sz_root_path: Path) -> List[str]: - """Return version details of Senzing installation""" - try: - sz_root_path = sz_root_path.joinpath("szBuildVersion.json") - with open(sz_root_path, encoding="utf-8") as file_version: - version_details = json.load(file_version) - except IOError as err: - print(f"\nERROR: Unable to read {sz_root_path} to retrieve version details - {err}") +def pre_check(project_path: Path) -> None: + """Check not trying to overwrite the V4 Senzing system install and the path doesn't exist""" + if str(project_path).startswith(str(SZ_SYS_PATH)): + print(f"\nProject cannot be created in {SZ_SYS_PATH}") sys.exit(1) - except json.JSONDecodeError as err: - print(f"\nERROR: {err}") - details: List[str] = [] - details.append(version_details.get("BUILD_VERSION", "")) - - if not all(details): - print(f"\nERROR: Problem reading values from version details, missing value(s) - {details}") + if project_path.exists(): + print(f"\n{project_path} exists, specify a different path") sys.exit(1) - return details - - -def replace_in_file(filename: Path, old_string: str, new_string: str) -> None: - """Replace strings in new project files""" +def update_sz_engine_config(config_file: Path, project_path: Path) -> None: + """Update sz_engine_config.ini with project paths""" try: - with open(filename, encoding="utf-8") as fr: - data = fr.read() - with open(filename, "w", encoding="utf-8") as fw: - fw.write(data.replace(old_string, new_string)) - except IOError as err: - raise err - - -def set_folder_permissions(path: Path, permissions: int, folders_to_ignore: Union[List[str], None] = None) -> None: - """Set permissions recursively on a folder, optionally ignore specific folders""" - if folders_to_ignore is None: - folders_to_ignore = [] - - path.chmod(permissions) - - dirs: List[Path] = [d for d in path.rglob("*") if d.is_dir() and not d.is_symlink() and d not in folders_to_ignore] - for dir_ in dirs: - dir_.chmod(permissions) - - -def set_file_permissions( - path: Path, - permissions: int, - files_to_ignore: Union[List[str], None] = None, - recursive: bool = False, -) -> None: - """Set permissions on files in a folder, optionally do recursively""" - if files_to_ignore is None: - files_to_ignore = [] - - files: List[Path] = [] - if recursive: - files = [f for f in path.rglob("*") if f.is_file() and f not in files_to_ignore] - else: - files = [f for f in path.iterdir() if f.is_file() and f not in files_to_ignore] + with open(config_file, "r", encoding="utf-8") as in_: + data = in_.read() + + data = ( + data.replace("${SENZING_DATA_DIR}", str(project_path / "data")) + .replace("${SENZING_CONFIG_PATH}", str(project_path / "etc")) + .replace("${SENZING_RESOURCES_DIR}", str(project_path / "resources")) + .replace("${SENZING_VAR_DIR}", str(project_path / "var")) + ) - for file in files: - file.chmod(permissions) + with open(config_file, "w", encoding="utf-8") as out: + out.write(data) + except OSError as err: + raise OSError(f"Couldn't update new {config_file}: {err}") from err def main() -> None: """main""" - cli_args = parse_cli_args() + proj_path = Path(cli_args.project_path).expanduser().resolve() + pre_check(proj_path) - # sz_path on normal rpm/deb install = /opt/senzing/g2 - # sz_install_root would then = /opt/senzing - # TODO Put back when in API package - sz_path = Path(__file__).resolve().parents[1] - # sz_path = Path("/opt/senzing/er").resolve() - sz_path_root = Path(__file__).resolve().parents[2] - project_path = Path(cli_args.path).expanduser().resolve() - - bin_path = project_path.joinpath("bin") - data_path = project_path.joinpath("data") - etc_path = project_path.joinpath("etc") - lib_path = project_path.joinpath("lib") - resources_path = project_path.joinpath("resources") - sdk_path = project_path.joinpath("sdk") - var_path = project_path.joinpath("var") - - if project_path.exists() and project_path.samefile(sz_path_root): - print(f"\nProject cannot be created in {sz_path_root}. Please specify a different path.") - sys.exit(1) - - if project_path.exists(): - print(f"\n{project_path} exists, please specify a different path.") - sys.exit(1) - - version_details = get_version_details(sz_path) - print(f"\nSenzing version: {version_details[0]}\n") - - ignore_files = ["sz_create_project"] - # Example: ignore_paths = [sz_path.joinpath('python')] - ignore_paths: List[str] = [] - excludes = ignore_files + ignore_paths - - # Copy sz_path to new project path - copytree(sz_path, project_path, ignore=ignore_patterns(*excludes), symlinks=True) - - # Copy resources/templates to etc - ignore_files = ["G2C.db", "setupEnv", "*.template", "g2config.json"] - copytree( - sz_path.joinpath("resources", "templates"), - etc_path, - ignore=ignore_patterns(*ignore_files), - ) - - # Copy setupEnv - copyfile( - sz_path.joinpath("resources", "templates", "setupEnv"), - project_path.joinpath("setupEnv"), - ) + try: + print(f"\nSenzing version: {get_build_details(V4_SYS_BUILD).version}\n") - # Copy G2C.db to runtime location - Path.mkdir(project_path.joinpath("var", "sqlite"), parents=True) - copyfile( - sz_path.joinpath("resources", "templates", "G2C.db"), - var_path.joinpath("sqlite", "G2C.db"), - ) + # Create project and copy main files + copy_files_dirs(COPY_TO_PROJ, SZ_SYS_PATH, proj_path) - # Copy data - copytree( - sz_path_root.joinpath("data"), - data_path, - ignore=ignore_patterns(*excludes), - symlinks=True, - ) + # Create and copy to proj_path/etc + copy_files_dirs(COPY_TO_ETC, SZ_SYS_PATH, proj_path / "etc") - # Files & strings to modify - update_files = [ - project_path.joinpath("setupEnv"), - etc_path.joinpath("sz_engine_config.ini"), - ] - - path_subs = [ - ("${SENZING_DIR}", project_path), - ("${SENZING_CONFIG_PATH}", etc_path), - ("${SENZING_DATA_DIR}", data_path), - ("${SENZING_RESOURCES_DIR}", resources_path), - ("${SENZING_VAR_DIR}", var_path), - ] - - for file in update_files: - for path in path_subs: - replace_in_file(file, path[0], str(path[1])) - - # Folder permissions - set_folder_permissions(project_path, 0o770) - - # Root of project - set_file_permissions(project_path, 0o660) - project_path.joinpath("setupEnv").chmod(0o770) - - # bin - set_file_permissions(bin_path, 0o770, recursive=True) - - # etc - set_file_permissions(etc_path, 0o660) - - # lib - set_file_permissions( - lib_path, - 0o660, - files_to_ignore=["g2.jar"], - ) + # Copy and modify setupEnv + setup_env(proj_path) - # resources - set_file_permissions(resources_path, 0o660, recursive=True) - resources_path.joinpath("templates", "setupEnv").chmod(0o770) + # Create proj_path/var/sqlite and copy G2C.db + copy_files_dirs(COPY_TO_VAR, SZ_SYS_PATH, proj_path / "var" / "sqlite") - # sdk - set_file_permissions(sdk_path, 0o664, recursive=True) + # Modify sz_engine_config.ini + update_sz_engine_config(proj_path / "etc" / "sz_engine_config.ini", proj_path) - # var - set_file_permissions(var_path, 0o660, recursive=True) - - print("Successfully created.") + # Set permissions on the project + set_permissions(proj_path, PERMISSIONS) + set_permissions(proj_path, PERMISSIONS_2) + except OSError as err: + print(f"\nERROR: {err}") + else: + print("Successfully created") if __name__ == "__main__": diff --git a/sz_tools/sz_create_project_prior1 b/sz_tools/sz_create_project_prior1 new file mode 100755 index 0000000..352f1ee --- /dev/null +++ b/sz_tools/sz_create_project_prior1 @@ -0,0 +1,268 @@ +#! /usr/bin/env python3 +"""Create a Senzing SDK project""" + +import argparse +import sys +from pathlib import Path +from shutil import copyfile, copytree, ignore_patterns + +# TODO - +from typing import List, Union + +from _project_helpers import ( + COPY_TO_PROJ, + PERMISSIONS, + PERMISSIONS_2, + SZ_SYS_PATH, + V4_SYS_BUILD, + copy_files_dirs, + get_build_details, + set_permissions, + setup_env, +) + +# sz_path on normal rpm/deb install = /opt/senzing/g2 +# sz_install_root would then = /opt/senzing +# TODO Put back when in API package +# SZ_SYS_PATH = Path(__file__).resolve().parents[1] +# sz_path = Path("/opt/senzing/er").resolve() +SZ_SYS_ROOT = Path(__file__).resolve().parents[2] +# V4_BUILD = SZ_SYS_PATH / "szBuildVersion.json" +COPY_TO_ETC = {"er/resources/templates/": {"files": ["*"], "excludes": ["G2C.*", "setupEnv", "g2config.json"]}} + +COPY_TO_VAR = {"er/resources/templates/G2C.db": {"files": [], "excludes": []}} + + +def parse_cli_args() -> argparse.Namespace: + """Parse the CLI arguments""" + arg_parser = argparse.ArgumentParser( + allow_abbrev=False, + description=" Create a new instance of a Senzing project", + formatter_class=argparse.RawTextHelpFormatter, + ) + arg_parser.add_argument( + "project_path", + metavar="path", + help="Path to create new Senzing project in, it must not exist", + ) + + return arg_parser.parse_args() + + +# TODO - +def pre_check(project_path: Path) -> None: + # TODO - + """Check not trying to overwrite the V4 Senzing system install, that path is a project, and versions are correct""" + # if proj_path.exists() and proj_path.samefile(SZ_SYS_ROOT): + # print(f"\nProject cannot be created in {SZ_SYS_ROOT}. Please specify a different path.") + # sys.exit(1) + if str(project_path).startswith(str(SZ_SYS_ROOT)): + print(f"\nProject cannot be created in {SZ_SYS_ROOT}") + sys.exit(1) + + if project_path.exists(): + print(f"\n{project_path} exists, specify a different path") + sys.exit(1) + + +def replace_in_file(filename: Path, old_string: str, new_string: str) -> None: + """Replace strings in new project files""" + + try: + with open(filename, encoding="utf-8") as fr: + data = fr.read() + with open(filename, "w", encoding="utf-8") as fw: + fw.write(data.replace(old_string, new_string)) + except IOError as err: + raise err + + +def set_folder_permissions(path: Path, permissions: int, folders_to_ignore: Union[List[str], None] = None) -> None: + """Set permissions recursively on a folder, optionally ignore specific folders""" + if folders_to_ignore is None: + folders_to_ignore = [] + + path.chmod(permissions) + + dirs: List[Path] = [d for d in path.rglob("*") if d.is_dir() and not d.is_symlink() and d not in folders_to_ignore] + for dir_ in dirs: + dir_.chmod(permissions) + + +def set_file_permissions( + path: Path, + permissions: int, + files_to_ignore: Union[List[str], None] = None, + recursive: bool = False, +) -> None: + """Set permissions on files in a folder, optionally do recursively""" + if files_to_ignore is None: + files_to_ignore = [] + + files: List[Path] = [] + if recursive: + files = [f for f in path.rglob("*") if f.is_file() and f not in files_to_ignore] + else: + files = [f for f in path.iterdir() if f.is_file() and f not in files_to_ignore] + + for file in files: + file.chmod(permissions) + + +# TODO - +def update_sz_engine_config(config_file: Path, project_path: Path) -> None: + """TODO""" + try: + with open(config_file, "r", encoding="utf-8") as in_: + data = in_.read() + + data = ( + data.replace("${SENZING_DATA_DIR}", str(project_path / "data")) + .replace("${SENZING_CONFIG_PATH}", str(project_path / "etc")) + .replace("${SENZING_RESOURCES_DIR}", str(project_path / "resources")) + .replace("${SENZING_VAR_DIR}", str(project_path / "var")) + ) + + with open(config_file, "w", encoding="utf-8") as out: + out.write(data) + except OSError as err: + raise OSError(f"ERROR: Couldn't update new {config_file}: {err}") from err + + +def main() -> None: + """main""" + + cli_args = parse_cli_args() + + # TODO - + # # sz_path on normal rpm/deb install = /opt/senzing/g2 + # # sz_install_root would then = /opt/senzing + # # TODO Put back when in API package + # sz_path = Path(__file__).resolve().parents[1] + # # sz_path = Path("/opt/senzing/er").resolve() + # sz_path_root = Path(__file__).resolve().parents[2] + proj_path = Path(cli_args.project_path).expanduser().resolve() + + bin_path = proj_path.joinpath("bin") + data_path = proj_path.joinpath("data") + etc_path = proj_path.joinpath("etc") + lib_path = proj_path.joinpath("lib") + resources_path = proj_path.joinpath("resources") + sdk_path = proj_path.joinpath("sdk") + var_path = proj_path.joinpath("var") + + # if proj_path.exists() and proj_path.samefile(SZ_SYS_ROOT): + # print(f"\nProject cannot be created in {SZ_SYS_ROOT}. Please specify a different path.") + # sys.exit(1) + + # if proj_path.exists(): + # print(f"\n{proj_path} exists, please specify a different path.") + # sys.exit(1) + pre_check(proj_path) + + # TODO - Add try/except and check all functions are raising not printing and being caught + + # version_details = get_version_details(SZ_SYS_PATH) + build_details = get_build_details(V4_SYS_BUILD) + # print(f"\nSenzing version: {build_details[0]}\n") + print(f"\nSenzing version: {build_details.version}\n") + print(f"\nSenzing version: {build_details}\n") + + # TODO - + # ignore_files = ["sz_create_project", "sz_update_project"] + # # Example: ignore_paths = [sz_path.joinpath('python')] + # ignore_paths: List[str] = [] + # excludes = ignore_files + ignore_paths + + # # Copy sz_path to new project path + # copytree(SZ_SYS_PATH, proj_path, ignore=ignore_patterns(*excludes), symlinks=True) + copy_files_dirs(COPY_TO_PROJ, SZ_SYS_PATH, proj_path) + # Copy resources/templates to etc + # ignore_files = ["G2C.db", "setupEnv", "*.template", "g2config.json"] + # copytree( + # SZ_SYS_PATH.joinpath("resources", "templates"), + # etc_path, + # ignore=ignore_patterns(*ignore_files), + # ) + copy_files_dirs(COPY_TO_ETC, SZ_SYS_PATH, proj_path / "etc") + + # Copy setupEnv + # copyfile( + # SZ_SYS_PATH.joinpath("resources", "templates", "setupEnv"), + # proj_path.joinpath("setupEnv"), + # ) + setup_env(proj_path) + + # Copy G2C.db to runtime location + # Path.mkdir(proj_path.joinpath("var", "sqlite"), parents=True) + # copyfile( + # SZ_SYS_PATH.joinpath("resources", "templates", "G2C.db"), + # var_path.joinpath("sqlite", "G2C.db"), + # ) + copy_files_dirs(COPY_TO_VAR, SZ_SYS_PATH, proj_path / "var" / "sqlite") + + # Copy data + # copytree( + # SZ_SYS_ROOT.joinpath("data"), + # data_path, + # ignore=ignore_patterns(*excludes), + # symlinks=True, + # ) + + # # Files & strings to modify + # update_files = [ + # proj_path.joinpath("setupEnv"), + # etc_path.joinpath("sz_engine_config.ini"), + # ] + + # path_subs = [ + # ("${SENZING_DIR}", proj_path), + # ("${SENZING_CONFIG_PATH}", etc_path), + # ("${SENZING_DATA_DIR}", data_path), + # ("${SENZING_RESOURCES_DIR}", resources_path), + # ("${SENZING_VAR_DIR}", var_path), + # ] + + # for file in update_files: + # for path in path_subs: + # replace_in_file(file, path[0], str(path[1])) + + update_sz_engine_config(proj_path / "etc" / "sz_engine_config.ini", proj_path) + + set_permissions(proj_path, PERMISSIONS) + set_permissions(proj_path, PERMISSIONS_2) + # # Folder permissions + # set_folder_permissions(proj_path, 0o770) + + # # Root of project + # set_file_permissions(proj_path, 0o660) + # proj_path.joinpath("setupEnv").chmod(0o770) + + # # bin + # set_file_permissions(bin_path, 0o770, recursive=True) + + # # etc + # set_file_permissions(etc_path, 0o660) + + # # lib + # set_file_permissions( + # lib_path, + # 0o660, + # files_to_ignore=["g2.jar"], + # ) + + # # resources + # set_file_permissions(resources_path, 0o660, recursive=True) + # resources_path.joinpath("templates", "setupEnv").chmod(0o770) + + # # sdk + # set_file_permissions(sdk_path, 0o664, recursive=True) + + # # var + # set_file_permissions(var_path, 0o660, recursive=True) + + print("Successfully created.") + + +if __name__ == "__main__": + main() diff --git a/sz_tools/sz_explorer b/sz_tools/sz_explorer index de1c8cd..fe5afd6 100755 --- a/sz_tools/sz_explorer +++ b/sz_tools/sz_explorer @@ -3011,7 +3011,6 @@ class EdaCmd(cmd.Cmd): f"\nSenzing Support Request: {colorize('https://senzing.zendesk.com/hc/en-us/requests/new', 'highlight2, underline')}\n" ) - # TODO - Ant - Use from helpers def histCheck(self): file_name = f".{MODULE_NAME}_history" @@ -3051,7 +3050,6 @@ class EdaCmd(cmd.Cmd): self.histFileError = None self.histAvail = True - # TODO - Ant - Use from helpers def do_history(self, arg): if self.histAvail: @@ -3062,7 +3060,6 @@ class EdaCmd(cmd.Cmd): else: print_message("History isn't available in this session", "warning") - # TODO - Ant - Use from helpers def do_shell(self, line): """\nRun OS shell commands: !\n""" if line: diff --git a/sz_tools/sz_export b/sz_tools/sz_export index 3a5ba3a..6a48d6c 100755 --- a/sz_tools/sz_export +++ b/sz_tools/sz_export @@ -490,8 +490,6 @@ if __name__ == "__main__": valid_flags = [flag for flag in flags if flag not in invalid_string_flags] - # TODO - # final_flags = SzEngineFlags.combine_flags(valid_flags) final_flags = combine_engine_flags(valid_flags) # Initialize the export diff --git a/sz_tools/sz_file_loader b/sz_tools/sz_file_loader index 551c970..dcf9539 100755 --- a/sz_tools/sz_file_loader +++ b/sz_tools/sz_file_loader @@ -392,19 +392,6 @@ def check_ingest_files(files: list[str]): json_good += 1 except JSONDecodeError: json_errors += 1 - # TODO - # try: - # csv_sample = file_.read(5000) - # csv_header = csv.Sniffer().has_header(csv_sample) - # csv_dialect = csv.Sniffer().sniff(csv_sample) - # print(f"\n{csv_header = }") - # print(f"\n{csv_dialect.__dict__ = }") - # reader = csv.reader(file_, csv_dialect) - # for _ in range(500): - # row = next(reader) - # print(row) - # except Exception as err: - # print(f"{err = }") except OSError as err: logger.info("") logger.error(err) @@ -464,7 +451,7 @@ def docker_redirects( # Have a file but not another redirect if files_list and not any(redirects): - file_parent = str(Path(files_list[0]).parent.resolve()) + file_parent = str(Path(files_list[0]).parent.expanduser().resolve()) logger.info( "Setting missing error, with info, shuffled files output paths to ingest file path, any of -ep, -wp, -sp, -rp not specified" ) @@ -553,9 +540,6 @@ def shuffle_ingest_file( logger.info("Shuffling to: %s", shuff_file) shuf_cmd = "gshuf" if sys.platform == "darwin" else "shuf" cmd = f"{shuf_cmd} {ingest_file} > {shuff_file}" - # TODO - # if sourceDict["FILE_FORMAT"] not in ("JSON", "UMF"): - # cmd = f"head -n1 {file_path} > {shuf_file_path} && tail -n+2 {file_path} | shuf >> {shuf_file_path}" try: _ = subprocess.run([cmd], capture_output=True, check=True, shell=True) @@ -913,19 +897,6 @@ def load_and_redo( for f in done: try: result = f.result() - # TODO - Test on OS & SG load with redo when engine fixed for process_redo_record() - # TODO - Collect and retry later? - # # If caught a retryable error resubmit the record to try again - # except SzRetryableError as err: - # logger.info("") - # logger.info( - # "Retrying record due to: %s - Operation: %s - Record: %s", - # err, - # mode_text[mode.__name__]["except_msg"], - # futures[f][0].strip(), - # ) - # logger.info("") - # more_recs = add_new_future(futures[f][0].strip()) except ( SzError, JSONDecodeError, @@ -963,7 +934,7 @@ def load_and_redo( finally: if add_future and not shutdown.is_set(): more_recs = add_new_future() - + del futures[f] # Early errors check to catch mapping errors, missing dsrc_code, etc diff --git a/sz_tools/sz_json_analyzer b/sz_tools/sz_json_analyzer index ea34766..db02109 100755 --- a/sz_tools/sz_json_analyzer +++ b/sz_tools/sz_json_analyzer @@ -26,8 +26,6 @@ MODULE_NAME = pathlib.Path(__file__).stem class JsonlReader: - """#TODO""" - def __init__(self, file_handle): self.file_handle = file_handle @@ -39,7 +37,6 @@ class JsonlReader: def get_environment_config(settings: str) -> Dict[str, Any]: - """# TODO""" try: sz_factory = SzAbstractFactoryCore(MODULE_NAME, settings) sz_configmgr = sz_factory.create_configmanager() @@ -54,7 +51,6 @@ def get_environment_config(settings: str) -> Dict[str, Any]: def get_file_config(file: str) -> Dict[str, Any]: - """#TODO""" try: with open(file, "r", encoding="utf-8") as config_file: config_json: Dict[str, Any] = json.loads(config_file.read()) @@ -65,8 +61,6 @@ def get_file_config(file: str) -> Dict[str, Any]: class SzJsonAnalyzer: - """#TODO""" - def __init__(self, config_data: Dict[str, Any]): self.attribute_lookup = {} @@ -107,11 +101,10 @@ class SzJsonAnalyzer: for record in config_data["G2_CONFIG"]["CFG_FTYPE"]: self.feature_lookup[record["FTYPE_CODE"]] = record - # TODO hack until 4.0 to move record_type higher + # NOTE - hack until 4.0 to move record_type higher self.feature_order["RECORD_TYPE"] = 1004 def register_attribute(self, attr_name: str) -> None: - """#TODO""" attr_data = {} if attr_name in self.attribute_lookup: attr_data = self.attribute_lookup[attr_name] @@ -143,8 +136,6 @@ class SzJsonAnalyzer: attr_name: str, attr_value: str, ) -> None: - """#TODO""" - if isinstance(attr_value, (list, dict)): errors.append(f"Expected integer or string for {attr_name}") else: @@ -157,7 +148,6 @@ class SzJsonAnalyzer: features[feature_key].append(attr_data) def update_feature_stats(self, feature: str, attribute: str, value: str) -> None: - """#TODO""" if attribute in self.feature_stats[feature]["attributes"]: self.feature_stats[feature]["attributes"][attribute]["count"] += 1 else: @@ -174,7 +164,6 @@ class SzJsonAnalyzer: self.feature_stats[feature]["attributes"][attribute]["values"][value] = 1 def update_unmapped_stats(self, attr_name: str, attr_value: str) -> None: - """#TODO""" if attr_name in self.unmapped_stats: self.unmapped_stats[attr_name]["count"] += 1 else: @@ -185,7 +174,6 @@ class SzJsonAnalyzer: self.unmapped_stats[attr_name]["values"][attr_value] = 1 def update_message_stats(self, cat: str, stat: str, row_num: Union[int, str] = "n/a") -> None: - """#TODO""" row_num = f"row {row_num}" if isinstance(row_num, int) else row_num if stat not in self.message_stats[cat]: self.message_stats[cat][stat] = {"count": 1, "rows": [row_num]} @@ -195,7 +183,6 @@ class SzJsonAnalyzer: self.message_stats[cat][stat]["rows"].append(row_num) def analyze_json(self, input_data: Dict[str, Any], input_row_num: Union[int, None]) -> None: - """#TODO""" self.record_count += 1 # print('-'*50) @@ -397,7 +384,6 @@ class SzJsonAnalyzer: self.update_message_stats(message[0], message[1], input_row_num) def get_report(self) -> List[List[Union[int, str]]]: - """#TODO""" table_headers = [ "Category", "Attribute", @@ -558,7 +544,6 @@ class SzJsonAnalyzer: # ---------------------------------------- def format_pretty_table(table_rows): - """#TODO""" table_object = prettytable.PrettyTable() table_object.horizontal_char = "\u2500" table_object.vertical_char = "\u2502" @@ -631,7 +616,6 @@ def format_pretty_table(table_rows): def format_csv_table(table_rows): - """#TODO""" output = io.StringIO() writer = csv.writer(output) writer.writerows(table_rows) @@ -639,7 +623,6 @@ def format_csv_table(table_rows): def print_report(report_string: str): - """#TODO""" less = subprocess.Popen(["less", "-FMXSR"], stdin=subprocess.PIPE) try: less.stdin.write(report_string.encode("utf-8")) diff --git a/sz_tools/sz_setup_config b/sz_tools/sz_setup_config index f6459dd..79bbcf0 100755 --- a/sz_tools/sz_setup_config +++ b/sz_tools/sz_setup_config @@ -1,6 +1,4 @@ #! /usr/bin/env python3 -"""# TODO""" - import argparse import pathlib import sys @@ -13,7 +11,6 @@ MODULE_NAME = pathlib.Path(__file__).stem def parse_cli_args() -> argparse.Namespace: - """# TODO""" arg_parser = argparse.ArgumentParser() arg_parser.add_argument( "-c", @@ -29,7 +26,7 @@ def parse_cli_args() -> argparse.Namespace: def main() -> None: - """# TODO""" + """main""" cli_args = parse_cli_args() # Check an engine configuration can be located diff --git a/sz_tools/sz_update_project b/sz_tools/sz_update_project index bba8813..1aaa47b 100755 --- a/sz_tools/sz_update_project +++ b/sz_tools/sz_update_project @@ -1,8 +1,7 @@ #! /usr/bin/env python3 -"""Upgrade a V3 or V4 Senzing SDK project""" +"""Upgrade a V3 or V4 Senzing SDK project to V4.x.x""" import argparse -import json import shutil import sys from contextlib import suppress @@ -10,33 +9,66 @@ from pathlib import Path from time import sleep from typing import Any -INPUT_CONFS = ("y", "Y", "yes", "YES") -SZ_SYS_PATH = Path("/opt/senzing/er") +from _project_helpers import ( + PERMISSIONS, + PERMISSIONS_2, + SZ_SYS_PATH, + V3_BACKUP_PATH, + V4_BUILD, + V4_SYS_BUILD, + V4_SYS_PATH, + SzBuildDetails, + get_build_details, + set_permissions, + setup_env, +) + +INPUT_CONFS = ("y", "Y", "yEs", "yES", "YES", "yes", "YEs", "yeS", "Yes", "YeS") MODULE_NAME = Path(__file__).stem -PROJ_BUILD = "g2BuildVersion.json" -SYS_BUILD = SZ_SYS_PATH / "szBuildVersion.json" -V3_BACKUP_PATH = "v3_upgrade_backups" +V3_BUILD = "g2BuildVersion.json" +V3_SYS_PATH = SZ_SYS_PATH / "g2" +V3_SYS_BUILD = V3_SYS_PATH / V3_BUILD UPGRADE_URL = "https://www.senzing.com/docs/4_beta/index.html" V3_BACKUP_PROJ = { "bin": {"files": ["g2dbencrypt", "g2saltadm", "g2ssadm"], "excludes": []}, "etc": {"files": ["senzing_governor.py"], "excludes": []}, "lib": {"files": ["libSpaceTimeBoxStandardizer.so", "libG2Hasher.so", "libG2SSAdm.so"], "excludes": []}, - "python": {"files": [], "excludes": []}, - "resources/templates": {"files": ["setupEnv"], "excludes": []}, + "python": { + "files": ["*"], + "excludes": [ + "CompressedFile.py", + "DumpStack.py", + "G2Audit.py", + "G2Command.py", + "G2ConfigTables.py", + "G2ConfigTool.py", + "G2Database.py", + "G2Explorer.py", + "G2Export.py", + "G2IniParams.py", + "G2Loader.py", + "G2Paths.py", + "G2Project.py", + "G2S3.py", + "G2SetupConfig.py", + "G2Snapshot.py", + "SenzingGo.py", + "senzing", + ], + }, "setupEnv": {"files": [], "excludes": []}, "g2BuildVersion.json": {"files": [], "excludes": []}, } V3_REMOVE_FROM_PROJ = { - "bin": {"files": ["*"], "excludes": ["bin"]}, + "bin": {"files": ["g2configupgrade", "g2dbencrypt", "g2dbupgrade", "g2saltadm", "g2ssadm"], "excludes": ["bin"]}, "data": {"files": [], "excludes": []}, "etc": {"files": ["senzing_governor.py"], "excludes": []}, - # "g2BuildVersion.json": {"files": [], "excludes": []}, "lib": { "files": [ "g2.jar", - "libG2.so", + "libG2*.so", "libG2Hasher.so", "libG2SSAdm.so", "libg2CompJavaScoreSet.so", @@ -56,6 +88,7 @@ V3_REMOVE_FROM_PROJ = { "files": [ "G2C.db*", "G2Module.ini", + "cfgVariant.json", "custom*.txt", "defaultGNRCP.config", "g2config.json", @@ -65,70 +98,32 @@ V3_REMOVE_FROM_PROJ = { ], "excludes": [], }, - "sdk": {"files": ["*"], "excludes": ["sdk"]}, + "sdk": {"files": ["*"], "excludes": []}, "setupEnv": {"files": [], "excludes": []}, } -V3_COPY_TO_PROJ = { - "LICENSE": {"files": [], "excludes": []}, - "NOTICES": {"files": [], "excludes": []}, - "README.1ST": {"files": [], "excludes": []}, - "bin": {"files": ["*"], "excludes": ["sz_create_project", "sz_update_project"]}, - "/opt/senzing/data": {"files": ["*"], "excludes": []}, - "lib": {"files": ["*"], "excludes": []}, - "resources": {"files": ["*"], "excludes": []}, - "sdk": {"files": ["*"], "excludes": []}, - "szBuildVersion.json": {"files": [], "excludes": []}, +COPY_TO_PROJ = { + "er/": {"files": ["*"], "excludes": ["sz_create_project", "sz_update_project"]}, + "data": {"files": ["*"], "excludes": []}, } - V3_RENAME_IN_PROJ = { "etc": {"from": "G2Module.ini", "to": "sz_engine_config.ini"}, } - -V3_RESET_PERMISSIONS = { - ".": { - "dir_pint": 0, - "file_pint": 0o660, - "files": ["LICENSE", "NOTICES", "README.1ST", "szBuildVersion.json"], - "excludes": ["setupEnv"], - "recursive": False, - }, - "setupEnv": {"dir_pint": 0, "file_pint": 0o770, "files": [], "excludes": [], "recursive": False}, - "bin": { - "dir_pint": 0o770, - "file_pint": 0o770, - "files": ["*"], - "excludes": ["__pycache__", "_sz_database.py", "_tool_helpers.py"], - "recursive": False, - }, - "data": {"dir_pint": 0o770, "file_pint": 0o660, "files": ["*"], "excludes": [], "recursive": True}, - "lib": {"dir_pint": 0o770, "file_pint": 0o660, "files": ["*"], "excludes": [], "recursive": False}, - "resources": {"dir_pint": 0o770, "file_pint": 0o660, "files": ["*"], "excludes": ["setupEnv"], "recursive": True}, - "resources/templates/setupEnv": { - "dir_pint": 0, - "file_pint": 0o770, - "files": [], - "excludes": [], - "recursive": False, - }, - "sdk": {"dir_pint": 0o770, "file_pint": 0o660, "files": ["*"], "excludes": [], "recursive": True}, - "v3_upgrade_backups": {"dir_pint": 0o770, "file_pint": 0, "files": [], "excludes": [], "recursive": False}, -} - # pylint: disable=W0106 def parse_cli_args() -> argparse.Namespace: """Parse the CLI arguments""" arg_parser = argparse.ArgumentParser( - description="Update an existing Senzing project to the system installed version of Senzing." + allow_abbrev=False, + description="Update an existing V3 or V4 Senzing project to the system installed V4 of Senzing.", ) arg_parser.add_argument( "project_path", metavar="path", - help="path of the project to update", + help="Path of the project to update", ) arg_parser.add_argument( "-f", @@ -136,7 +131,7 @@ def parse_cli_args() -> argparse.Namespace: dest="force_mode", default=False, action="store_true", - help="upgrade without prompts. WARNING: Use with caution, ensure you have a backup of the project", + help="Upgrade without prompts. WARNING: Use with caution, ensure you have a backup of the project", ) return arg_parser.parse_args() @@ -154,53 +149,53 @@ def dir_listing(path: Path) -> list[Path]: return listing -def pre_check(path: Path, proj_build_file: Path, sys_build_file: Path, listing: list[Path]) -> tuple[str, str]: - """Check not trying to overwrite the Senzing system install, that path is a project and versions are correct""" - if not SZ_SYS_PATH.exists(): - print(f"\nERROR: Couldn't locate Senzing SDK system install at {SZ_SYS_PATH}") +def pre_check(project_path: Path) -> tuple[SzBuildDetails, SzBuildDetails]: + """Check not trying to overwrite the V4 Senzing system install, that path is a project, and versions are correct""" + if not V4_SYS_PATH.exists(): + print(f"\nERROR: Couldn't locate Senzing SDK V4 system install at {V4_SYS_PATH}") sys.exit(1) - if path.samefile(SZ_SYS_PATH): - print(f"\nERROR: {path} is the Senzing system installation path and not a Senzing project") + if str(project_path).startswith(str(SZ_SYS_PATH)): + print(f"\nERROR: {project_path} is the Senzing system installation, not a Senzing project") sys.exit(1) - if proj_build_file not in listing: - print(f"\nERROR: {path} isn't a Senzing project, expected it to contain the file {proj_build_file.name}") + v3_project_build_file = project_path / V3_BUILD + v4_project_build_file = project_path / V4_BUILD + proj_listing = dir_listing(project_path) + if v3_project_build_file not in proj_listing and v4_project_build_file not in proj_listing: + print(f"\nERROR: {project_path} isn't a Senzing project, expected it to contain either:") + print(f"\tExisting V3 project: {v3_project_build_file}") + print(f"\tExisting V4 project: {v4_project_build_file}") sys.exit(1) try: - proj_version = get_build_version(proj_build_file) - sys_version = get_build_version(sys_build_file) - except OSError as err: - print(f"\nERROR: Trying to read {proj_build_file} and {sys_build_file}: {err}") + proj_build_file = v3_project_build_file if v3_project_build_file in proj_listing else v4_project_build_file + proj_build_details = get_build_details(proj_build_file) + sys_build_details = get_build_details(V4_SYS_BUILD) + except (OSError, TypeError) as err: + print(f"\nERROR: Trying to read {proj_build_file} or {V4_SYS_BUILD} to collect version information: {err}") sys.exit(1) - if int(proj_version[:1]) not in (3, 4) or sys_version[:1] != "4": - print( - f"\nERROR: {MODULE_NAME} updates from V3, or V4.x to V4.y, project version: {proj_version} installed Senzing SDK version: {sys_version}" - ) + if proj_build_details.major not in (3, 4) or sys_build_details.major != 4: + print(f"\nERROR: {MODULE_NAME} updates a V3 project to V4, or V4.n.n to a newer release") + print(f"\tProject version: {proj_build_details.version}") + print(f"\tSystem install version: {sys_build_details.version}") sys.exit(1) - return (proj_version, sys_version) + if sys_build_details.version_parsed < proj_build_details.version_parsed: + print("\nNo update required, project is newer than the system install") + print(f"\tProject version: {proj_build_details.version}") + print(f"\tSystem install version: {sys_build_details.version}") + sys.exit(0) + if sys_build_details.version_parsed == proj_build_details.version_parsed: + print(f"\nNo update required, project and system install are the same version - {proj_build_details.version}") + sys.exit(0) -def get_build_version(path: Path) -> str: - """Retrieve the build version from a build file""" - try: - with open(path, "r", encoding="utf-8") as f: - data = json.load(f) - version = data["VERSION"] - except (OSError, json.JSONDecodeError) as err: - print(f"\nERROR: Couldn't get the version information from {path}: {err}") - sys.exit(1) - except KeyError as err: - print(f"\nERROR: Couldn't retrieve {err} from {path}") - sys.exit(1) + return (proj_build_details, sys_build_details) - return version - -def remove_dir(dir_: Path, excludes: list[Path]): +def remove_dir(dir_: Path, excludes: list[Path]) -> None: """Recursively remove a directory""" try: for path in dir_.iterdir(): @@ -212,10 +207,10 @@ def remove_dir(dir_: Path, excludes: list[Path]): with suppress(FileNotFoundError): dir_.rmdir() except OSError as err: - raise err + raise OSError(f"Couldn't remove directory: {err}") from err -def remove_files_dirs(to_remove: dict[str, Any], target_dir: Path): +def remove_files_dirs(to_remove: dict[str, Any], target_dir: Path) -> None: """Remove files/directories that are no longer required""" for r_path, r_dict in to_remove.items(): target: Path = target_dir / r_path @@ -223,57 +218,60 @@ def remove_files_dirs(to_remove: dict[str, Any], target_dir: Path): excludes = [target / e for e in r_dict["excludes"]] try: - if target.is_file() or target.is_symlink(): - target.unlink(missing_ok=True) + if target.is_dir(): + if not files or (files and files[0] == "*"): + remove_dir(target, excludes) - if target.is_dir() and not files or (files and files[0] == "*"): - remove_dir(target, excludes) + if files and files[0] != "*": + target_files = [] + for f in files: + target_files.append(target / f) if "*" not in f else target_files.extend(list(target.glob(f))) - if target.is_dir() and files and files[0] != "*": - target_files = [] - for f in files: - target_files.append(target / f) if "*" not in f else target_files.extend(list(target.glob(f))) + for target_file in target_files: + target_file.unlink(missing_ok=True) - for target_file in target_files: - target_file.unlink(missing_ok=True) + if target.is_file() or target.is_symlink(): + target.unlink(missing_ok=True) except OSError as err: - raise OSError(f"ERROR: Couldn't delete a file or directory: {err}") from err + raise OSError(f"Couldn't delete a file or directory: {err}") from err -def copy_files_dirs(to_copy: dict[str, Any], source_dir: Path, target_dir: Path): - """Copy files/directories within and to tne project""" +def copy_files_dirs(to_copy: dict[str, Any], source_dir: Path, target_dir: Path) -> None: + """Copy files/directories within and to a project""" for c_path, c_dict in to_copy.items(): excludes = c_dict["excludes"] files = c_dict["files"] - source: Path = source_dir / c_path - target: Path = target_dir / c_path - - if c_path.startswith("/"): - source = Path(c_path) - try: - target = target_dir / source.relative_to(SZ_SYS_PATH) - except ValueError: - target = target_dir / target.name + source = source_dir / c_path try: + if source.is_dir(): + # If the key in to_copy ends with / copy everything in source dir to target_dir + # er/ as the key copies everything from /opt/senzing/er to target_dir + # + # If the key in to_copy doesn't end with / copy source dir and everything in it to target_dir + # data as the key copies /opt/senzing/data to target_dir/data + target = target_dir if c_path.endswith("/") else target_dir / c_path + + # Copy entire contents of the source directory + if not files or (files and files[0] == "*"): + shutil.copytree(source, target, ignore=shutil.ignore_patterns(*excludes), dirs_exist_ok=True) + + # Create the source directory in the target and only copy listed files + if files and files[0] != "*": + target.mkdir(exist_ok=True, parents=True) + for source_file in [source / f for f in files]: + shutil.copy(source_file, target / source_file.name) + if source.is_file(): - shutil.copy( - source, - target, - ) - - if source.is_dir() and not files or (files and files[0] == "*"): - shutil.copytree(source, target, ignore=shutil.ignore_patterns(*excludes), dirs_exist_ok=True) - - if source.is_dir() and files and files[0] != "*": - target.mkdir(exist_ok=True, parents=True) - for source_file in [source / f for f in files]: - shutil.copy(source_file, target / source_file.name) + # Single file copy always copies only the file, if the key to to_copy is er/szBuildVersion.json + # szBuildVersion.json is copied to target_dir and not target_dir/er/szBuildVersion.json + target = target_dir / source.name + shutil.copy(source, target) except OSError as err: - raise OSError(f"ERROR: Couldn't copy a file or directory: {err}") from err + raise OSError(f"Couldn't copy a file or directory: {err}") from err -def rename_files(to_rename: dict[str, dict[str, str]], target_dir: Path): +def rename_files(to_rename: dict[str, dict[str, str]], target_dir: Path) -> None: """Rename existing project files that had a name change""" try: @@ -283,108 +281,52 @@ def rename_files(to_rename: dict[str, dict[str, str]], target_dir: Path): with suppress(FileNotFoundError): current.rename(new) except OSError as err: - raise OSError(f"ERROR: Couldn't rename a file or directory: {err}") from err - - -def setup_env(proj_path: Path): - """Create a new setupEnv and replace place holders with paths for the project""" - try: - shutil.copy(proj_path / "resources/templates/setupEnv", proj_path) - setup_path = proj_path / "setupEnv" - - with open(setup_path, "r", encoding="utf-8") as in_: - data = in_.read() - - data = data.replace("${SENZING_DIR}", str(proj_path)).replace("${SENZING_CONFIG_PATH}", str(proj_path / "etc")) - - with open(setup_path, "w", encoding="utf-8") as out: - out.write(data) - except OSError as err: - raise OSError(f"ERROR: Couldn't create a new setupEnv file: {err}") from err - - -def set_permissions(proj_path: Path, permissions: dict[str, dict[str, Any]]): - """ - Reset permissions for files and dirs copied to the projector, or dirs removed and replaced - completely e.g., data/ - """ - try: - for p_path, p_dict in permissions.items(): - dir_pint = p_dict["dir_pint"] - file_pint = p_dict["file_pint"] - files = p_dict["files"] - recursive = p_dict["recursive"] - target = proj_path if p_path.startswith(".") else proj_path / p_path - excludes = [target / e for e in p_dict["excludes"]] - - if target.is_file(): - target.chmod(file_pint) - - if target.is_dir() and dir_pint != 0: - target.chmod(dir_pint) - d_chmods = ( - [d for d in target.glob("*") if d.is_dir() and not d.is_symlink() and d not in excludes] - if not recursive - else [d for d in target.rglob("*") if d.is_dir() and not d.is_symlink() and d not in excludes] - ) - for dir_ in d_chmods: - dir_.chmod(dir_pint) - - if target.is_dir() and (files and files[0] == "*"): - f_chmods = ( - [f for f in target.glob("*") if f.is_file() and not f.is_symlink() and f not in excludes] - if not recursive - else [f for f in target.rglob("*") if f.is_file() and not f.is_symlink() and f not in excludes] - ) - for file in f_chmods: - Path(target / file).chmod(file_pint) - - if target.is_dir() and files and files[0] != "*": - for file in files: - Path(target / file).chmod(file_pint) - except OSError as err: - raise OSError(f"ERROR: Couldn't set a permission: {err}") from err + raise OSError(f"Couldn't rename a file or directory: {err}") from err def main() -> None: """main""" cli_args = parse_cli_args() - proj_path = Path(cli_args.project_path).resolve() - proj_ver, sys_ver = pre_check(proj_path, proj_path / PROJ_BUILD, SZ_SYS_PATH / SYS_BUILD, dir_listing(proj_path)) + proj_path = Path(cli_args.project_path).expanduser().resolve() + proj_build, sys_build = pre_check(proj_path) + proj_is_v3 = bool(proj_build.major == 3) if not cli_args.force_mode: - print(f"\nWARNING: If you don't have a backup of the project ({proj_path}), create one before completing this!") + print("\nWARNING: If you don't have a backup of the project, create one before continuing!") sleep(3) - if input(f"\nContinue updating the project from version {proj_ver} to {sys_ver}? (y/n) ") not in INPUT_CONFS: + if ( + input(f"\nUpdate the project from version {proj_build.build_version} to {sys_build.build_version}? (y/n) ") + not in INPUT_CONFS + ): sys.exit(0) print("\nUpdating...") else: - print(f"\nUpdating project from version {proj_ver} to {sys_ver}...") + print(f"\nUpdating project from version {proj_build.build_version} to {sys_build.build_version}...") try: - # Backup some of the V3 project files - if proj_ver[:1] == "3": + if proj_is_v3: v3_backup_path = proj_path / V3_BACKUP_PATH v3_backup_path.mkdir(exist_ok=True) copy_files_dirs(V3_BACKUP_PROJ, proj_path, v3_backup_path) - - rename_files(V3_RENAME_IN_PROJ, proj_path) - remove_files_dirs(V3_REMOVE_FROM_PROJ, proj_path) - copy_files_dirs(V3_COPY_TO_PROJ, SZ_SYS_PATH, proj_path) - setup_env(proj_path) - set_permissions(proj_path, V3_RESET_PERMISSIONS) - except OSError as err: - shutil.copytree(v3_backup_path, proj_path, dirs_exist_ok=True) - print(f"\n{err}") - print( - "\nIf the error is file or directory permission related, run again with a user with appropriate privileges" - ) + rename_files(V3_RENAME_IN_PROJ, proj_path) + remove_files_dirs(V3_REMOVE_FROM_PROJ, proj_path) + copy_files_dirs(COPY_TO_PROJ, SZ_SYS_PATH, proj_path) + if proj_is_v3: + setup_env(proj_path) + set_permissions(proj_path, PERMISSIONS) + set_permissions(proj_path, PERMISSIONS_2) + except (OSError, TypeError) as err: + if proj_is_v3: + shutil.copytree(v3_backup_path, proj_path, dirs_exist_ok=True) + print(f"\nERROR: {err}") + print("\nIf the error is file or directory permission related, run again with appropriate privileges") print( "\nIf the error is missing file or directory, check senzingsdk-setup, senzingsdk-tools, and senzingsdk-poc are installed" ) else: - # Remove if no errors so re-running can find the file - proj_path.joinpath(PROJ_BUILD).unlink(missing_ok=True) + if proj_is_v3: # Only delete if no errors so re-running can find the file again + proj_path.joinpath(V3_BUILD).unlink(missing_ok=True) + print(f"\nProject successfully updated. Refer to {UPGRADE_URL} for additional upgrade instructions") diff --git a/sz_tools/sz_update_project_prior1 b/sz_tools/sz_update_project_prior1 new file mode 100755 index 0000000..d9e4a22 --- /dev/null +++ b/sz_tools/sz_update_project_prior1 @@ -0,0 +1,483 @@ +#! /usr/bin/env python3 +# TODO - sz_project_update +# TODO - +# TODO - +# TODO - +# TODO - +# TODO - + +# TODO - +"""Upgrade a V3 or V4 Senzing SDK project""" + +import argparse +import json +import shutil +import sys +from contextlib import suppress +from pathlib import Path +from time import sleep +from typing import Any, NamedTuple + +INPUT_CONFS = ("y", "Y", "yes", "YES") +MODULE_NAME = Path(__file__).stem +# TODO - +# V3_PROJ_BUILD = "g2BuildVersion.json" +# V4_PROJ_BUILD = "szBuildVersion.json" +V3_BUILD = "g2BuildVersion.json" +V4_BUILD = "szBuildVersion.json" +V3_SYS_PATH = Path("/opt/senzing/g2") +V4_SYS_PATH = Path("/opt/senzing/er") +V3_SYS_BUILD = V3_SYS_PATH / V3_BUILD +V4_SYS_BUILD = V4_SYS_PATH / V4_BUILD +V3_BACKUP_PATH = "v3_upgrade_backups" +UPGRADE_URL = "https://www.senzing.com/docs/4_beta/index.html" + +V3_BACKUP_PROJ = { + "bin": {"files": ["g2dbencrypt", "g2saltadm", "g2ssadm"], "excludes": []}, + "etc": {"files": ["senzing_governor.py"], "excludes": []}, + "lib": {"files": ["libSpaceTimeBoxStandardizer.so", "libG2Hasher.so", "libG2SSAdm.so"], "excludes": []}, + "python": {"files": [], "excludes": []}, + "resources/templates": {"files": ["setupEnv"], "excludes": []}, + "setupEnv": {"files": [], "excludes": []}, + "g2BuildVersion.json": {"files": [], "excludes": []}, +} + +V3_REMOVE_FROM_PROJ = { + "bin": {"files": ["*"], "excludes": ["bin"]}, + "data": {"files": [], "excludes": []}, + "etc": {"files": ["senzing_governor.py"], "excludes": []}, + # "g2BuildVersion.json": {"files": [], "excludes": []}, + "lib": { + "files": [ + "g2.jar", + "libG2.so", + "libG2Hasher.so", + "libG2SSAdm.so", + "libg2CompJavaScoreSet.so", + "libg2DistinctFeatJava.so", + "libg2EFeatJava.so", + "libg2JVMPlugin.so", + "libg2StdJava.so", + "libmariadbplugin.so", + "libSpaceTimeBoxStandardizer.so", + ], + "excludes": ["lib"], + }, + "python": {"files": ["*"], "excludes": []}, + "resources/config": {"files": ["g2core-configuration-upgrade-*.gtc"], "excludes": []}, + "resources/schema": {"files": ["g2core-schema-*-create.sql", "g2core-schema-*-upgrade-*.sql"], "excludes": []}, + "resources/templates": { + "files": [ + "G2C.db*", + "G2Module.ini", + "custom*.txt", + "defaultGNRCP.config", + "g2config.json", + "senzing_governor.py", + "setupEnv", + "stb.config", + ], + "excludes": [], + }, + "sdk": {"files": ["*"], "excludes": ["sdk"]}, + "setupEnv": {"files": [], "excludes": []}, +} + +V3_COPY_TO_PROJ = { + "LICENSE": {"files": [], "excludes": []}, + "NOTICES": {"files": [], "excludes": []}, + "README.1ST": {"files": [], "excludes": []}, + "bin": {"files": ["*"], "excludes": ["sz_create_project", "sz_update_project"]}, + "/opt/senzing/data": {"files": ["*"], "excludes": []}, + "lib": {"files": ["*"], "excludes": []}, + "resources": {"files": ["*"], "excludes": []}, + "sdk": {"files": ["*"], "excludes": []}, + "szBuildVersion.json": {"files": [], "excludes": []}, +} + + +V3_RENAME_IN_PROJ = { + "etc": {"from": "G2Module.ini", "to": "sz_engine_config.ini"}, +} + + +V3_RESET_PERMISSIONS = { + ".": { + "dir_pint": 0, + "file_pint": 0o660, + "files": ["LICENSE", "NOTICES", "README.1ST", "szBuildVersion.json"], + "excludes": ["setupEnv"], + "recursive": False, + }, + "setupEnv": {"dir_pint": 0, "file_pint": 0o770, "files": [], "excludes": [], "recursive": False}, + "bin": { + "dir_pint": 0o770, + "file_pint": 0o770, + "files": ["*"], + "excludes": ["__pycache__", "_sz_database.py", "_tool_helpers.py"], + "recursive": False, + }, + "data": {"dir_pint": 0o770, "file_pint": 0o660, "files": ["*"], "excludes": [], "recursive": True}, + "lib": {"dir_pint": 0o770, "file_pint": 0o660, "files": ["*"], "excludes": [], "recursive": False}, + "resources": {"dir_pint": 0o770, "file_pint": 0o660, "files": ["*"], "excludes": ["setupEnv"], "recursive": True}, + "resources/templates/setupEnv": { + "dir_pint": 0, + "file_pint": 0o770, + "files": [], + "excludes": [], + "recursive": False, + }, + "sdk": {"dir_pint": 0o770, "file_pint": 0o660, "files": ["*"], "excludes": [], "recursive": True}, + "v3_upgrade_backups": {"dir_pint": 0o770, "file_pint": 0, "files": [], "excludes": [], "recursive": False}, +} + + +# TODO - +# VersionTokens = namedtuple("VersionTokens", ["major", "minor", "patch"]) +class VersionTokens(NamedTuple): + major: int + minor: int + patch: int + + +# pylint: disable=W0106 + + +def parse_cli_args() -> argparse.Namespace: + """Parse the CLI arguments""" + arg_parser = argparse.ArgumentParser( + description="Update an existing Senzing project to the system installed version of Senzing." + ) + arg_parser.add_argument( + "project_path", + metavar="path", + help="path of the project to update", + ) + arg_parser.add_argument( + "-f", + "--force", + dest="force_mode", + default=False, + action="store_true", + help="upgrade without prompts. WARNING: Use with caution, ensure you have a backup of the project", + ) + return arg_parser.parse_args() + + +def dir_listing(path: Path) -> list[Path]: + """Get a listing of the specified path to check for files or directories""" + try: + listing = list(path.iterdir()) + except OSError as err: + if type(err).__name__ == "NotADirectoryError": + print(f"\nERROR: {path} is not a directory, expecting one") + print(f"\nERROR: {err}") + sys.exit(1) + + return listing + + +# TODO - +# pre_check(proj_path, proj_path / PROJ_BUILD, SZ_SYS_PATH / SYS_BUILD) +# def pre_check(project_path: Path, proj_build_file: Path, sys_build_file: Path, listing: list[Path]) -> tuple[str, str]: +def pre_check(project_path: Path) -> tuple[str, str]: + """Check not trying to overwrite the V4 Senzing system install, that path is a project, and versions are correct""" + if not V4_SYS_PATH.exists(): + print(f"\nERROR: Couldn't locate Senzing SDK system install at {V4_SYS_PATH}") + sys.exit(1) + + # TODO - + # if project_path.samefile(SZ_SYS_PATH): + if str(project_path).startswith(str(V4_SYS_PATH)): + print(f"\nERROR: {project_path} is the Senzing system installation, not a Senzing project") + sys.exit(1) + + # TODO - + # if proj_build_file not in listing: + # if proj_build_file not in dir_listing(project_path): + v3_project_build_file = project_path / V3_BUILD + v4_project_build_file = project_path / V4_BUILD + proj_listing = dir_listing(project_path) + print(f"\n{v3_project_build_file = }", flush=True) + print(f"{v4_project_build_file = }", flush=True) + print(f"{proj_listing = }", flush=True) + # if v3_project_build_file or v4_project_build_file not in dir_listing(project_path): + if v3_project_build_file not in proj_listing and v4_project_build_file not in proj_listing: + print(f"\nERROR: {project_path} isn't a Senzing project, expected it to contain either:") + print(f"\tExisting V3 project - {v3_project_build_file}") + print(f"\tExisting V4 project - {v4_project_build_file}") + sys.exit(1) + + # TODO - Does this work on V3 and V4? + try: + proj_build_file = v3_project_build_file if v3_project_build_file in proj_listing else v4_project_build_file + # TODO - What if don't get an int or error? + # proj_major_version, _, _ = get_build_versions(proj_build_file) + proj_version, proj_version_tokens = get_build_versions(proj_build_file) + # sys_build_file = V3_SYS_BUILD if proj_version_tokens.major == 3 else V4_SYS_BUILD + # sys_build_file = V3_SYS_BUILD if V3_SYS_BUILD.exists() else V4_SYS_BUILD + # sys_major_version, _, _ = get_build_versions(sys_build_file) + sys_version, sys_version_tokens = get_build_versions(V4_SYS_BUILD) + except OSError as err: + print(f"\nERROR: Trying to read {proj_build_file} or {V4_SYS_BUILD} to collect version information: {err}") + sys.exit(1) + + if proj_version_tokens.major not in (3, 4) or sys_version_tokens.major != 4: + print(f"\nERROR: {MODULE_NAME} updates V3 to V4, or V4.n.n to a newer release") + print(f"\tProject version: {proj_version_tokens.major}, system install version: {sys_version_tokens.major}") + sys.exit(1) + + # TODO - + if proj_version_tokens.major == 4 and (sum(sys_version_tokens) == sum(proj_version_tokens)): + print(f"No update required, project and system install are the same version - {proj_version}") + sys.exit(0) + + return (proj_version, sys_version) + + +# TODO - Should this check it looks like a version? If not above needs to check it. +# TODO - Have this return major, etc? +# def get_build_version(path: Path) -> str: +# TODO - Move to helpers? +# def get_build_versions(path: Path) -> list[int]: +# def get_build_versions(path: Path) -> list[int]: +def get_build_versions(path: Path) -> tuple[str, VersionTokens]: + """Return the version string and major, minor, and patch build versions from a build file""" + err_msg = f"ERROR: Couldn't get the version information from {path}" + + try: + with open(path, "r", encoding="utf-8") as f: + # data = json.load(f) + # version = data["VERSION"] + # version_str: str = data["VERSION"] + version_str: str = json.load(f)["VERSION"] + except (OSError, json.JSONDecodeError) as err: + print(f"\n{err_msg}: {err}") + sys.exit(1) + except KeyError as err: + print(f"\n{err_msg}, missing key {err}") + sys.exit(1) + + # TODO - test this + # str_tokens = version_str.split(".") + if not (str_tokens := version_str.split(".")) or (str_tokens and str_tokens[0] == ""): + print(f"\n{err_msg}, VERSION was blank") + sys.exit(1) + # TODO - + print(f"\n{str_tokens = }", flush=True) + + if len(str_tokens) != 3: + print(f"\nERROR: Version information should consist of major, minor, and patch, it is {version_str}") + sys.exit(1) + + try: + # tokens_list = [int(v) for v in vers_tokens] + vers_tokens = VersionTokens._make([int(v) for v in str_tokens]) + # t = [11, 22] + # Point._make(t) + except ValueError: + print(f"\nERROR: Version information should consist of integers, it is {version_str}") + sys.exit(1) + + # TODO - + # return version + return (version_str, vers_tokens) + + +def remove_dir(dir_: Path, excludes: list[Path]): + """Recursively remove a directory""" + try: + for path in dir_.iterdir(): + if path in excludes: + continue + path.unlink(missing_ok=True) if (path.is_file() or path.is_symlink()) else remove_dir(path, excludes) + + if not excludes: + with suppress(FileNotFoundError): + dir_.rmdir() + except OSError as err: + raise err + + +def remove_files_dirs(to_remove: dict[str, Any], target_dir: Path): + """Remove files/directories that are no longer required""" + for r_path, r_dict in to_remove.items(): + target: Path = target_dir / r_path + files = r_dict["files"] + excludes = [target / e for e in r_dict["excludes"]] + + try: + if target.is_file() or target.is_symlink(): + target.unlink(missing_ok=True) + + if target.is_dir() and not files or (files and files[0] == "*"): + remove_dir(target, excludes) + + if target.is_dir() and files and files[0] != "*": + target_files = [] + for f in files: + target_files.append(target / f) if "*" not in f else target_files.extend(list(target.glob(f))) + + for target_file in target_files: + target_file.unlink(missing_ok=True) + except OSError as err: + raise OSError(f"ERROR: Couldn't delete a file or directory: {err}") from err + + +def copy_files_dirs(to_copy: dict[str, Any], source_dir: Path, target_dir: Path): + """Copy files/directories within and to tne project""" + for c_path, c_dict in to_copy.items(): + excludes = c_dict["excludes"] + files = c_dict["files"] + source: Path = source_dir / c_path + target: Path = target_dir / c_path + + if c_path.startswith("/"): + source = Path(c_path) + try: + target = target_dir / source.relative_to(SZ_SYS_PATH) + except ValueError: + target = target_dir / target.name + + try: + if source.is_file(): + shutil.copy( + source, + target, + ) + + if source.is_dir() and not files or (files and files[0] == "*"): + shutil.copytree(source, target, ignore=shutil.ignore_patterns(*excludes), dirs_exist_ok=True) + + if source.is_dir() and files and files[0] != "*": + target.mkdir(exist_ok=True, parents=True) + for source_file in [source / f for f in files]: + shutil.copy(source_file, target / source_file.name) + except OSError as err: + raise OSError(f"ERROR: Couldn't copy a file or directory: {err}") from err + + +def rename_files(to_rename: dict[str, dict[str, str]], target_dir: Path): + """Rename existing project files that had a name change""" + + try: + for r_path, r_dict in to_rename.items(): + current = target_dir / r_path / r_dict["from"] + new = target_dir / r_path / r_dict["to"] + with suppress(FileNotFoundError): + current.rename(new) + except OSError as err: + raise OSError(f"ERROR: Couldn't rename a file or directory: {err}") from err + + +def setup_env(proj_path: Path): + """Create a new setupEnv and replace place holders with paths for the project""" + try: + shutil.copy(proj_path / "resources/templates/setupEnv", proj_path) + setup_path = proj_path / "setupEnv" + + with open(setup_path, "r", encoding="utf-8") as in_: + data = in_.read() + + data = data.replace("${SENZING_DIR}", str(proj_path)).replace("${SENZING_CONFIG_PATH}", str(proj_path / "etc")) + + with open(setup_path, "w", encoding="utf-8") as out: + out.write(data) + except OSError as err: + raise OSError(f"ERROR: Couldn't create a new setupEnv file: {err}") from err + + +def set_permissions(proj_path: Path, permissions: dict[str, dict[str, Any]]): + """ + Reset permissions for files and dirs copied to the projector, or dirs removed and replaced + completely e.g., data/ + """ + try: + for p_path, p_dict in permissions.items(): + dir_pint = p_dict["dir_pint"] + file_pint = p_dict["file_pint"] + files = p_dict["files"] + recursive = p_dict["recursive"] + target = proj_path if p_path.startswith(".") else proj_path / p_path + excludes = [target / e for e in p_dict["excludes"]] + + if target.is_file(): + target.chmod(file_pint) + + if target.is_dir() and dir_pint != 0: + target.chmod(dir_pint) + d_chmods = ( + [d for d in target.glob("*") if d.is_dir() and not d.is_symlink() and d not in excludes] + if not recursive + else [d for d in target.rglob("*") if d.is_dir() and not d.is_symlink() and d not in excludes] + ) + for dir_ in d_chmods: + dir_.chmod(dir_pint) + + if target.is_dir() and (files and files[0] == "*"): + f_chmods = ( + [f for f in target.glob("*") if f.is_file() and not f.is_symlink() and f not in excludes] + if not recursive + else [f for f in target.rglob("*") if f.is_file() and not f.is_symlink() and f not in excludes] + ) + for file in f_chmods: + Path(target / file).chmod(file_pint) + + if target.is_dir() and files and files[0] != "*": + for file in files: + Path(target / file).chmod(file_pint) + except OSError as err: + raise OSError(f"ERROR: Couldn't set a permission: {err}") from err + + +def main() -> None: + """main""" + cli_args = parse_cli_args() + proj_path = Path(cli_args.project_path).resolve() + # TODO - + # proj_ver, sys_ver = pre_check(proj_path, proj_path / PROJ_BUILD, SZ_SYS_PATH / SYS_BUILD, dir_listing(proj_path)) + # proj_ver, sys_ver = pre_check(proj_path, proj_path / PROJ_BUILD, SZ_SYS_PATH / SYS_BUILD) + proj_ver, sys_ver = pre_check(proj_path) + # TODO - + print(f"\n{proj_ver = }", flush=True) + print(f"{sys_ver = }", flush=True) + sys.exit() + + if not cli_args.force_mode: + print(f"\nWARNING: If you don't have a backup of the project ({proj_path}), create one before completing this!") + sleep(3) + if input(f"\nContinue updating the project from version {proj_ver} to {sys_ver}? (y/n) ") not in INPUT_CONFS: + sys.exit(0) + print("\nUpdating...") + else: + print(f"\nUpdating project from version {proj_ver} to {sys_ver}...") + + try: + # Backup some of the V3 project files + if proj_ver[:1] == "3": + v3_backup_path = proj_path / V3_BACKUP_PATH + v3_backup_path.mkdir(exist_ok=True) + copy_files_dirs(V3_BACKUP_PROJ, proj_path, v3_backup_path) + + rename_files(V3_RENAME_IN_PROJ, proj_path) + remove_files_dirs(V3_REMOVE_FROM_PROJ, proj_path) + copy_files_dirs(V3_COPY_TO_PROJ, SZ_SYS_PATH, proj_path) + setup_env(proj_path) + set_permissions(proj_path, V3_RESET_PERMISSIONS) + except OSError as err: + shutil.copytree(v3_backup_path, proj_path, dirs_exist_ok=True) + print(f"\n{err}") + print( + "\nIf the error is file or directory permission related, run again with a user with appropriate privileges" + ) + print( + "\nIf the error is missing file or directory, check senzingsdk-setup, senzingsdk-tools, and senzingsdk-poc are installed" + ) + else: + # Remove if no errors so re-running can find the file + proj_path.joinpath(PROJ_BUILD).unlink(missing_ok=True) + print(f"\nProject successfully updated. Refer to {UPGRADE_URL} for additional upgrade instructions") + + +if __name__ == "__main__": + main() diff --git a/sz_tools/sz_update_project_prior2 b/sz_tools/sz_update_project_prior2 new file mode 100755 index 0000000..3981ce7 --- /dev/null +++ b/sz_tools/sz_update_project_prior2 @@ -0,0 +1,492 @@ +#! /usr/bin/env python3 +# TODO - sz_project_update +# TODO - +# TODO - +# TODO - +# TODO - +# TODO - + +# TODO - +"""Upgrade a V3 or V4 Senzing SDK project""" + +import argparse +import json +import shutil +import sys +from contextlib import suppress +from dataclasses import dataclass +from pathlib import Path +from time import sleep +from typing import Any + +INPUT_CONFS = ("y", "Y", "yes", "YES") +MODULE_NAME = Path(__file__).stem +# TODO - +# V3_PROJ_BUILD = "g2BuildVersion.json" +# V4_PROJ_BUILD = "szBuildVersion.json" +V3_BUILD = "g2BuildVersion.json" +V4_BUILD = "szBuildVersion.json" +V3_SYS_PATH = Path("/opt/senzing/g2") +V4_SYS_PATH = Path("/opt/senzing/er") +V3_SYS_BUILD = V3_SYS_PATH / V3_BUILD +V4_SYS_BUILD = V4_SYS_PATH / V4_BUILD +V3_BACKUP_PATH = "v3_upgrade_backups" +UPGRADE_URL = "https://www.senzing.com/docs/4_beta/index.html" + +V3_BACKUP_PROJ = { + "bin": {"files": ["g2dbencrypt", "g2saltadm", "g2ssadm"], "excludes": []}, + "etc": {"files": ["senzing_governor.py"], "excludes": []}, + "lib": {"files": ["libSpaceTimeBoxStandardizer.so", "libG2Hasher.so", "libG2SSAdm.so"], "excludes": []}, + "python": {"files": [], "excludes": []}, + "resources/templates": {"files": ["setupEnv"], "excludes": []}, + "setupEnv": {"files": [], "excludes": []}, + "g2BuildVersion.json": {"files": [], "excludes": []}, +} + +V3_REMOVE_FROM_PROJ = { + "bin": {"files": ["*"], "excludes": ["bin"]}, + "data": {"files": [], "excludes": []}, + "etc": {"files": ["senzing_governor.py"], "excludes": []}, + # "g2BuildVersion.json": {"files": [], "excludes": []}, + "lib": { + "files": [ + "g2.jar", + "libG2.so", + "libG2Hasher.so", + "libG2SSAdm.so", + "libg2CompJavaScoreSet.so", + "libg2DistinctFeatJava.so", + "libg2EFeatJava.so", + "libg2JVMPlugin.so", + "libg2StdJava.so", + "libmariadbplugin.so", + "libSpaceTimeBoxStandardizer.so", + ], + "excludes": ["lib"], + }, + "python": {"files": ["*"], "excludes": []}, + "resources/config": {"files": ["g2core-configuration-upgrade-*.gtc"], "excludes": []}, + "resources/schema": {"files": ["g2core-schema-*-create.sql", "g2core-schema-*-upgrade-*.sql"], "excludes": []}, + "resources/templates": { + "files": [ + "G2C.db*", + "G2Module.ini", + "custom*.txt", + "defaultGNRCP.config", + "g2config.json", + "senzing_governor.py", + "setupEnv", + "stb.config", + ], + "excludes": [], + }, + "sdk": {"files": ["*"], "excludes": ["sdk"]}, + "setupEnv": {"files": [], "excludes": []}, +} + +# TODO - +# V3_COPY_TO_PROJ = { +# "LICENSE": {"files": [], "excludes": []}, +# "NOTICES": {"files": [], "excludes": []}, +# "README.1ST": {"files": [], "excludes": []}, +# "bin": {"files": ["*"], "excludes": ["sz_create_project", "sz_update_project"]}, +# "/opt/senzing/data": {"files": ["*"], "excludes": []}, +# "lib": {"files": ["*"], "excludes": []}, +# "resources": {"files": ["*"], "excludes": []}, +# "sdk": {"files": ["*"], "excludes": []}, +# "szBuildVersion.json": {"files": [], "excludes": []}, +# } +COPY_TO_PROJ = { + "LICENSE": {"files": [], "excludes": []}, + "NOTICES": {"files": [], "excludes": []}, + "README.1ST": {"files": [], "excludes": []}, + "bin": {"files": ["*"], "excludes": ["sz_create_project", "sz_update_project"]}, + "/opt/senzing/data": {"files": ["*"], "excludes": []}, + "lib": {"files": ["*"], "excludes": []}, + "resources": {"files": ["*"], "excludes": []}, + "sdk": {"files": ["*"], "excludes": []}, + "szBuildVersion.json": {"files": [], "excludes": []}, +} + + +V3_RENAME_IN_PROJ = { + "etc": {"from": "G2Module.ini", "to": "sz_engine_config.ini"}, +} + + +V3_RESET_PERMISSIONS = { + ".": { + "dir_pint": 0, + "file_pint": 0o660, + "files": ["LICENSE", "NOTICES", "README.1ST", "szBuildVersion.json"], + "excludes": ["setupEnv"], + "recursive": False, + }, + "setupEnv": {"dir_pint": 0, "file_pint": 0o770, "files": [], "excludes": [], "recursive": False}, + "bin": { + "dir_pint": 0o770, + "file_pint": 0o770, + "files": ["*"], + "excludes": ["__pycache__", "_sz_database.py", "_tool_helpers.py"], + "recursive": False, + }, + "data": {"dir_pint": 0o770, "file_pint": 0o660, "files": ["*"], "excludes": [], "recursive": True}, + "lib": {"dir_pint": 0o770, "file_pint": 0o660, "files": ["*"], "excludes": [], "recursive": False}, + "resources": {"dir_pint": 0o770, "file_pint": 0o660, "files": ["*"], "excludes": ["setupEnv"], "recursive": True}, + "resources/templates/setupEnv": { + "dir_pint": 0, + "file_pint": 0o770, + "files": [], + "excludes": [], + "recursive": False, + }, + "sdk": {"dir_pint": 0o770, "file_pint": 0o660, "files": ["*"], "excludes": [], "recursive": True}, + "v3_upgrade_backups": {"dir_pint": 0o770, "file_pint": 0, "files": [], "excludes": [], "recursive": False}, +} + + +# TODO - +# VersionTokens = namedtuple("VersionTokens", ["major", "minor", "patch"]) +# class VersionDetails(NamedTuple): +@dataclass() +class VersionDetails: + """Version information for a project or Senzing SDK system install""" + + version: str + major: int + minor: int + patch: int + + def __post_init__(self) -> None: + self.sum_mmp = sum((self.major, self.minor, self.patch)) + + +# pylint: disable=W0106 + + +def parse_cli_args() -> argparse.Namespace: + """Parse the CLI arguments""" + arg_parser = argparse.ArgumentParser( + description="Update an existing Senzing project to the system installed version of Senzing." + ) + arg_parser.add_argument( + "project_path", + metavar="path", + help="path of the project to update", + ) + arg_parser.add_argument( + "-f", + "--force", + dest="force_mode", + default=False, + action="store_true", + help="upgrade without prompts. WARNING: Use with caution, ensure you have a backup of the project", + ) + return arg_parser.parse_args() + + +def dir_listing(path: Path) -> list[Path]: + """Get a listing of the specified path to check for files or directories""" + try: + listing = list(path.iterdir()) + except OSError as err: + if type(err).__name__ == "NotADirectoryError": + print(f"\nERROR: {path} is not a directory, expecting one") + print(f"\nERROR: {err}") + sys.exit(1) + + return listing + + +def pre_check(project_path: Path) -> tuple[VersionDetails, VersionDetails]: + """Check not trying to overwrite the V4 Senzing system install, that path is a project, and versions are correct""" + if not V4_SYS_PATH.exists(): + print(f"\nERROR: Couldn't locate Senzing SDK V4 system install at {V4_SYS_PATH}") + sys.exit(1) + + if str(project_path).startswith(str(V4_SYS_PATH.parent)): + print(f"\nERROR: {project_path} is the Senzing system installation, not a Senzing project") + sys.exit(1) + + v3_project_build_file = project_path / V3_BUILD + v4_project_build_file = project_path / V4_BUILD + proj_listing = dir_listing(project_path) + if v3_project_build_file not in proj_listing and v4_project_build_file not in proj_listing: + print(f"\nERROR: {project_path} isn't a Senzing project, expected it to contain either:") + print(f"\tExisting V3 project - {v3_project_build_file}") + print(f"\tExisting V4 project - {v4_project_build_file}") + sys.exit(1) + + try: + proj_build_file = v3_project_build_file if v3_project_build_file in proj_listing else v4_project_build_file + # proj_version, proj_version_tokens = get_build_version(proj_build_file) + # sys_version, sys_version_tokens = get_build_version(V4_SYS_BUILD) + proj_version_details = get_build_version(proj_build_file) + sys_version_details = get_build_version(V4_SYS_BUILD) + except OSError as err: + print(f"\nERROR: Trying to read {proj_build_file} or {V4_SYS_BUILD} to collect version information: {err}") + sys.exit(1) + + # if proj_version_tokens.major not in (3, 4) or sys_version_tokens.major != 4: + if proj_version_details.major not in (3, 4) or sys_version_details.major != 4: + print(f"\nERROR: {MODULE_NAME} updates a V3 project to V4, or V4.n.n to a newer release") + print(f"\tProject version: {proj_version_details.major}, system install version: {sys_version_details.major}") + sys.exit(1) + + if proj_version_details.major == 4 and sys_version_details.sum_mmp == proj_version_details.sum_mmp: + print(f"No update required, project and system install are the same version - {proj_version_details.version}") + sys.exit(0) + + return (proj_version_details, sys_version_details) + + +# TODO - Move to helpers? +def get_build_version(path: Path) -> VersionDetails: + """Return the version string and major, minor, and patch versions from a build file""" + err_msg = f"ERROR: Couldn't get the version information from {path}" + + try: + with open(path, "r", encoding="utf-8") as f: + version_str: str = json.load(f)["VERSION"] + except (OSError, json.JSONDecodeError) as err: + print(f"\n{err_msg}: {err}") + sys.exit(1) + except KeyError as err: + print(f"\n{err_msg}, missing key {err}") + sys.exit(1) + + if not (str_tokens := version_str.split(".")) or (str_tokens and str_tokens[0] == ""): + print(f"\n{err_msg}, VERSION was blank") + sys.exit(1) + + if len(str_tokens) != 3: + print(f"\nERROR: Version information should consist of major, minor, and patch, it is {version_str}") + sys.exit(1) + + try: + # TODO - + # vers_tokens = VersionDetails._make([int(v) for v in str_tokens]) + # int_tokens = [int(v) for v in str_tokens] + return VersionDetails(version_str, *[int(v) for v in str_tokens]) + # vers_tokens = VersionDetails(version_str, str_tokens) + except ValueError: + print(f"\nERROR: Version information should consist of integers, it is {version_str}") + sys.exit(1) + + # TODO - + # return (version_str, vers_details) + + +def remove_dir(dir_: Path, excludes: list[Path]) -> None: + """Recursively remove a directory""" + try: + for path in dir_.iterdir(): + if path in excludes: + continue + path.unlink(missing_ok=True) if (path.is_file() or path.is_symlink()) else remove_dir(path, excludes) + + if not excludes: + with suppress(FileNotFoundError): + dir_.rmdir() + except OSError as err: + raise err + + +def remove_files_dirs(to_remove: dict[str, Any], target_dir: Path) -> None: + """Remove files/directories that are no longer required""" + for r_path, r_dict in to_remove.items(): + target: Path = target_dir / r_path + files = r_dict["files"] + excludes = [target / e for e in r_dict["excludes"]] + + try: + if target.is_file() or target.is_symlink(): + target.unlink(missing_ok=True) + + if target.is_dir() and not files or (files and files[0] == "*"): + remove_dir(target, excludes) + + if target.is_dir() and files and files[0] != "*": + target_files = [] + for f in files: + target_files.append(target / f) if "*" not in f else target_files.extend(list(target.glob(f))) + + for target_file in target_files: + target_file.unlink(missing_ok=True) + except OSError as err: + raise OSError(f"ERROR: Couldn't delete a file or directory: {err}") from err + + +def copy_files_dirs(to_copy: dict[str, Any], source_dir: Path, target_dir: Path) -> None: + # def copy_files_dirs(to_copy: dict[str, Any], source_dir: Path, target_dir: Path, sz_sys_path: Path) -> None: + """Copy files/directories within and to a project""" + # TODO - + print(f"\n{to_copy = }", flush=True) + print(f"\n{source_dir = }", flush=True) + print(f"\n{target_dir = }", flush=True) + # print(f"\n{sz_sys_path = }", flush=True) + for c_path, c_dict in to_copy.items(): + excludes = c_dict["excludes"] + files = c_dict["files"] + source: Path = source_dir / c_path + target: Path = target_dir / c_path + + if c_path.startswith("/"): + source = Path(c_path) + try: + # TODO - + # target = target_dir / source.relative_to(SZ_SYS_PATH) + # target = target_dir / source.relative_to(sz_sys_path) + target = target_dir / source.relative_to(V4_SYS_PATH) + except ValueError: + target = target_dir / target.name + + try: + if source.is_file(): + shutil.copy( + source, + target, + ) + + if source.is_dir() and not files or (files and files[0] == "*"): + shutil.copytree(source, target, ignore=shutil.ignore_patterns(*excludes), dirs_exist_ok=True) + + if source.is_dir() and files and files[0] != "*": + target.mkdir(exist_ok=True, parents=True) + for source_file in [source / f for f in files]: + shutil.copy(source_file, target / source_file.name) + except OSError as err: + raise OSError(f"ERROR: Couldn't copy a file or directory: {err}") from err + + +def rename_files(to_rename: dict[str, dict[str, str]], target_dir: Path) -> None: + """Rename existing project files that had a name change""" + + try: + for r_path, r_dict in to_rename.items(): + current = target_dir / r_path / r_dict["from"] + new = target_dir / r_path / r_dict["to"] + with suppress(FileNotFoundError): + current.rename(new) + except OSError as err: + raise OSError(f"ERROR: Couldn't rename a file or directory: {err}") from err + + +def setup_env(proj_path: Path) -> None: + """Create a new setupEnv and replace place holders with paths for the project""" + try: + shutil.copy(proj_path / "resources/templates/setupEnv", proj_path) + setup_path = proj_path / "setupEnv" + + with open(setup_path, "r", encoding="utf-8") as in_: + data = in_.read() + + data = data.replace("${SENZING_DIR}", str(proj_path)).replace("${SENZING_CONFIG_PATH}", str(proj_path / "etc")) + + with open(setup_path, "w", encoding="utf-8") as out: + out.write(data) + except OSError as err: + raise OSError(f"ERROR: Couldn't create a new setupEnv file: {err}") from err + + +def set_permissions(proj_path: Path, permissions: dict[str, dict[str, Any]]) -> None: + """Set permissions for files/dirs copied to the project, or dirs removed and replaced completely e.g., data/""" + try: + for p_path, p_dict in permissions.items(): + dir_pint = p_dict["dir_pint"] + file_pint = p_dict["file_pint"] + files = p_dict["files"] + recursive = p_dict["recursive"] + target = proj_path if p_path.startswith(".") else proj_path / p_path + excludes = [target / e for e in p_dict["excludes"]] + + if target.is_file(): + target.chmod(file_pint) + + if target.is_dir() and dir_pint != 0: + target.chmod(dir_pint) + d_chmods = ( + [d for d in target.glob("*") if d.is_dir() and not d.is_symlink() and d not in excludes] + if not recursive + else [d for d in target.rglob("*") if d.is_dir() and not d.is_symlink() and d not in excludes] + ) + for dir_ in d_chmods: + dir_.chmod(dir_pint) + + if target.is_dir() and (files and files[0] == "*"): + f_chmods = ( + [f for f in target.glob("*") if f.is_file() and not f.is_symlink() and f not in excludes] + if not recursive + else [f for f in target.rglob("*") if f.is_file() and not f.is_symlink() and f not in excludes] + ) + for file in f_chmods: + Path(target / file).chmod(file_pint) + + if target.is_dir() and files and files[0] != "*": + for file in files: + Path(target / file).chmod(file_pint) + except OSError as err: + raise OSError(f"ERROR: Couldn't set a permission: {err}") from err + + +def main() -> None: + """main""" + cli_args = parse_cli_args() + proj_path = Path(cli_args.project_path).resolve() + proj_versions, sys_versions = pre_check(proj_path) + proj_is_v3 = bool(proj_versions.major == 3) + # sz_sys_path = V3_SYS_PATH if proj_is_v3 else V4_SYS_PATH + + if not cli_args.force_mode: + print(f"\nWARNING: If you don't have a backup of the project ({proj_path}), create one before continuing!") + sleep(3) + if ( + input( + f"\nContinue updating the project from version {proj_versions.version} to {sys_versions.version}? (y/n) " + ) + not in INPUT_CONFS + ): + sys.exit(0) + print("\nUpdating...") + else: + print(f"\nUpdating project from version {proj_versions.version} to {sys_versions.version}...") + + try: + if proj_is_v3: + v3_backup_path = proj_path / V3_BACKUP_PATH + v3_backup_path.mkdir(exist_ok=True) + # copy_files_dirs( + # V3_BACKUP_PROJ, proj_path, v3_backup_path, sz_sys_path + # ) # Backup some of the V3 project files + copy_files_dirs(V3_BACKUP_PROJ, proj_path, v3_backup_path) # Backup some of the V3 project files + rename_files(V3_RENAME_IN_PROJ, proj_path) + remove_files_dirs(V3_REMOVE_FROM_PROJ, proj_path) + + # rename_files(V3_RENAME_IN_PROJ, proj_path) + # remove_files_dirs(V3_REMOVE_FROM_PROJ, proj_path) + # copy_files_dirs(COPY_TO_PROJ, SZ_SYS_PATH, proj_path) + # copy_files_dirs(COPY_TO_PROJ, sz_sys_path, proj_path, sz_sys_path) + # copy_files_dirs(COPY_TO_PROJ, sz_sys_path, proj_path, V4_SYS_PATH) + copy_files_dirs(COPY_TO_PROJ, V4_SYS_PATH, proj_path) + # setup_env(proj_path) + if proj_is_v3: + setup_env(proj_path) + set_permissions(proj_path, V3_RESET_PERMISSIONS) + except OSError as err: + shutil.copytree(v3_backup_path, proj_path, dirs_exist_ok=True) + print(f"\n{err}") + print( + "\nIf the error is file or directory permission related, run again with a user with appropriate privileges" + ) + print( + "\nIf the error is missing file or directory, check senzingsdk-setup, senzingsdk-tools, and senzingsdk-poc are installed" + ) + else: + if proj_is_v3: # Remove if no errors, otherwise re-running can find the file + # proj_path.joinpath(PROJ_BUILD).unlink(missing_ok=True) + proj_path.joinpath(V3_BUILD).unlink(missing_ok=True) + print(f"\nProject successfully updated. Refer to {UPGRADE_URL} for additional upgrade instructions") + + +if __name__ == "__main__": + main() diff --git a/sz_tools/sz_update_project_prior3 b/sz_tools/sz_update_project_prior3 new file mode 100755 index 0000000..0341c1f --- /dev/null +++ b/sz_tools/sz_update_project_prior3 @@ -0,0 +1,478 @@ +#! /usr/bin/env python3 +"""Upgrade a V3 or V4 Senzing SDK project to V4.x.x""" + +import argparse +import json +import shutil +import sys +from contextlib import suppress +from dataclasses import dataclass, field +from pathlib import Path +from time import sleep +from typing import Any + +from packaging import version as p_version + +INPUT_CONFS = ("y", "Y", "yEs", "yES", "YES", "yes", "YEs", "yeS", "Yes", "YeS") +MODULE_NAME = Path(__file__).stem +V3_BACKUP_PATH = "v3_upgrade_backups" +V3_BUILD = "g2BuildVersion.json" +V4_BUILD = "szBuildVersion.json" +V3_SYS_PATH = Path("/opt/senzing/g2") +V4_SYS_PATH = Path("/opt/senzing/er") +V3_SYS_BUILD = V3_SYS_PATH / V3_BUILD +V4_SYS_BUILD = V4_SYS_PATH / V4_BUILD +UPGRADE_URL = "https://www.senzing.com/docs/4_beta/index.html" + +V3_BACKUP_PROJ = { + "bin": {"files": ["g2dbencrypt", "g2saltadm", "g2ssadm"], "excludes": []}, + "etc": {"files": ["senzing_governor.py"], "excludes": []}, + "lib": {"files": ["libSpaceTimeBoxStandardizer.so", "libG2Hasher.so", "libG2SSAdm.so"], "excludes": []}, + "python": {"files": [], "excludes": []}, + "resources/templates": {"files": ["setupEnv"], "excludes": []}, + "setupEnv": {"files": [], "excludes": []}, + "g2BuildVersion.json": {"files": [], "excludes": []}, +} + +V3_REMOVE_FROM_PROJ = { + "bin": {"files": ["*"], "excludes": ["bin"]}, + "data": {"files": [], "excludes": []}, + "etc": {"files": ["senzing_governor.py"], "excludes": []}, + "lib": { + "files": ["*"], + # "files": [ + # "g2.jar", + # "libG2.so", + # "libG2Hasher.so", + # "libG2SSAdm.so", + # "libg2CompJavaScoreSet.so", + # "libg2DistinctFeatJava.so", + # "libg2EFeatJava.so", + # "libg2JVMPlugin.so", + # "libg2StdJava.so", + # "libmariadbplugin.so", + # "libSpaceTimeBoxStandardizer.so", + # ], + "excludes": ["lib"], + }, + "python": {"files": ["*"], "excludes": []}, + "resources/config": {"files": ["g2core-configuration-upgrade-*.gtc"], "excludes": []}, + "resources/schema": {"files": ["g2core-schema-*-create.sql", "g2core-schema-*-upgrade-*.sql"], "excludes": []}, + "resources/templates": { + "files": [ + "G2C.db*", + "G2Module.ini", + "custom*.txt", + "defaultGNRCP.config", + "g2config.json", + "senzing_governor.py", + "setupEnv", + "stb.config", + ], + "excludes": [], + }, + "sdk": {"files": ["*"], "excludes": ["sdk"]}, + "setupEnv": {"files": [], "excludes": []}, +} + +COPY_TO_PROJ = { + "LICENSE": {"files": [], "excludes": []}, + "NOTICES": {"files": [], "excludes": []}, + "README.1ST": {"files": [], "excludes": []}, + "bin": {"files": ["*"], "excludes": ["sz_create_project", "sz_update_project"]}, + "/opt/senzing/data": {"files": ["*"], "excludes": []}, + "lib": {"files": ["*"], "excludes": []}, + "resources": {"files": ["*"], "excludes": []}, + "sdk": {"files": ["*"], "excludes": []}, + "szBuildVersion.json": {"files": [], "excludes": []}, +} + +V3_RENAME_IN_PROJ = { + "etc": {"from": "G2Module.ini", "to": "sz_engine_config.ini"}, +} + +# TODO - Rename and check +SET_PERMISSIONS = { + ".": { + "dir_pint": 0, + "file_pint": 0o660, + "files": ["LICENSE", "NOTICES", "README.1ST", "szBuildVersion.json"], + "excludes": ["setupEnv"], + "recursive": False, + }, + "setupEnv": {"dir_pint": 0, "file_pint": 0o770, "files": [], "excludes": [], "recursive": False}, + "bin": { + "dir_pint": 0o770, + "file_pint": 0o770, + "files": ["*"], + "excludes": ["__pycache__", "_sz_database.py", "_tool_helpers.py"], + "recursive": False, + }, + "data": {"dir_pint": 0o770, "file_pint": 0o660, "files": ["*"], "excludes": [], "recursive": True}, + "lib": {"dir_pint": 0o770, "file_pint": 0o660, "files": ["*"], "excludes": [], "recursive": False}, + "resources": {"dir_pint": 0o770, "file_pint": 0o660, "files": ["*"], "excludes": ["setupEnv"], "recursive": True}, + "resources/templates/setupEnv": { + "dir_pint": 0, + "file_pint": 0o770, + "files": [], + "excludes": [], + "recursive": False, + }, + "sdk": {"dir_pint": 0o770, "file_pint": 0o660, "files": ["*"], "excludes": [], "recursive": True}, + "v3_upgrade_backups": {"dir_pint": 0o770, "file_pint": 0, "files": [], "excludes": [], "recursive": False}, +} + + +@dataclass() +class BuildDetails: + """Build information for a project or Senzing SDK system install""" + + # version: str + # # TODO - + # version_parsed: p_version.Version + # major: int + # minor: int + # patch: int + # build: int + platform: str + version: str + build_version: str + build_number: str + major: int = field(init=False) + minor: int = field(init=False) + micro: int = field(init=False) + + def __post_init__(self) -> None: + # self.sum_mmp = sum((self.major, self.minor, self.patch)) + self.version_parsed = p_version.parse(self.version) + self.build_version_parsed = p_version.parse(self.build_version) + self.major = self.version_parsed.major + self.minor = self.version_parsed.minor + self.micro = self.version_parsed.micro + + +# pylint: disable=W0106 + + +def parse_cli_args() -> argparse.Namespace: + """Parse the CLI arguments""" + arg_parser = argparse.ArgumentParser( + allow_abbrev=False, + description="Update an existing V3 or V4 Senzing project to the system installed V4 of Senzing.", + ) + arg_parser.add_argument( + "project_path", + metavar="path", + help="Path of the project to update", + ) + arg_parser.add_argument( + "-f", + "--force", + dest="force_mode", + default=False, + action="store_true", + help="Upgrade without prompts. WARNING: Use with caution, ensure you have a backup of the project", + ) + return arg_parser.parse_args() + + +def dir_listing(path: Path) -> list[Path]: + """Get a listing of the specified path to check for files or directories""" + try: + listing = list(path.iterdir()) + except OSError as err: + if type(err).__name__ == "NotADirectoryError": + print(f"\nERROR: {path} is not a directory, expecting one") + print(f"\nERROR: {err}") + sys.exit(1) + + return listing + + +def pre_check(project_path: Path) -> tuple[BuildDetails, BuildDetails]: + """Check not trying to overwrite the V4 Senzing system install, that path is a project, and versions are correct""" + if not V4_SYS_PATH.exists(): + print(f"\nERROR: Couldn't locate Senzing SDK V4 system install at {V4_SYS_PATH}") + sys.exit(1) + + if str(project_path).startswith(str(V4_SYS_PATH.parent)): + print(f"\nERROR: {project_path} is the Senzing system installation, not a Senzing project") + sys.exit(1) + + v3_project_build_file = project_path / V3_BUILD + v4_project_build_file = project_path / V4_BUILD + proj_listing = dir_listing(project_path) + if v3_project_build_file not in proj_listing and v4_project_build_file not in proj_listing: + print(f"\nERROR: {project_path} isn't a Senzing project, expected it to contain either:") + print(f"\tExisting V3 project: {v3_project_build_file}") + print(f"\tExisting V4 project: {v4_project_build_file}") + sys.exit(1) + + try: + proj_build_file = v3_project_build_file if v3_project_build_file in proj_listing else v4_project_build_file + proj_version_details = get_build_details(proj_build_file) + sys_version_details = get_build_details(V4_SYS_BUILD) + except OSError as err: + print(f"\nERROR: Trying to read {proj_build_file} or {V4_SYS_BUILD} to collect version information: {err}") + sys.exit(1) + + if proj_version_details.major not in (3, 4) or sys_version_details.major != 4: + print(f"\nERROR: {MODULE_NAME} updates a V3 project to V4, or V4.n.n to a newer release") + print(f"\tProject version: {proj_version_details.version}") + print(f"\tSystem install version: {sys_version_details.version}") + sys.exit(1) + + # TODO - + # if sys_version_details.sum_mmp < proj_version_details.sum_mmp: + if sys_version_details.version_parsed < proj_version_details.version_parsed: + print("\nNo update required, project is newer than the system install") + print(f"\tProject version: {proj_version_details.version}") + print(f"\tSystem install version: {sys_version_details.version}") + sys.exit(0) + + # TODO - + # if proj_version_details.major == 4 and sys_version_details.sum_mmp == proj_version_details.sum_mmp: + if proj_version_details.major == 4 and sys_version_details.version_parsed == proj_version_details.version_parsed: + print(f"\nNo update required, project and system install are the same version - {proj_version_details.version}") + sys.exit(0) + + return (proj_version_details, sys_version_details) + + +# TODO - Move to helpers +def get_build_details(path: Path) -> BuildDetails: + """Return dataclass with version string, major, minor, and patch versions from a build file""" + err_msg = f"ERROR: Couldn't get the version information from {path}" + + try: + with open(path, "r", encoding="utf-8") as f: + # TODO - + # version_str: str = json.load(f)["VERSION"] + # version_str: str = json.load(f)["BUILD_VERSION"] + version_dict: Any = json.load(f) + version_dict = {k.lower(): v for k, v in version_dict.items() if k != "DATA_VERSION"} + # TODO - + print(f"\n{version_dict = }", flush=True) + except (OSError, json.JSONDecodeError) as err: + print(f"\n{err_msg}: {err}") + sys.exit(1) + # except KeyError as err: + # print(f"\n{err_msg}, missing key {err}") + # sys.exit(1) + + # if not (str_tokens := version_str.split(".")) or (str_tokens and str_tokens[0] == ""): + # # TODO - + # # print(f"\n{err_msg}, VERSION was blank") + # print(f"\n{err_msg}, BUILD_VERSION was blank or misformed") + # sys.exit(1) + + # # TODO - + # # if len(str_tokens) != 3: + # if len(str_tokens) != 4: + # print(f"\nERROR: Version information should consist of major, minor, patch, and build, it is {version_str}") + # sys.exit(1) + + try: + # TODO - + # return VersionDetails(version_str, *[int(v) for v in str_tokens]) + # return VersionDetails(version_str, p_version.parse(version_str), *[int(v) for v in str_tokens]) + return BuildDetails(**version_dict) + except ValueError: + # print(f"\nERROR: Version information should consist of integers, it is {version_str}") + sys.exit(1) + + +def remove_dir(dir_: Path, excludes: list[Path]) -> None: + """Recursively remove a directory""" + try: + for path in dir_.iterdir(): + if path in excludes: + continue + path.unlink(missing_ok=True) if (path.is_file() or path.is_symlink()) else remove_dir(path, excludes) + + if not excludes: + with suppress(FileNotFoundError): + dir_.rmdir() + except OSError as err: + raise err + + +def remove_files_dirs(to_remove: dict[str, Any], target_dir: Path) -> None: + """Remove files/directories that are no longer required""" + for r_path, r_dict in to_remove.items(): + target: Path = target_dir / r_path + files = r_dict["files"] + excludes = [target / e for e in r_dict["excludes"]] + + try: + if target.is_file() or target.is_symlink(): + target.unlink(missing_ok=True) + + if target.is_dir() and not files or (files and files[0] == "*"): + remove_dir(target, excludes) + + if target.is_dir() and files and files[0] != "*": + target_files = [] + for f in files: + target_files.append(target / f) if "*" not in f else target_files.extend(list(target.glob(f))) + + for target_file in target_files: + target_file.unlink(missing_ok=True) + except OSError as err: + raise OSError(f"ERROR: Couldn't delete a file or directory: {err}") from err + + +def copy_files_dirs(to_copy: dict[str, Any], source_dir: Path, target_dir: Path) -> None: + """Copy files/directories within and to a project""" + for c_path, c_dict in to_copy.items(): + excludes = c_dict["excludes"] + files = c_dict["files"] + source: Path = source_dir / c_path + target: Path = target_dir / c_path + + if c_path.startswith("/"): + source = Path(c_path) + try: + target = target_dir / source.relative_to(V4_SYS_PATH) + except ValueError: + target = target_dir / target.name + + try: + if source.is_file(): + shutil.copy( + source, + target, + ) + + if source.is_dir() and not files or (files and files[0] == "*"): + shutil.copytree(source, target, ignore=shutil.ignore_patterns(*excludes), dirs_exist_ok=True) + + if source.is_dir() and files and files[0] != "*": + target.mkdir(exist_ok=True, parents=True) + for source_file in [source / f for f in files]: + shutil.copy(source_file, target / source_file.name) + except OSError as err: + raise OSError(f"ERROR: Couldn't copy a file or directory: {err}") from err + + +def rename_files(to_rename: dict[str, dict[str, str]], target_dir: Path) -> None: + """Rename existing project files that had a name change""" + + try: + for r_path, r_dict in to_rename.items(): + current = target_dir / r_path / r_dict["from"] + new = target_dir / r_path / r_dict["to"] + with suppress(FileNotFoundError): + current.rename(new) + except OSError as err: + raise OSError(f"ERROR: Couldn't rename a file or directory: {err}") from err + + +def setup_env(proj_path: Path) -> None: + """Create a new setupEnv and replace place holders with paths for the project""" + try: + shutil.copy(proj_path / "resources/templates/setupEnv", proj_path) + setup_path = proj_path / "setupEnv" + + with open(setup_path, "r", encoding="utf-8") as in_: + data = in_.read() + + data = data.replace("${SENZING_DIR}", str(proj_path)).replace("${SENZING_CONFIG_PATH}", str(proj_path / "etc")) + + with open(setup_path, "w", encoding="utf-8") as out: + out.write(data) + except OSError as err: + raise OSError(f"ERROR: Couldn't create a new setupEnv file: {err}") from err + + +def set_permissions(proj_path: Path, permissions: dict[str, dict[str, Any]]) -> None: + """Set permissions for files/dirs copied to the project, or dirs removed and replaced completely e.g., data/""" + try: + for p_path, p_dict in permissions.items(): + dir_pint = p_dict["dir_pint"] + file_pint = p_dict["file_pint"] + files = p_dict["files"] + recursive = p_dict["recursive"] + target = proj_path if p_path.startswith(".") else proj_path / p_path + excludes = [target / e for e in p_dict["excludes"]] + + if target.is_file(): + target.chmod(file_pint) + + if target.is_dir() and dir_pint != 0: + target.chmod(dir_pint) + d_chmods = ( + [d for d in target.glob("*") if d.is_dir() and not d.is_symlink() and d not in excludes] + if not recursive + else [d for d in target.rglob("*") if d.is_dir() and not d.is_symlink() and d not in excludes] + ) + for dir_ in d_chmods: + dir_.chmod(dir_pint) + + if target.is_dir() and (files and files[0] == "*"): + f_chmods = ( + [f for f in target.glob("*") if f.is_file() and not f.is_symlink() and f not in excludes] + if not recursive + else [f for f in target.rglob("*") if f.is_file() and not f.is_symlink() and f not in excludes] + ) + for file in f_chmods: + Path(target / file).chmod(file_pint) + + if target.is_dir() and files and files[0] != "*": + for file in files: + Path(target / file).chmod(file_pint) + except OSError as err: + raise OSError(f"ERROR: Couldn't set a permission: {err}") from err + + +def main() -> None: + """main""" + cli_args = parse_cli_args() + proj_path = Path(cli_args.project_path).resolve() + proj_build, sys_build = pre_check(proj_path) + print(f"\n{proj_build = }", flush=True) + print(f"\n{proj_build.version_parsed = } - {proj_build.major}", flush=True) + sys.exit() + proj_is_v3 = bool(proj_build.major == 3) + + if not cli_args.force_mode: + print("\nWARNING: If you don't have a backup of the project, create one before continuing!") + sleep(3) + if ( + input(f"\nUpdate the project from version {proj_build.version} to {sys_build.version}? (y/n) ") + not in INPUT_CONFS + ): + sys.exit(0) + print("\nUpdating...") + else: + print(f"\nUpdating project from version {proj_build.version} to {sys_build.version}...") + + try: + if proj_is_v3: + v3_backup_path = proj_path / V3_BACKUP_PATH + v3_backup_path.mkdir(exist_ok=True) + copy_files_dirs(V3_BACKUP_PROJ, proj_path, v3_backup_path) # Backup some of the V3 project files + rename_files(V3_RENAME_IN_PROJ, proj_path) + remove_files_dirs(V3_REMOVE_FROM_PROJ, proj_path) + + copy_files_dirs(COPY_TO_PROJ, V4_SYS_PATH, proj_path) + if proj_is_v3: + setup_env(proj_path) + # TODO - Check V4 permissions are good, is sz_create_project good in the first place when V4 to V4? + set_permissions(proj_path, SET_PERMISSIONS) + except OSError as err: + shutil.copytree(v3_backup_path, proj_path, dirs_exist_ok=True) + print(f"\n{err}") + print("\nIf the error is file or directory permission related, run again with appropriate privileges") + print( + "\nIf the error is missing file or directory, check senzingsdk-setup, senzingsdk-tools, and senzingsdk-poc are installed" + ) + else: + if proj_is_v3: # Only delete if no errors so re-running can find the file again + proj_path.joinpath(V3_BUILD).unlink(missing_ok=True) + + print(f"\nProject successfully updated. Refer to {UPGRADE_URL} for additional upgrade instructions") + + +if __name__ == "__main__": + main() diff --git a/sz_tools/sz_update_project_prior4 b/sz_tools/sz_update_project_prior4 new file mode 100755 index 0000000..9cf51f8 --- /dev/null +++ b/sz_tools/sz_update_project_prior4 @@ -0,0 +1,456 @@ +#! /usr/bin/env python3 +"""Upgrade a V3 or V4 Senzing SDK project to V4.x.x""" + +import argparse +import json +import shutil +import sys +from contextlib import suppress +from dataclasses import dataclass, field +from pathlib import Path +from time import sleep +from typing import Any + +from packaging import version as p_version + +INPUT_CONFS = ("y", "Y", "yEs", "yES", "YES", "yes", "YEs", "yeS", "Yes", "YeS") +MODULE_NAME = Path(__file__).stem +V3_BACKUP_PATH = "v3_upgrade_backups" +V3_BUILD = "g2BuildVersion.json" +V4_BUILD = "szBuildVersion.json" +V3_SYS_PATH = Path("/opt/senzing/g2") +V4_SYS_PATH = Path("/opt/senzing/er") +V3_SYS_BUILD = V3_SYS_PATH / V3_BUILD +V4_SYS_BUILD = V4_SYS_PATH / V4_BUILD +UPGRADE_URL = "https://www.senzing.com/docs/4_beta/index.html" + +V3_BACKUP_PROJ = { + "bin": {"files": ["g2dbencrypt", "g2saltadm", "g2ssadm"], "excludes": []}, + "etc": {"files": ["senzing_governor.py"], "excludes": []}, + "lib": {"files": ["libSpaceTimeBoxStandardizer.so", "libG2Hasher.so", "libG2SSAdm.so"], "excludes": []}, + "python": {"files": [], "excludes": []}, + "resources/templates": {"files": ["setupEnv"], "excludes": []}, + "setupEnv": {"files": [], "excludes": []}, + "g2BuildVersion.json": {"files": [], "excludes": []}, +} + +V3_REMOVE_FROM_PROJ = { + "bin": {"files": ["*"], "excludes": ["bin"]}, + "data": {"files": [], "excludes": []}, + "etc": {"files": ["senzing_governor.py"], "excludes": []}, + "lib": { + "files": ["*"], + # TODO - Others simplified? + # "files": [ + # "g2.jar", + # "libG2.so", + # "libG2Hasher.so", + # "libG2SSAdm.so", + # "libg2CompJavaScoreSet.so", + # "libg2DistinctFeatJava.so", + # "libg2EFeatJava.so", + # "libg2JVMPlugin.so", + # "libg2StdJava.so", + # "libmariadbplugin.so", + # "libSpaceTimeBoxStandardizer.so", + # ], + "excludes": ["lib"], + }, + "python": {"files": ["*"], "excludes": []}, + "resources/config": {"files": ["g2core-configuration-upgrade-*.gtc"], "excludes": []}, + "resources/schema": {"files": ["g2core-schema-*-create.sql", "g2core-schema-*-upgrade-*.sql"], "excludes": []}, + "resources/templates": { + "files": [ + "G2C.db*", + "G2Module.ini", + "custom*.txt", + "defaultGNRCP.config", + "g2config.json", + "senzing_governor.py", + "setupEnv", + "stb.config", + ], + "excludes": [], + }, + "sdk": {"files": ["*"], "excludes": ["sdk"]}, + "setupEnv": {"files": [], "excludes": []}, +} + +COPY_TO_PROJ = { + "LICENSE": {"files": [], "excludes": []}, + "NOTICES": {"files": [], "excludes": []}, + "README.1ST": {"files": [], "excludes": []}, + "bin": {"files": ["*"], "excludes": ["sz_create_project", "sz_update_project"]}, + # TODO - + "/opt/senzing/data": {"files": ["*"], "excludes": []}, + "lib": {"files": ["*"], "excludes": []}, + "resources": {"files": ["*"], "excludes": []}, + "sdk": {"files": ["*"], "excludes": []}, + "szBuildVersion.json": {"files": [], "excludes": []}, +} + +V3_RENAME_IN_PROJ = { + "etc": {"from": "G2Module.ini", "to": "sz_engine_config.ini"}, +} + +# TODO - Rename and check +SET_PERMISSIONS = { + ".": { + "dir_pint": 0, + "file_pint": 0o660, + "files": ["LICENSE", "NOTICES", "README.1ST", "szBuildVersion.json"], + "excludes": ["setupEnv"], + "recursive": False, + }, + "setupEnv": {"dir_pint": 0, "file_pint": 0o770, "files": [], "excludes": [], "recursive": False}, + "bin": { + "dir_pint": 0o770, + "file_pint": 0o770, + "files": ["*"], + "excludes": ["__pycache__", "_sz_database.py", "_tool_helpers.py"], + "recursive": False, + }, + "data": {"dir_pint": 0o770, "file_pint": 0o660, "files": ["*"], "excludes": [], "recursive": True}, + "lib": {"dir_pint": 0o770, "file_pint": 0o660, "files": ["*"], "excludes": [], "recursive": False}, + "resources": {"dir_pint": 0o770, "file_pint": 0o660, "files": ["*"], "excludes": ["setupEnv"], "recursive": True}, + "resources/templates/setupEnv": { + "dir_pint": 0, + "file_pint": 0o770, + "files": [], + "excludes": [], + "recursive": False, + }, + "sdk": {"dir_pint": 0o770, "file_pint": 0o660, "files": ["*"], "excludes": [], "recursive": True}, + "v3_upgrade_backups": {"dir_pint": 0o770, "file_pint": 0, "files": [], "excludes": [], "recursive": False}, +} + + +@dataclass() +class SzBuildDetails: + """Build information for a project or Senzing SDK system install""" + + platform: str + version: str + build_version: str + build_number: str + major: int = field(init=False) + minor: int = field(init=False) + micro: int = field(init=False) + + def __post_init__(self) -> None: + self.version_parsed = p_version.parse(self.version) + self.build_version_parsed = p_version.parse(self.build_version) + self.major = self.version_parsed.major + self.minor = self.version_parsed.minor + self.micro = self.version_parsed.micro + + +# pylint: disable=W0106 + + +def parse_cli_args() -> argparse.Namespace: + """Parse the CLI arguments""" + arg_parser = argparse.ArgumentParser( + allow_abbrev=False, + description="Update an existing V3 or V4 Senzing project to the system installed V4 of Senzing.", + ) + arg_parser.add_argument( + "project_path", + metavar="path", + help="Path of the project to update", + ) + arg_parser.add_argument( + "-f", + "--force", + dest="force_mode", + default=False, + action="store_true", + help="Upgrade without prompts. WARNING: Use with caution, ensure you have a backup of the project", + ) + return arg_parser.parse_args() + + +def dir_listing(path: Path) -> list[Path]: + """Get a listing of the specified path to check for files or directories""" + try: + listing = list(path.iterdir()) + except OSError as err: + if type(err).__name__ == "NotADirectoryError": + print(f"\nERROR: {path} is not a directory, expecting one") + print(f"\nERROR: {err}") + sys.exit(1) + + return listing + + +def pre_check(project_path: Path) -> tuple[SzBuildDetails, SzBuildDetails]: + """Check not trying to overwrite the V4 Senzing system install, that path is a project, and versions are correct""" + if not V4_SYS_PATH.exists(): + print(f"\nERROR: Couldn't locate Senzing SDK V4 system install at {V4_SYS_PATH}") + sys.exit(1) + + if str(project_path).startswith(str(V4_SYS_PATH.parent)): + print(f"\nERROR: {project_path} is the Senzing system installation, not a Senzing project") + sys.exit(1) + + v3_project_build_file = project_path / V3_BUILD + v4_project_build_file = project_path / V4_BUILD + proj_listing = dir_listing(project_path) + if v3_project_build_file not in proj_listing and v4_project_build_file not in proj_listing: + print(f"\nERROR: {project_path} isn't a Senzing project, expected it to contain either:") + print(f"\tExisting V3 project: {v3_project_build_file}") + print(f"\tExisting V4 project: {v4_project_build_file}") + sys.exit(1) + + try: + proj_build_file = v3_project_build_file if v3_project_build_file in proj_listing else v4_project_build_file + proj_build_details = get_build_details(proj_build_file) + sys_build_details = get_build_details(V4_SYS_BUILD) + except OSError as err: + print(f"\nERROR: Trying to read {proj_build_file} or {V4_SYS_BUILD} to collect version information: {err}") + sys.exit(1) + + if proj_build_details.major not in (3, 4) or sys_build_details.major != 4: + print(f"\nERROR: {MODULE_NAME} updates a V3 project to V4, or V4.n.n to a newer release") + print(f"\tProject version: {proj_build_details.version}") + print(f"\tSystem install version: {sys_build_details.version}") + sys.exit(1) + + # TODO - + if sys_build_details.version_parsed < proj_build_details.version_parsed: + print("\nNo update required, project is newer than the system install") + print(f"\tProject version: {proj_build_details.version}") + print(f"\tSystem install version: {sys_build_details.version}") + sys.exit(0) + + # TODO - + if sys_build_details.version_parsed == proj_build_details.version_parsed: + print(f"\nNo update required, project and system install are the same version - {proj_build_details.version}") + sys.exit(0) + + return (proj_build_details, sys_build_details) + + +# TODO - Move to helpers +def get_build_details(path: Path) -> SzBuildDetails: + """Return dataclass with the details from a build file.""" + try: + with open(path, "r", encoding="utf-8") as f: + # Ignore DATA_VERSION V3 build files had it, V4 doesn't + version_dict = {k.lower(): v for k, v in json.load(f).items() if k != "DATA_VERSION"} + return SzBuildDetails(**version_dict) + except (OSError, json.JSONDecodeError) as err: + print(f"\nERROR: Couldn't get the build information from {path}: {err}") + sys.exit(1) + except TypeError as err: + print(f"\nERROR: When creating build information from {path}: {err}") + sys.exit(1) + + +def remove_dir(dir_: Path, excludes: list[Path]) -> None: + """Recursively remove a directory""" + try: + for path in dir_.iterdir(): + if path in excludes: + continue + path.unlink(missing_ok=True) if (path.is_file() or path.is_symlink()) else remove_dir(path, excludes) + + if not excludes: + with suppress(FileNotFoundError): + dir_.rmdir() + except OSError as err: + raise err + + +def remove_files_dirs(to_remove: dict[str, Any], target_dir: Path) -> None: + """Remove files/directories that are no longer required""" + for r_path, r_dict in to_remove.items(): + target: Path = target_dir / r_path + files = r_dict["files"] + excludes = [target / e for e in r_dict["excludes"]] + + try: + if target.is_file() or target.is_symlink(): + target.unlink(missing_ok=True) + + if target.is_dir() and not files or (files and files[0] == "*"): + remove_dir(target, excludes) + + if target.is_dir() and files and files[0] != "*": + target_files = [] + for f in files: + target_files.append(target / f) if "*" not in f else target_files.extend(list(target.glob(f))) + + for target_file in target_files: + target_file.unlink(missing_ok=True) + except OSError as err: + raise OSError(f"ERROR: Couldn't delete a file or directory: {err}") from err + + +def copy_files_dirs(to_copy: dict[str, Any], source_dir: Path, target_dir: Path) -> None: + """Copy files/directories within and to a project""" + # TODO - + print(f"\n{to_copy = }", flush=True) + print(f"{source_dir = }", flush=True) + print(f"{target_dir = }", flush=True) + for c_path, c_dict in to_copy.items(): + print(f"\n{c_path = } - {c_dict}", flush=True) + excludes = c_dict["excludes"] + files = c_dict["files"] + source: Path = source_dir / c_path + target: Path = target_dir / c_path + print(f"\n{excludes = }", flush=True) + print(f"{files = }", flush=True) + print(f"{source = }", flush=True) + print(f"{target = }", flush=True) + + if c_path.startswith("/"): + print("\nStarts with/...", flush=True) + source = Path(c_path) + try: + print("here", flush=True) + print(f"\tTrying relative to: {target_dir = } - {source.relative_to(V4_SYS_PATH) = }", flush=True) + target = target_dir / source.relative_to(V4_SYS_PATH) + except ValueError: + print(f"\tTrying target.name: {target = } - {target_dir / target.name}", flush=True) + target = target_dir / target.name + print(f"\tNow: {source = }", flush=True) + print(f"\tNow: {target = }", flush=True) + + try: + if source.is_file(): + # TODO - remove shutil + shutil.copy( + source, + target, + ) + + if source.is_dir() and not files or (files and files[0] == "*"): + shutil.copytree(source, target, ignore=shutil.ignore_patterns(*excludes), dirs_exist_ok=True) + + if source.is_dir() and files and files[0] != "*": + target.mkdir(exist_ok=True, parents=True) + for source_file in [source / f for f in files]: + shutil.copy(source_file, target / source_file.name) + except OSError as err: + raise OSError(f"ERROR: Couldn't copy a file or directory: {err}") from err + + +def rename_files(to_rename: dict[str, dict[str, str]], target_dir: Path) -> None: + """Rename existing project files that had a name change""" + + try: + for r_path, r_dict in to_rename.items(): + current = target_dir / r_path / r_dict["from"] + new = target_dir / r_path / r_dict["to"] + with suppress(FileNotFoundError): + current.rename(new) + except OSError as err: + raise OSError(f"ERROR: Couldn't rename a file or directory: {err}") from err + + +def setup_env(proj_path: Path) -> None: + """Create a new setupEnv and replace place holders with paths for the project""" + try: + shutil.copy(proj_path / "resources/templates/setupEnv", proj_path) + setup_path = proj_path / "setupEnv" + + with open(setup_path, "r", encoding="utf-8") as in_: + data = in_.read() + + data = data.replace("${SENZING_DIR}", str(proj_path)).replace("${SENZING_CONFIG_PATH}", str(proj_path / "etc")) + + with open(setup_path, "w", encoding="utf-8") as out: + out.write(data) + except OSError as err: + raise OSError(f"ERROR: Couldn't create a new setupEnv file: {err}") from err + + +def set_permissions(proj_path: Path, permissions: dict[str, dict[str, Any]]) -> None: + """Set permissions for files/dirs copied to the project, or dirs removed and replaced completely e.g., data/""" + try: + for p_path, p_dict in permissions.items(): + dir_pint = p_dict["dir_pint"] + file_pint = p_dict["file_pint"] + files = p_dict["files"] + recursive = p_dict["recursive"] + target = proj_path if p_path.startswith(".") else proj_path / p_path + excludes = [target / e for e in p_dict["excludes"]] + + if target.is_file(): + target.chmod(file_pint) + + if target.is_dir() and dir_pint != 0: + target.chmod(dir_pint) + d_chmods = ( + [d for d in target.glob("*") if d.is_dir() and not d.is_symlink() and d not in excludes] + if not recursive + else [d for d in target.rglob("*") if d.is_dir() and not d.is_symlink() and d not in excludes] + ) + for dir_ in d_chmods: + dir_.chmod(dir_pint) + + if target.is_dir() and (files and files[0] == "*"): + f_chmods = ( + [f for f in target.glob("*") if f.is_file() and not f.is_symlink() and f not in excludes] + if not recursive + else [f for f in target.rglob("*") if f.is_file() and not f.is_symlink() and f not in excludes] + ) + for file in f_chmods: + Path(target / file).chmod(file_pint) + + if target.is_dir() and files and files[0] != "*": + for file in files: + Path(target / file).chmod(file_pint) + except OSError as err: + raise OSError(f"ERROR: Couldn't set a permission: {err}") from err + + +def main() -> None: + """main""" + cli_args = parse_cli_args() + proj_path = Path(cli_args.project_path).resolve() + proj_build, sys_build = pre_check(proj_path) + proj_is_v3 = bool(proj_build.major == 3) + + if not cli_args.force_mode: + print("\nWARNING: If you don't have a backup of the project, create one before continuing!") + sleep(3) + if ( + input(f"\nUpdate the project from version {proj_build.build_version} to {sys_build.build_version}? (y/n) ") + not in INPUT_CONFS + ): + sys.exit(0) + print("\nUpdating...") + else: + print(f"\nUpdating project from version {proj_build.build_version} to {sys_build.build_version}...") + + try: + if proj_is_v3: + v3_backup_path = proj_path / V3_BACKUP_PATH + v3_backup_path.mkdir(exist_ok=True) + copy_files_dirs(V3_BACKUP_PROJ, proj_path, v3_backup_path) # Backup some of the V3 project files + rename_files(V3_RENAME_IN_PROJ, proj_path) + remove_files_dirs(V3_REMOVE_FROM_PROJ, proj_path) + + copy_files_dirs(COPY_TO_PROJ, V4_SYS_PATH, proj_path) + if proj_is_v3: + setup_env(proj_path) + # TODO - Check V4 permissions are good, is sz_create_project good in the first place when V4 to V4? + set_permissions(proj_path, SET_PERMISSIONS) + except OSError as err: + shutil.copytree(v3_backup_path, proj_path, dirs_exist_ok=True) + print(f"\n{err}") + print("\nIf the error is file or directory permission related, run again with appropriate privileges") + print( + "\nIf the error is missing file or directory, check senzingsdk-setup, senzingsdk-tools, and senzingsdk-poc are installed" + ) + else: + if proj_is_v3: # Only delete if no errors so re-running can find the file again + proj_path.joinpath(V3_BUILD).unlink(missing_ok=True) + + print(f"\nProject successfully updated. Refer to {UPGRADE_URL} for additional upgrade instructions") + + +if __name__ == "__main__": + main() diff --git a/sz_tools/sz_update_project_prior5 b/sz_tools/sz_update_project_prior5 new file mode 100755 index 0000000..cb4b3ef --- /dev/null +++ b/sz_tools/sz_update_project_prior5 @@ -0,0 +1,515 @@ +#! /usr/bin/env python3 +"""Upgrade a V3 or V4 Senzing SDK project to V4.x.x""" + +import argparse +import json +import shutil +import sys +from contextlib import suppress +from dataclasses import dataclass, field +from pathlib import Path +from time import sleep +from typing import Any + +from packaging import version as p_version + +INPUT_CONFS = ("y", "Y", "yEs", "yES", "YES", "yes", "YEs", "yeS", "Yes", "YeS") +MODULE_NAME = Path(__file__).stem +V3_BACKUP_PATH = "v3_upgrade_backups" +V3_BUILD = "g2BuildVersion.json" +V4_BUILD = "szBuildVersion.json" +# TODO - +SZ_SYS_PATH = Path("/opt/senzing") +V3_SYS_PATH = SZ_SYS_PATH / "g2" +V4_SYS_PATH = SZ_SYS_PATH / "er" +V4_DATA_PATH = SZ_SYS_PATH / "data" + +# V3_SYS_PATH = Path("/opt/senzing/g2") +# V4_SYS_PATH = Path("/opt/senzing/er") +V3_SYS_BUILD = V3_SYS_PATH / V3_BUILD +V4_SYS_BUILD = V4_SYS_PATH / V4_BUILD +UPGRADE_URL = "https://www.senzing.com/docs/4_beta/index.html" + +V3_BACKUP_PROJ = { + "bin": {"files": ["g2dbencrypt", "g2saltadm", "g2ssadm"], "excludes": []}, + "etc": {"files": ["senzing_governor.py"], "excludes": []}, + "lib": {"files": ["libSpaceTimeBoxStandardizer.so", "libG2Hasher.so", "libG2SSAdm.so"], "excludes": []}, + "python": {"files": [], "excludes": []}, + "resources/templates": {"files": ["setupEnv"], "excludes": []}, + "setupEnv": {"files": [], "excludes": []}, + "g2BuildVersion.json": {"files": [], "excludes": []}, +} + +V3_REMOVE_FROM_PROJ = { + "bin": {"files": ["*"], "excludes": ["bin"]}, + "data": {"files": [], "excludes": []}, + "etc": {"files": ["senzing_governor.py"], "excludes": []}, + "lib": { + "files": ["*"], + # TODO - Others simplified? + # "files": [ + # "g2.jar", + # "libG2.so", + # "libG2Hasher.so", + # "libG2SSAdm.so", + # "libg2CompJavaScoreSet.so", + # "libg2DistinctFeatJava.so", + # "libg2EFeatJava.so", + # "libg2JVMPlugin.so", + # "libg2StdJava.so", + # "libmariadbplugin.so", + # "libSpaceTimeBoxStandardizer.so", + # ], + "excludes": ["lib"], + }, + "python": {"files": ["*"], "excludes": []}, + "resources/config": {"files": ["g2core-configuration-upgrade-*.gtc"], "excludes": []}, + "resources/schema": {"files": ["g2core-schema-*-create.sql", "g2core-schema-*-upgrade-*.sql"], "excludes": []}, + "resources/templates": { + "files": [ + "G2C.db*", + "G2Module.ini", + "custom*.txt", + "defaultGNRCP.config", + "g2config.json", + "senzing_governor.py", + "setupEnv", + "stb.config", + ], + "excludes": [], + }, + "sdk": {"files": ["*"], "excludes": ["sdk"]}, + "setupEnv": {"files": [], "excludes": []}, +} + +# COPY_TO_PROJ = { +# "LICENSE": {"files": [], "excludes": []}, +# "NOTICES": {"files": [], "excludes": []}, +# "README.1ST": {"files": [], "excludes": []}, +# "bin": {"files": ["*"], "excludes": ["sz_create_project", "sz_update_project"]}, +# # TODO - +# "/opt/senzing/data": {"files": ["*"], "excludes": []}, +# "lib": {"files": ["*"], "excludes": []}, +# "resources": {"files": ["*"], "excludes": []}, +# "sdk": {"files": ["*"], "excludes": []}, +# "szBuildVersion.json": {"files": [], "excludes": []}, +# } +# TODO - Change files to objects? + +# TODO - diff between dir and dir/ +COPY_TO_PROJ = { + # V4_SYS_PATH: {"files": ["*"], "excludes": ["sz_create_project", "sz_update_project"]}, + "er/": {"files": ["*"], "excludes": ["sz_create_project", "sz_update_project"]}, + "data": {"files": ["*"], "excludes": []}, + # TODO - + # V4_DATA_PATH: {"files": ["data"], "excludes": []}, + # "LICENSE": {"files": [], "excludes": []}, + # "NOTICES": {"files": [], "excludes": []}, + # "README.1ST": {"files": [], "excludes": []}, + # "bin": {"files": ["*"], "excludes": ["sz_create_project", "sz_update_project"]}, + # # TODO - + # "/opt/senzing/data": {"files": ["*"], "excludes": []}, + # "lib": {"files": ["*"], "excludes": []}, + # "resources": {"files": ["*"], "excludes": []}, + # "sdk": {"files": ["*"], "excludes": []}, + # "szBuildVersion.json": {"files": [], "excludes": []}, +} +COPY_TO_PROJ_2 = { + # V4_DATA_PATH: {"files": ["data"], "excludes": []}, + V4_DATA_PATH: {"files": ["*"], "excludes": []}, +} + +V3_RENAME_IN_PROJ = { + "etc": {"from": "G2Module.ini", "to": "sz_engine_config.ini"}, +} + +# TODO - Rename and check +SET_PERMISSIONS = { + ".": { + "dir_pint": 0, + "file_pint": 0o660, + "files": ["LICENSE", "NOTICES", "README.1ST", "szBuildVersion.json"], + "excludes": ["setupEnv"], + "recursive": False, + }, + "setupEnv": {"dir_pint": 0, "file_pint": 0o770, "files": [], "excludes": [], "recursive": False}, + "bin": { + "dir_pint": 0o770, + "file_pint": 0o770, + "files": ["*"], + "excludes": ["__pycache__", "_sz_database.py", "_tool_helpers.py"], + "recursive": False, + }, + "data": {"dir_pint": 0o770, "file_pint": 0o660, "files": ["*"], "excludes": [], "recursive": True}, + "lib": {"dir_pint": 0o770, "file_pint": 0o660, "files": ["*"], "excludes": [], "recursive": False}, + "resources": {"dir_pint": 0o770, "file_pint": 0o660, "files": ["*"], "excludes": ["setupEnv"], "recursive": True}, + "resources/templates/setupEnv": { + "dir_pint": 0, + "file_pint": 0o770, + "files": [], + "excludes": [], + "recursive": False, + }, + "sdk": {"dir_pint": 0o770, "file_pint": 0o660, "files": ["*"], "excludes": [], "recursive": True}, + "v3_upgrade_backups": {"dir_pint": 0o770, "file_pint": 0, "files": [], "excludes": [], "recursive": False}, +} + + +@dataclass() +class SzBuildDetails: + """Build information for a project or Senzing SDK system install""" + + platform: str + version: str + build_version: str + build_number: str + major: int = field(init=False) + minor: int = field(init=False) + micro: int = field(init=False) + + def __post_init__(self) -> None: + self.version_parsed = p_version.parse(self.version) + self.build_version_parsed = p_version.parse(self.build_version) + self.major = self.version_parsed.major + self.minor = self.version_parsed.minor + self.micro = self.version_parsed.micro + + +# pylint: disable=W0106 + + +def parse_cli_args() -> argparse.Namespace: + """Parse the CLI arguments""" + arg_parser = argparse.ArgumentParser( + allow_abbrev=False, + description="Update an existing V3 or V4 Senzing project to the system installed V4 of Senzing.", + ) + arg_parser.add_argument( + "project_path", + metavar="path", + help="Path of the project to update", + ) + arg_parser.add_argument( + "-f", + "--force", + dest="force_mode", + default=False, + action="store_true", + help="Upgrade without prompts. WARNING: Use with caution, ensure you have a backup of the project", + ) + return arg_parser.parse_args() + + +def dir_listing(path: Path) -> list[Path]: + """Get a listing of the specified path to check for files or directories""" + try: + listing = list(path.iterdir()) + except OSError as err: + if type(err).__name__ == "NotADirectoryError": + print(f"\nERROR: {path} is not a directory, expecting one") + print(f"\nERROR: {err}") + sys.exit(1) + + return listing + + +def pre_check(project_path: Path) -> tuple[SzBuildDetails, SzBuildDetails]: + """Check not trying to overwrite the V4 Senzing system install, that path is a project, and versions are correct""" + if not V4_SYS_PATH.exists(): + print(f"\nERROR: Couldn't locate Senzing SDK V4 system install at {V4_SYS_PATH}") + sys.exit(1) + + # TODO - + # if str(project_path).startswith(str(V4_SYS_PATH.parent)): + if str(project_path).startswith(str(SZ_SYS_PATH)): + print(f"\nERROR: {project_path} is the Senzing system installation, not a Senzing project") + sys.exit(1) + + v3_project_build_file = project_path / V3_BUILD + v4_project_build_file = project_path / V4_BUILD + proj_listing = dir_listing(project_path) + if v3_project_build_file not in proj_listing and v4_project_build_file not in proj_listing: + print(f"\nERROR: {project_path} isn't a Senzing project, expected it to contain either:") + print(f"\tExisting V3 project: {v3_project_build_file}") + print(f"\tExisting V4 project: {v4_project_build_file}") + sys.exit(1) + + try: + proj_build_file = v3_project_build_file if v3_project_build_file in proj_listing else v4_project_build_file + proj_build_details = get_build_details(proj_build_file) + sys_build_details = get_build_details(V4_SYS_BUILD) + except OSError as err: + print(f"\nERROR: Trying to read {proj_build_file} or {V4_SYS_BUILD} to collect version information: {err}") + sys.exit(1) + + if proj_build_details.major not in (3, 4) or sys_build_details.major != 4: + print(f"\nERROR: {MODULE_NAME} updates a V3 project to V4, or V4.n.n to a newer release") + print(f"\tProject version: {proj_build_details.version}") + print(f"\tSystem install version: {sys_build_details.version}") + sys.exit(1) + + # TODO - + if sys_build_details.version_parsed < proj_build_details.version_parsed: + print("\nNo update required, project is newer than the system install") + print(f"\tProject version: {proj_build_details.version}") + print(f"\tSystem install version: {sys_build_details.version}") + sys.exit(0) + + # TODO - + if sys_build_details.version_parsed == proj_build_details.version_parsed: + print(f"\nNo update required, project and system install are the same version - {proj_build_details.version}") + sys.exit(0) + + return (proj_build_details, sys_build_details) + + +# TODO - Move to helpers +def get_build_details(path: Path) -> SzBuildDetails: + """Return dataclass with the details from a build file.""" + try: + with open(path, "r", encoding="utf-8") as f: + # Ignore DATA_VERSION V3 build files had it, V4 doesn't + version_dict = {k.lower(): v for k, v in json.load(f).items() if k != "DATA_VERSION"} + return SzBuildDetails(**version_dict) + except (OSError, json.JSONDecodeError) as err: + print(f"\nERROR: Couldn't get the build information from {path}: {err}") + sys.exit(1) + except TypeError as err: + print(f"\nERROR: When creating build information from {path}: {err}") + sys.exit(1) + + +def remove_dir(dir_: Path, excludes: list[Path]) -> None: + """Recursively remove a directory""" + try: + for path in dir_.iterdir(): + if path in excludes: + continue + path.unlink(missing_ok=True) if (path.is_file() or path.is_symlink()) else remove_dir(path, excludes) + + if not excludes: + with suppress(FileNotFoundError): + dir_.rmdir() + except OSError as err: + raise err + + +def remove_files_dirs(to_remove: dict[str, Any], target_dir: Path) -> None: + """Remove files/directories that are no longer required""" + for r_path, r_dict in to_remove.items(): + target: Path = target_dir / r_path + files = r_dict["files"] + excludes = [target / e for e in r_dict["excludes"]] + + try: + if target.is_file() or target.is_symlink(): + target.unlink(missing_ok=True) + + if target.is_dir() and not files or (files and files[0] == "*"): + remove_dir(target, excludes) + + if target.is_dir() and files and files[0] != "*": + target_files = [] + for f in files: + target_files.append(target / f) if "*" not in f else target_files.extend(list(target.glob(f))) + + for target_file in target_files: + target_file.unlink(missing_ok=True) + except OSError as err: + raise OSError(f"ERROR: Couldn't delete a file or directory: {err}") from err + + +def copy_files_dirs(to_copy: dict[str, Any], source_dir: Path, target_dir: Path) -> None: + """Copy files/directories within and to a project""" + # TODO - + print(f"\n{to_copy = }", flush=True) + print(f"{source_dir = }", flush=True) + print(f"{target_dir = }", flush=True) + for c_path, c_dict in to_copy.items(): + print(f"{Path(c_path).exists() = }", flush=True) + print(f"\n{c_path = } - {c_dict} ", flush=True) + excludes = c_dict["excludes"] + files = c_dict["files"] + + source: Path = source_dir / c_path + # TODO - + # target: Path = target_dir / c_path + target = target_dir if c_path.endswith("/") else target_dir / c_path + + # target: Path = target_dir + # if Path(c_path).exists(): + # source = Path(c_path) + # target = Path + + print(f"\n{excludes = }", flush=True) + print(f"{files = }", flush=True) + print(f"{source = }", flush=True) + print(f"{target = }", flush=True) + + # if c_path.startswith("/"): + # print("\nStarts with/...", flush=True) + # source = Path(c_path) + # try: + # print("here", flush=True) + # print(f"\tTrying relative to: {target_dir = } - {source.relative_to(V4_SYS_PATH) = }", flush=True) + # target = target_dir / source.relative_to(V4_SYS_PATH) + # except ValueError: + # print(f"\tTrying target.name: {target = } - {target_dir / target.name}", flush=True) + # target = target_dir / target.name + # print(f"\tNow: {source = }", flush=True) + # print(f"\tNow: {target = }", flush=True) + + try: + if source.is_file(): + # TODO - remove shutil + shutil.copy( + source, + target, + ) + + # # TODO + # if source_dir.is_dir() and files and source_dir.name == files[0]: + # print("Yes in...", flush=True) + # source = source_dir + # target = target_dir / source_dir.name + # print(f"\n{source = }", flush=True) + # print(f"\n{target = }", flush=True) + # target.mkdir(exist_ok=True, parents=True) + # shutil.copytree(source, target, ignore=shutil.ignore_patterns(*excludes), dirs_exist_ok=True) + + # Copy entire contents of the source directory, but not the directory + if source.is_dir() and not files or (files and files[0] == "*"): + print("Yes in 2...", flush=True) + shutil.copytree(source, target, ignore=shutil.ignore_patterns(*excludes), dirs_exist_ok=True) + + # Create the source directory in the target and only copy listed files + if source.is_dir() and files and files[0] != "*": + print("Yes in 3...", flush=True) + target.mkdir(exist_ok=True, parents=True) + for source_file in [source / f for f in files]: + shutil.copy(source_file, target / source_file.name) + except OSError as err: + raise OSError(f"ERROR: Couldn't copy a file or directory: {err}") from err + + +def rename_files(to_rename: dict[str, dict[str, str]], target_dir: Path) -> None: + """Rename existing project files that had a name change""" + + try: + for r_path, r_dict in to_rename.items(): + current = target_dir / r_path / r_dict["from"] + new = target_dir / r_path / r_dict["to"] + with suppress(FileNotFoundError): + current.rename(new) + except OSError as err: + raise OSError(f"ERROR: Couldn't rename a file or directory: {err}") from err + + +def setup_env(proj_path: Path) -> None: + """Create a new setupEnv and replace place holders with paths for the project""" + try: + shutil.copy(proj_path / "resources/templates/setupEnv", proj_path) + setup_path = proj_path / "setupEnv" + + with open(setup_path, "r", encoding="utf-8") as in_: + data = in_.read() + + data = data.replace("${SENZING_DIR}", str(proj_path)).replace("${SENZING_CONFIG_PATH}", str(proj_path / "etc")) + + with open(setup_path, "w", encoding="utf-8") as out: + out.write(data) + except OSError as err: + raise OSError(f"ERROR: Couldn't create a new setupEnv file: {err}") from err + + +def set_permissions(proj_path: Path, permissions: dict[str, dict[str, Any]]) -> None: + """Set permissions for files/dirs copied to the project, or dirs removed and replaced completely e.g., data/""" + try: + for p_path, p_dict in permissions.items(): + dir_pint = p_dict["dir_pint"] + file_pint = p_dict["file_pint"] + files = p_dict["files"] + recursive = p_dict["recursive"] + target = proj_path if p_path.startswith(".") else proj_path / p_path + excludes = [target / e for e in p_dict["excludes"]] + + if target.is_file(): + target.chmod(file_pint) + + if target.is_dir() and dir_pint != 0: + target.chmod(dir_pint) + d_chmods = ( + [d for d in target.glob("*") if d.is_dir() and not d.is_symlink() and d not in excludes] + if not recursive + else [d for d in target.rglob("*") if d.is_dir() and not d.is_symlink() and d not in excludes] + ) + for dir_ in d_chmods: + dir_.chmod(dir_pint) + + if target.is_dir() and (files and files[0] == "*"): + f_chmods = ( + [f for f in target.glob("*") if f.is_file() and not f.is_symlink() and f not in excludes] + if not recursive + else [f for f in target.rglob("*") if f.is_file() and not f.is_symlink() and f not in excludes] + ) + for file in f_chmods: + Path(target / file).chmod(file_pint) + + if target.is_dir() and files and files[0] != "*": + for file in files: + Path(target / file).chmod(file_pint) + except OSError as err: + raise OSError(f"ERROR: Couldn't set a permission: {err}") from err + + +def main() -> None: + """main""" + cli_args = parse_cli_args() + # TODO - expanduser + proj_path = Path(cli_args.project_path).resolve() + proj_build, sys_build = pre_check(proj_path) + proj_is_v3 = bool(proj_build.major == 3) + + if not cli_args.force_mode: + print("\nWARNING: If you don't have a backup of the project, create one before continuing!") + sleep(3) + if ( + input(f"\nUpdate the project from version {proj_build.build_version} to {sys_build.build_version}? (y/n) ") + not in INPUT_CONFS + ): + sys.exit(0) + print("\nUpdating...") + else: + print(f"\nUpdating project from version {proj_build.build_version} to {sys_build.build_version}...") + + try: + if proj_is_v3: + v3_backup_path = proj_path / V3_BACKUP_PATH + v3_backup_path.mkdir(exist_ok=True) + copy_files_dirs(V3_BACKUP_PROJ, proj_path, v3_backup_path) # Backup some of the V3 project files + rename_files(V3_RENAME_IN_PROJ, proj_path) + remove_files_dirs(V3_REMOVE_FROM_PROJ, proj_path) + # TODO - + # input("...") + # copy_files_dirs(COPY_TO_PROJ, V4_SYS_PATH, proj_path) + copy_files_dirs(COPY_TO_PROJ, SZ_SYS_PATH, proj_path) + if proj_is_v3: + setup_env(proj_path) + # TODO - Check V4 permissions are good, is sz_create_project good in the first place when V4 to V4? + set_permissions(proj_path, SET_PERMISSIONS) + except OSError as err: + shutil.copytree(v3_backup_path, proj_path, dirs_exist_ok=True) + print(f"\n{err}") + print("\nIf the error is file or directory permission related, run again with appropriate privileges") + print( + "\nIf the error is missing file or directory, check senzingsdk-setup, senzingsdk-tools, and senzingsdk-poc are installed" + ) + else: + if proj_is_v3: # Only delete if no errors so re-running can find the file again + proj_path.joinpath(V3_BUILD).unlink(missing_ok=True) + + print(f"\nProject successfully updated. Refer to {UPGRADE_URL} for additional upgrade instructions") + + +if __name__ == "__main__": + main() diff --git a/sz_tools/sz_update_project_prior6 b/sz_tools/sz_update_project_prior6 new file mode 100755 index 0000000..f011fdd --- /dev/null +++ b/sz_tools/sz_update_project_prior6 @@ -0,0 +1,465 @@ +#! /usr/bin/env python3 +"""Upgrade a V3 or V4 Senzing SDK project to V4.x.x""" + +import argparse +import json +import shutil +import sys +from contextlib import suppress +from dataclasses import dataclass, field +from pathlib import Path +from time import sleep +from typing import Any + +from packaging import version as p_version + +INPUT_CONFS = ("y", "Y", "yEs", "yES", "YES", "yes", "YEs", "yeS", "Yes", "YeS") +MODULE_NAME = Path(__file__).stem +V3_BACKUP_PATH = "v3_upgrade_backups" +V3_BUILD = "g2BuildVersion.json" +V4_BUILD = "szBuildVersion.json" +SZ_SYS_PATH = Path("/opt/senzing") +V3_SYS_PATH = SZ_SYS_PATH / "g2" +V4_SYS_PATH = SZ_SYS_PATH / "er" +V4_DATA_PATH = SZ_SYS_PATH / "data" +V3_SYS_BUILD = V3_SYS_PATH / V3_BUILD +V4_SYS_BUILD = V4_SYS_PATH / V4_BUILD +UPGRADE_URL = "https://www.senzing.com/docs/4_beta/index.html" + +# TODO - files -> objects? +# TODO - "*" +V3_BACKUP_PROJ = { + "bin": {"files": ["g2dbencrypt", "g2saltadm", "g2ssadm"], "excludes": []}, + "etc": {"files": ["senzing_governor.py"], "excludes": []}, + "lib": {"files": ["libSpaceTimeBoxStandardizer.so", "libG2Hasher.so", "libG2SSAdm.so"], "excludes": []}, + "python": {"files": [], "excludes": []}, + "resources/templates": {"files": ["setupEnv"], "excludes": []}, + "setupEnv": {"files": [], "excludes": []}, + "g2BuildVersion.json": {"files": [], "excludes": []}, +} + +V3_REMOVE_FROM_PROJ = { + "bin": {"files": ["*"], "excludes": ["bin"]}, + "data": {"files": [], "excludes": []}, + "etc": {"files": ["senzing_governor.py"], "excludes": []}, + "lib": { + "files": ["*"], + # TODO - Others simplified? + # "files": [ + # "g2.jar", + # "libG2.so", + # "libG2Hasher.so", + # "libG2SSAdm.so", + # "libg2CompJavaScoreSet.so", + # "libg2DistinctFeatJava.so", + # "libg2EFeatJava.so", + # "libg2JVMPlugin.so", + # "libg2StdJava.so", + # "libmariadbplugin.so", + # "libSpaceTimeBoxStandardizer.so", + # ], + "excludes": ["lib"], + }, + "python": {"files": ["*"], "excludes": []}, + "resources/config": {"files": ["g2core-configuration-upgrade-*.gtc"], "excludes": []}, + "resources/schema": {"files": ["g2core-schema-*-create.sql", "g2core-schema-*-upgrade-*.sql"], "excludes": []}, + "resources/templates": { + "files": [ + "G2C.db*", + "G2Module.ini", + "custom*.txt", + "defaultGNRCP.config", + "g2config.json", + "senzing_governor.py", + "setupEnv", + "stb.config", + ], + "excludes": [], + }, + "sdk": {"files": ["*"], "excludes": ["sdk"]}, + "setupEnv": {"files": [], "excludes": []}, +} + +# TODO - diff between dir and dir/ +# er/ copies everything in /opt/senzing/er into the target (minus excludes) +# data copies everything in /opt/senzing/er into the target +COPY_TO_PROJ = { + "er/": {"files": ["*"], "excludes": ["sz_create_project", "sz_update_project"]}, + "data": {"files": ["*"], "excludes": []}, + "er/szBuildVersion.json": {"files": ["*"], "excludes": []}, +} + +V3_RENAME_IN_PROJ = { + "etc": {"from": "G2Module.ini", "to": "sz_engine_config.ini"}, +} + +# TODO - Rename and check +SET_PERMISSIONS = { + ".": { + "dir_pint": 0, + "file_pint": 0o660, + "files": ["LICENSE", "NOTICES", "README.1ST", "szBuildVersion.json"], + "excludes": ["setupEnv"], + "recursive": False, + }, + "setupEnv": {"dir_pint": 0, "file_pint": 0o770, "files": [], "excludes": [], "recursive": False}, + "bin": { + "dir_pint": 0o770, + "file_pint": 0o770, + "files": ["*"], + "excludes": ["__pycache__", "_sz_database.py", "_tool_helpers.py"], + "recursive": False, + }, + "data": {"dir_pint": 0o770, "file_pint": 0o660, "files": ["*"], "excludes": [], "recursive": True}, + "lib": {"dir_pint": 0o770, "file_pint": 0o660, "files": ["*"], "excludes": [], "recursive": False}, + "resources": {"dir_pint": 0o770, "file_pint": 0o660, "files": ["*"], "excludes": ["setupEnv"], "recursive": True}, + "resources/templates/setupEnv": { + "dir_pint": 0, + "file_pint": 0o770, + "files": [], + "excludes": [], + "recursive": False, + }, + "sdk": {"dir_pint": 0o770, "file_pint": 0o660, "files": ["*"], "excludes": [], "recursive": True}, + "v3_upgrade_backups": {"dir_pint": 0o770, "file_pint": 0, "files": [], "excludes": [], "recursive": False}, +} + + +@dataclass() +class SzBuildDetails: + """Build information for a project or Senzing SDK system install""" + + platform: str + version: str + build_version: str + build_number: str + major: int = field(init=False) + minor: int = field(init=False) + micro: int = field(init=False) + + def __post_init__(self) -> None: + self.version_parsed = p_version.parse(self.version) + self.build_version_parsed = p_version.parse(self.build_version) + self.major = self.version_parsed.major + self.minor = self.version_parsed.minor + self.micro = self.version_parsed.micro + + +# pylint: disable=W0106 + + +def parse_cli_args() -> argparse.Namespace: + """Parse the CLI arguments""" + arg_parser = argparse.ArgumentParser( + allow_abbrev=False, + description="Update an existing V3 or V4 Senzing project to the system installed V4 of Senzing.", + ) + arg_parser.add_argument( + "project_path", + metavar="path", + help="Path of the project to update", + ) + arg_parser.add_argument( + "-f", + "--force", + dest="force_mode", + default=False, + action="store_true", + help="Upgrade without prompts. WARNING: Use with caution, ensure you have a backup of the project", + ) + return arg_parser.parse_args() + + +def dir_listing(path: Path) -> list[Path]: + """Get a listing of the specified path to check for files or directories""" + try: + listing = list(path.iterdir()) + except OSError as err: + if type(err).__name__ == "NotADirectoryError": + print(f"\nERROR: {path} is not a directory, expecting one") + print(f"\nERROR: {err}") + sys.exit(1) + + return listing + + +def pre_check(project_path: Path) -> tuple[SzBuildDetails, SzBuildDetails]: + """Check not trying to overwrite the V4 Senzing system install, that path is a project, and versions are correct""" + if not V4_SYS_PATH.exists(): + print(f"\nERROR: Couldn't locate Senzing SDK V4 system install at {V4_SYS_PATH}") + sys.exit(1) + + if str(project_path).startswith(str(SZ_SYS_PATH)): + print(f"\nERROR: {project_path} is the Senzing system installation, not a Senzing project") + sys.exit(1) + + v3_project_build_file = project_path / V3_BUILD + v4_project_build_file = project_path / V4_BUILD + proj_listing = dir_listing(project_path) + if v3_project_build_file not in proj_listing and v4_project_build_file not in proj_listing: + print(f"\nERROR: {project_path} isn't a Senzing project, expected it to contain either:") + print(f"\tExisting V3 project: {v3_project_build_file}") + print(f"\tExisting V4 project: {v4_project_build_file}") + sys.exit(1) + + try: + proj_build_file = v3_project_build_file if v3_project_build_file in proj_listing else v4_project_build_file + proj_build_details = get_build_details(proj_build_file) + sys_build_details = get_build_details(V4_SYS_BUILD) + except OSError as err: + print(f"\nERROR: Trying to read {proj_build_file} or {V4_SYS_BUILD} to collect version information: {err}") + sys.exit(1) + + if proj_build_details.major not in (3, 4) or sys_build_details.major != 4: + print(f"\nERROR: {MODULE_NAME} updates a V3 project to V4, or V4.n.n to a newer release") + print(f"\tProject version: {proj_build_details.version}") + print(f"\tSystem install version: {sys_build_details.version}") + sys.exit(1) + + # TODO - + if sys_build_details.version_parsed < proj_build_details.version_parsed: + print("\nNo update required, project is newer than the system install") + print(f"\tProject version: {proj_build_details.version}") + print(f"\tSystem install version: {sys_build_details.version}") + sys.exit(0) + + # TODO - + if sys_build_details.version_parsed == proj_build_details.version_parsed: + print(f"\nNo update required, project and system install are the same version - {proj_build_details.version}") + sys.exit(0) + + return (proj_build_details, sys_build_details) + + +# TODO - Move to helpers +def get_build_details(path: Path) -> SzBuildDetails: + """Return dataclass with the details from a build file.""" + try: + with open(path, "r", encoding="utf-8") as f: + # Ignore DATA_VERSION, V3 build files had it V4 doesn't + version_dict = {k.lower(): v for k, v in json.load(f).items() if k != "DATA_VERSION"} + return SzBuildDetails(**version_dict) + except (OSError, json.JSONDecodeError) as err: + print(f"\nERROR: Couldn't get the build information from {path}: {err}") + sys.exit(1) + except TypeError as err: + print(f"\nERROR: When creating build information from {path}: {err}") + sys.exit(1) + + +def remove_dir(dir_: Path, excludes: list[Path]) -> None: + """Recursively remove a directory""" + try: + for path in dir_.iterdir(): + if path in excludes: + continue + path.unlink(missing_ok=True) if (path.is_file() or path.is_symlink()) else remove_dir(path, excludes) + + if not excludes: + with suppress(FileNotFoundError): + dir_.rmdir() + except OSError as err: + raise err + + +def remove_files_dirs(to_remove: dict[str, Any], target_dir: Path) -> None: + """Remove files/directories that are no longer required""" + for r_path, r_dict in to_remove.items(): + target: Path = target_dir / r_path + files = r_dict["files"] + excludes = [target / e for e in r_dict["excludes"]] + + try: + if target.is_file() or target.is_symlink(): + target.unlink(missing_ok=True) + + if target.is_dir() and not files or (files and files[0] == "*"): + remove_dir(target, excludes) + + if target.is_dir() and files and files[0] != "*": + target_files = [] + for f in files: + target_files.append(target / f) if "*" not in f else target_files.extend(list(target.glob(f))) + + for target_file in target_files: + target_file.unlink(missing_ok=True) + except OSError as err: + raise OSError(f"ERROR: Couldn't delete a file or directory: {err}") from err + + +def copy_files_dirs(to_copy: dict[str, Any], source_dir: Path, target_dir: Path) -> None: + """Copy files/directories within and to a project""" + print(f"\n{to_copy = }", flush=True) + print(f"{source_dir = }", flush=True) + print(f"{target_dir = }", flush=True) + for c_path, c_dict in to_copy.items(): + excludes = c_dict["excludes"] + files = c_dict["files"] + source = source_dir / c_path + # TODO - + target = target_dir + + # TODO - + # TODO - diff between dir and dir/ + # If the key in to_copy + # er/ copies everything in /opt/senzing/er into the target (minus excludes) + # data copies everything in /opt/senzing/er into the target + # target = target_dir if c_path.endswith("/") else target_dir / c_path + + if source.is_dir(): + target = target_dir if c_path.endswith("/") else target_dir / c_path + + if source.is_file(): + target = target_dir / source.name + + print("\n\n---------------------------------------------------") + print(f"{c_path = }", flush=True) + print(f"{source = }", flush=True) + print(f"{target = }", flush=True) + + try: + if source.is_file(): + # TODO - remove shutil + print(f"\nYes file: {source = }", flush=True) + # TODO - + + shutil.copy( + source, + target, + ) + + # Copy entire contents of the source directory + # if source.is_dir() and not files or (files and files[0] == "*"): + if source.is_dir() and (not files or (files and files[0] == "*")): + print("\nhere1...") + shutil.copytree(source, target, ignore=shutil.ignore_patterns(*excludes), dirs_exist_ok=True) + + # Create the source directory in the target and only copy listed files + # if source.is_dir() and files and files[0] != "*": + if source.is_dir() and (files and files[0] != "*"): + print("\nhere2...") + target.mkdir(exist_ok=True, parents=True) + for source_file in [source / f for f in files]: + shutil.copy(source_file, target / source_file.name) + except OSError as err: + raise OSError(f"ERROR: Couldn't copy a file or directory: {err}") from err + + +def rename_files(to_rename: dict[str, dict[str, str]], target_dir: Path) -> None: + """Rename existing project files that had a name change""" + + try: + for r_path, r_dict in to_rename.items(): + current = target_dir / r_path / r_dict["from"] + new = target_dir / r_path / r_dict["to"] + with suppress(FileNotFoundError): + current.rename(new) + except OSError as err: + raise OSError(f"ERROR: Couldn't rename a file or directory: {err}") from err + + +def setup_env(proj_path: Path) -> None: + """Create a new setupEnv and replace place holders with paths for the project""" + try: + shutil.copy(proj_path / "resources/templates/setupEnv", proj_path) + setup_path = proj_path / "setupEnv" + + with open(setup_path, "r", encoding="utf-8") as in_: + data = in_.read() + + data = data.replace("${SENZING_DIR}", str(proj_path)).replace("${SENZING_CONFIG_PATH}", str(proj_path / "etc")) + + with open(setup_path, "w", encoding="utf-8") as out: + out.write(data) + except OSError as err: + raise OSError(f"ERROR: Couldn't create a new setupEnv file: {err}") from err + + +def set_permissions(proj_path: Path, permissions: dict[str, dict[str, Any]]) -> None: + """Set permissions for files/dirs copied to the project, or dirs removed and replaced completely e.g., data/""" + try: + for p_path, p_dict in permissions.items(): + dir_pint = p_dict["dir_pint"] + file_pint = p_dict["file_pint"] + files = p_dict["files"] + recursive = p_dict["recursive"] + target = proj_path if p_path.startswith(".") else proj_path / p_path + excludes = [target / e for e in p_dict["excludes"]] + + if target.is_file(): + target.chmod(file_pint) + + if target.is_dir() and dir_pint != 0: + target.chmod(dir_pint) + d_chmods = ( + [d for d in target.glob("*") if d.is_dir() and not d.is_symlink() and d not in excludes] + if not recursive + else [d for d in target.rglob("*") if d.is_dir() and not d.is_symlink() and d not in excludes] + ) + for dir_ in d_chmods: + dir_.chmod(dir_pint) + + if target.is_dir() and (files and files[0] == "*"): + f_chmods = ( + [f for f in target.glob("*") if f.is_file() and not f.is_symlink() and f not in excludes] + if not recursive + else [f for f in target.rglob("*") if f.is_file() and not f.is_symlink() and f not in excludes] + ) + for file in f_chmods: + Path(target / file).chmod(file_pint) + + if target.is_dir() and files and files[0] != "*": + for file in files: + Path(target / file).chmod(file_pint) + except OSError as err: + raise OSError(f"ERROR: Couldn't set a permission: {err}") from err + + +def main() -> None: + """main""" + cli_args = parse_cli_args() + # TODO - expanduser + proj_path = Path(cli_args.project_path).resolve() + proj_build, sys_build = pre_check(proj_path) + proj_is_v3 = bool(proj_build.major == 3) + + if not cli_args.force_mode: + print("\nWARNING: If you don't have a backup of the project, create one before continuing!") + sleep(3) + if ( + input(f"\nUpdate the project from version {proj_build.build_version} to {sys_build.build_version}? (y/n) ") + not in INPUT_CONFS + ): + sys.exit(0) + print("\nUpdating...") + else: + print(f"\nUpdating project from version {proj_build.build_version} to {sys_build.build_version}...") + + try: + if proj_is_v3: + v3_backup_path = proj_path / V3_BACKUP_PATH + v3_backup_path.mkdir(exist_ok=True) + copy_files_dirs(V3_BACKUP_PROJ, proj_path, v3_backup_path) # Backup some of the V3 project files + rename_files(V3_RENAME_IN_PROJ, proj_path) + remove_files_dirs(V3_REMOVE_FROM_PROJ, proj_path) + copy_files_dirs(COPY_TO_PROJ, SZ_SYS_PATH, proj_path) + if proj_is_v3: + setup_env(proj_path) + # TODO - Check V4 permissions are good, is sz_create_project good in the first place when V4 to V4? + set_permissions(proj_path, SET_PERMISSIONS) + except OSError as err: + shutil.copytree(v3_backup_path, proj_path, dirs_exist_ok=True) + print(f"\n{err}") + print("\nIf the error is file or directory permission related, run again with appropriate privileges") + print( + "\nIf the error is missing file or directory, check senzingsdk-setup, senzingsdk-tools, and senzingsdk-poc are installed" + ) + else: + if proj_is_v3: # Only delete if no errors so re-running can find the file again + proj_path.joinpath(V3_BUILD).unlink(missing_ok=True) + + print(f"\nProject successfully updated. Refer to {UPGRADE_URL} for additional upgrade instructions") + + +if __name__ == "__main__": + main() diff --git a/sz_tools/sz_update_project_prior7 b/sz_tools/sz_update_project_prior7 new file mode 100755 index 0000000..edb9274 --- /dev/null +++ b/sz_tools/sz_update_project_prior7 @@ -0,0 +1,480 @@ +#! /usr/bin/env python3 +"""Upgrade a V3 or V4 Senzing SDK project to V4.x.x""" + +import argparse +import json +import shutil +import sys +from contextlib import suppress +from dataclasses import dataclass, field +from pathlib import Path +from time import sleep +from typing import Any + +from packaging import version as p_version + +INPUT_CONFS = ("y", "Y", "yEs", "yES", "YES", "yes", "YEs", "yeS", "Yes", "YeS") +MODULE_NAME = Path(__file__).stem +V3_BACKUP_PATH = "v3_upgrade_backups" +V3_BUILD = "g2BuildVersion.json" +V4_BUILD = "szBuildVersion.json" +SZ_SYS_PATH = Path("/opt/senzing") +V3_SYS_PATH = SZ_SYS_PATH / "g2" +V4_SYS_PATH = SZ_SYS_PATH / "er" +V4_DATA_PATH = SZ_SYS_PATH / "data" +V3_SYS_BUILD = V3_SYS_PATH / V3_BUILD +V4_SYS_BUILD = V4_SYS_PATH / V4_BUILD +UPGRADE_URL = "https://www.senzing.com/docs/4_beta/index.html" + +# TODO - files -> objects? +# TODO - "*" +V3_BACKUP_PROJ = { + "bin": {"files": ["g2dbencrypt", "g2saltadm", "g2ssadm"], "excludes": []}, + "etc": {"files": ["senzing_governor.py"], "excludes": []}, + "lib": {"files": ["libSpaceTimeBoxStandardizer.so", "libG2Hasher.so", "libG2SSAdm.so"], "excludes": []}, + "python": {"files": [], "excludes": []}, + "resources/templates": {"files": ["setupEnv"], "excludes": []}, + "setupEnv": {"files": [], "excludes": []}, + "g2BuildVersion.json": {"files": [], "excludes": []}, +} + +V3_REMOVE_FROM_PROJ = { + "bin": {"files": ["*"], "excludes": ["bin"]}, + "data": {"files": [], "excludes": []}, + "etc": {"files": ["senzing_governor.py"], "excludes": []}, + "lib": { + "files": ["*"], + # TODO - Others simplified? + # "files": [ + # "g2.jar", + # "libG2.so", + # "libG2Hasher.so", + # "libG2SSAdm.so", + # "libg2CompJavaScoreSet.so", + # "libg2DistinctFeatJava.so", + # "libg2EFeatJava.so", + # "libg2JVMPlugin.so", + # "libg2StdJava.so", + # "libmariadbplugin.so", + # "libSpaceTimeBoxStandardizer.so", + # ], + "excludes": ["lib"], + }, + "python": {"files": ["*"], "excludes": []}, + "resources/config": {"files": ["g2core-configuration-upgrade-*.gtc"], "excludes": []}, + "resources/schema": {"files": ["g2core-schema-*-create.sql", "g2core-schema-*-upgrade-*.sql"], "excludes": []}, + "resources/templates": { + "files": [ + "G2C.db*", + "G2Module.ini", + "custom*.txt", + "defaultGNRCP.config", + "g2config.json", + "senzing_governor.py", + "setupEnv", + "stb.config", + ], + "excludes": [], + }, + "sdk": {"files": ["*"], "excludes": ["sdk"]}, + "setupEnv": {"files": [], "excludes": []}, +} + +# TODO - diff between dir and dir/ +# er/ copies everything in /opt/senzing/er into the target (minus excludes) +# data copies everything in /opt/senzing/er into the target +COPY_TO_PROJ = { + "er/": {"files": ["*"], "excludes": ["sz_create_project", "sz_update_project"]}, + "data": {"files": ["*"], "excludes": []}, + "er/szBuildVersion.json": {"files": ["*"], "excludes": []}, +} + +V3_RENAME_IN_PROJ = { + "etc": {"from": "G2Module.ini", "to": "sz_engine_config.ini"}, +} + +# TODO - Rename and check +SET_PERMISSIONS = { + ".": { + "dir_pint": 0, + "file_pint": 0o660, + "files": ["LICENSE", "NOTICES", "README.1ST", "szBuildVersion.json"], + "excludes": ["setupEnv"], + "recursive": False, + }, + "setupEnv": {"dir_pint": 0, "file_pint": 0o770, "files": [], "excludes": [], "recursive": False}, + "bin": { + "dir_pint": 0o770, + "file_pint": 0o770, + "files": ["*"], + "excludes": ["__pycache__", "_sz_database.py", "_tool_helpers.py"], + "recursive": False, + }, + "data": {"dir_pint": 0o770, "file_pint": 0o660, "files": ["*"], "excludes": [], "recursive": True}, + "lib": {"dir_pint": 0o770, "file_pint": 0o660, "files": ["*"], "excludes": [], "recursive": False}, + "resources": {"dir_pint": 0o770, "file_pint": 0o660, "files": ["*"], "excludes": ["setupEnv"], "recursive": True}, + "resources/templates/setupEnv": { + "dir_pint": 0, + "file_pint": 0o770, + "files": [], + "excludes": [], + "recursive": False, + }, + "sdk": {"dir_pint": 0o770, "file_pint": 0o660, "files": ["*"], "excludes": [], "recursive": True}, + "v3_upgrade_backups": {"dir_pint": 0o770, "file_pint": 0, "files": [], "excludes": [], "recursive": False}, +} + + +@dataclass() +class SzBuildDetails: + """Build information for a project or Senzing SDK system install""" + + platform: str + version: str + build_version: str + build_number: str + major: int = field(init=False) + minor: int = field(init=False) + micro: int = field(init=False) + + def __post_init__(self) -> None: + self.version_parsed = p_version.parse(self.version) + self.build_version_parsed = p_version.parse(self.build_version) + self.major = self.version_parsed.major + self.minor = self.version_parsed.minor + self.micro = self.version_parsed.micro + + +# pylint: disable=W0106 + + +def parse_cli_args() -> argparse.Namespace: + """Parse the CLI arguments""" + arg_parser = argparse.ArgumentParser( + allow_abbrev=False, + description="Update an existing V3 or V4 Senzing project to the system installed V4 of Senzing.", + ) + arg_parser.add_argument( + "project_path", + metavar="path", + help="Path of the project to update", + ) + arg_parser.add_argument( + "-f", + "--force", + dest="force_mode", + default=False, + action="store_true", + help="Upgrade without prompts. WARNING: Use with caution, ensure you have a backup of the project", + ) + return arg_parser.parse_args() + + +def dir_listing(path: Path) -> list[Path]: + """Get a listing of the specified path to check for files or directories""" + try: + listing = list(path.iterdir()) + except OSError as err: + if type(err).__name__ == "NotADirectoryError": + print(f"\nERROR: {path} is not a directory, expecting one") + print(f"\nERROR: {err}") + sys.exit(1) + + return listing + + +def pre_check(project_path: Path) -> tuple[SzBuildDetails, SzBuildDetails]: + """Check not trying to overwrite the V4 Senzing system install, that path is a project, and versions are correct""" + if not V4_SYS_PATH.exists(): + print(f"\nERROR: Couldn't locate Senzing SDK V4 system install at {V4_SYS_PATH}") + sys.exit(1) + + if str(project_path).startswith(str(SZ_SYS_PATH)): + print(f"\nERROR: {project_path} is the Senzing system installation, not a Senzing project") + sys.exit(1) + + v3_project_build_file = project_path / V3_BUILD + v4_project_build_file = project_path / V4_BUILD + proj_listing = dir_listing(project_path) + if v3_project_build_file not in proj_listing and v4_project_build_file not in proj_listing: + print(f"\nERROR: {project_path} isn't a Senzing project, expected it to contain either:") + print(f"\tExisting V3 project: {v3_project_build_file}") + print(f"\tExisting V4 project: {v4_project_build_file}") + sys.exit(1) + + try: + proj_build_file = v3_project_build_file if v3_project_build_file in proj_listing else v4_project_build_file + proj_build_details = get_build_details(proj_build_file) + sys_build_details = get_build_details(V4_SYS_BUILD) + except OSError as err: + print(f"\nERROR: Trying to read {proj_build_file} or {V4_SYS_BUILD} to collect version information: {err}") + sys.exit(1) + + if proj_build_details.major not in (3, 4) or sys_build_details.major != 4: + print(f"\nERROR: {MODULE_NAME} updates a V3 project to V4, or V4.n.n to a newer release") + print(f"\tProject version: {proj_build_details.version}") + print(f"\tSystem install version: {sys_build_details.version}") + sys.exit(1) + + # TODO - + if sys_build_details.version_parsed < proj_build_details.version_parsed: + print("\nNo update required, project is newer than the system install") + print(f"\tProject version: {proj_build_details.version}") + print(f"\tSystem install version: {sys_build_details.version}") + sys.exit(0) + + # TODO - + if sys_build_details.version_parsed == proj_build_details.version_parsed: + print(f"\nNo update required, project and system install are the same version - {proj_build_details.version}") + sys.exit(0) + + return (proj_build_details, sys_build_details) + + +# TODO - Move to helpers +def get_build_details(path: Path) -> SzBuildDetails: + """Return dataclass with the details from a build file.""" + try: + with open(path, "r", encoding="utf-8") as f: + # Ignore DATA_VERSION, V3 build files had it V4 doesn't + version_dict = {k.lower(): v for k, v in json.load(f).items() if k != "DATA_VERSION"} + return SzBuildDetails(**version_dict) + except (OSError, json.JSONDecodeError) as err: + print(f"\nERROR: Couldn't get the build information from {path}: {err}") + sys.exit(1) + except TypeError as err: + print(f"\nERROR: When creating build information from {path}: {err}") + sys.exit(1) + + +def remove_dir(dir_: Path, excludes: list[Path]) -> None: + """Recursively remove a directory""" + try: + for path in dir_.iterdir(): + if path in excludes: + continue + path.unlink(missing_ok=True) if (path.is_file() or path.is_symlink()) else remove_dir(path, excludes) + + if not excludes: + with suppress(FileNotFoundError): + dir_.rmdir() + except OSError as err: + raise err + + +def remove_files_dirs(to_remove: dict[str, Any], target_dir: Path) -> None: + """Remove files/directories that are no longer required""" + for r_path, r_dict in to_remove.items(): + target: Path = target_dir / r_path + files = r_dict["files"] + excludes = [target / e for e in r_dict["excludes"]] + + try: + if target.is_file() or target.is_symlink(): + target.unlink(missing_ok=True) + + if target.is_dir() and not files or (files and files[0] == "*"): + remove_dir(target, excludes) + + if target.is_dir() and files and files[0] != "*": + target_files = [] + for f in files: + target_files.append(target / f) if "*" not in f else target_files.extend(list(target.glob(f))) + + for target_file in target_files: + target_file.unlink(missing_ok=True) + except OSError as err: + raise OSError(f"ERROR: Couldn't delete a file or directory: {err}") from err + + +def copy_files_dirs(to_copy: dict[str, Any], source_dir: Path, target_dir: Path) -> None: + """Copy files/directories within and to a project""" + print(f"\n{to_copy = }", flush=True) + print(f"{source_dir = }", flush=True) + print(f"{target_dir = }", flush=True) + for c_path, c_dict in to_copy.items(): + excludes = c_dict["excludes"] + files = c_dict["files"] + source = source_dir / c_path + # TODO - + target = target_dir + + # TODO - + # TODO - diff between dir and dir/ + # If the key in to_copy + # er/ copies everything in /opt/senzing/er into the target (minus excludes) + # data copies everything in /opt/senzing/er into the target + # target = target_dir if c_path.endswith("/") else target_dir / c_path + + try: + if source.is_dir(): + target = target_dir if c_path.endswith("/") else target_dir / c_path + + # Copy entire contents of the source directory + if not files or (files and files[0] == "*"): + print("\nhere1...") + shutil.copytree(source, target, ignore=shutil.ignore_patterns(*excludes), dirs_exist_ok=True) + + # Create the source directory in the target and only copy listed files + if files and files[0] != "*": + print("\nhere2...") + target.mkdir(exist_ok=True, parents=True) + for source_file in [source / f for f in files]: + shutil.copy(source_file, target / source_file.name) + + if source.is_file(): + target = target_dir / source.name + except OSError as err: + raise OSError(f"ERROR: Couldn't copy a file or directory: {err}") from err + + # print("\n\n---------------------------------------------------") + # print(f"{c_path = }", flush=True) + # print(f"{source = }", flush=True) + # print(f"{target = }", flush=True) + + # try: + # if source.is_file(): + # # TODO - remove shutil + # print(f"\nYes file: {source = }", flush=True) + # # TODO - + + # shutil.copy( + # source, + # target, + # ) + + # # Copy entire contents of the source directory + # # if source.is_dir() and not files or (files and files[0] == "*"): + # if source.is_dir() and (not files or (files and files[0] == "*")): + # print("\nhere1...") + # shutil.copytree(source, target, ignore=shutil.ignore_patterns(*excludes), dirs_exist_ok=True) + + # # Create the source directory in the target and only copy listed files + # # if source.is_dir() and files and files[0] != "*": + # if source.is_dir() and (files and files[0] != "*"): + # print("\nhere2...") + # target.mkdir(exist_ok=True, parents=True) + # for source_file in [source / f for f in files]: + # shutil.copy(source_file, target / source_file.name) + # except OSError as err: + # raise OSError(f"ERROR: Couldn't copy a file or directory: {err}") from err + + +def rename_files(to_rename: dict[str, dict[str, str]], target_dir: Path) -> None: + """Rename existing project files that had a name change""" + + try: + for r_path, r_dict in to_rename.items(): + current = target_dir / r_path / r_dict["from"] + new = target_dir / r_path / r_dict["to"] + with suppress(FileNotFoundError): + current.rename(new) + except OSError as err: + raise OSError(f"ERROR: Couldn't rename a file or directory: {err}") from err + + +def setup_env(proj_path: Path) -> None: + """Create a new setupEnv and replace place holders with paths for the project""" + try: + shutil.copy(proj_path / "resources/templates/setupEnv", proj_path) + setup_path = proj_path / "setupEnv" + + with open(setup_path, "r", encoding="utf-8") as in_: + data = in_.read() + + data = data.replace("${SENZING_DIR}", str(proj_path)).replace("${SENZING_CONFIG_PATH}", str(proj_path / "etc")) + + with open(setup_path, "w", encoding="utf-8") as out: + out.write(data) + except OSError as err: + raise OSError(f"ERROR: Couldn't create a new setupEnv file: {err}") from err + + +def set_permissions(proj_path: Path, permissions: dict[str, dict[str, Any]]) -> None: + """Set permissions for files/dirs copied to the project, or dirs removed and replaced completely e.g., data/""" + try: + for p_path, p_dict in permissions.items(): + dir_pint = p_dict["dir_pint"] + file_pint = p_dict["file_pint"] + files = p_dict["files"] + recursive = p_dict["recursive"] + target = proj_path if p_path.startswith(".") else proj_path / p_path + excludes = [target / e for e in p_dict["excludes"]] + + if target.is_file(): + target.chmod(file_pint) + + if target.is_dir() and dir_pint != 0: + target.chmod(dir_pint) + d_chmods = ( + [d for d in target.glob("*") if d.is_dir() and not d.is_symlink() and d not in excludes] + if not recursive + else [d for d in target.rglob("*") if d.is_dir() and not d.is_symlink() and d not in excludes] + ) + for dir_ in d_chmods: + dir_.chmod(dir_pint) + + if target.is_dir() and (files and files[0] == "*"): + f_chmods = ( + [f for f in target.glob("*") if f.is_file() and not f.is_symlink() and f not in excludes] + if not recursive + else [f for f in target.rglob("*") if f.is_file() and not f.is_symlink() and f not in excludes] + ) + for file in f_chmods: + Path(target / file).chmod(file_pint) + + if target.is_dir() and files and files[0] != "*": + for file in files: + Path(target / file).chmod(file_pint) + except OSError as err: + raise OSError(f"ERROR: Couldn't set a permission: {err}") from err + + +def main() -> None: + """main""" + cli_args = parse_cli_args() + # TODO - expanduser + proj_path = Path(cli_args.project_path).resolve() + proj_build, sys_build = pre_check(proj_path) + proj_is_v3 = bool(proj_build.major == 3) + + if not cli_args.force_mode: + print("\nWARNING: If you don't have a backup of the project, create one before continuing!") + sleep(3) + if ( + input(f"\nUpdate the project from version {proj_build.build_version} to {sys_build.build_version}? (y/n) ") + not in INPUT_CONFS + ): + sys.exit(0) + print("\nUpdating...") + else: + print(f"\nUpdating project from version {proj_build.build_version} to {sys_build.build_version}...") + + try: + if proj_is_v3: + v3_backup_path = proj_path / V3_BACKUP_PATH + v3_backup_path.mkdir(exist_ok=True) + copy_files_dirs(V3_BACKUP_PROJ, proj_path, v3_backup_path) # Backup some of the V3 project files + rename_files(V3_RENAME_IN_PROJ, proj_path) + remove_files_dirs(V3_REMOVE_FROM_PROJ, proj_path) + copy_files_dirs(COPY_TO_PROJ, SZ_SYS_PATH, proj_path) + if proj_is_v3: + setup_env(proj_path) + # TODO - Check V4 permissions are good, is sz_create_project good in the first place when V4 to V4? + set_permissions(proj_path, SET_PERMISSIONS) + except OSError as err: + shutil.copytree(v3_backup_path, proj_path, dirs_exist_ok=True) + print(f"\n{err}") + print("\nIf the error is file or directory permission related, run again with appropriate privileges") + print( + "\nIf the error is missing file or directory, check senzingsdk-setup, senzingsdk-tools, and senzingsdk-poc are installed" + ) + else: + if proj_is_v3: # Only delete if no errors so re-running can find the file again + proj_path.joinpath(V3_BUILD).unlink(missing_ok=True) + + print(f"\nProject successfully updated. Refer to {UPGRADE_URL} for additional upgrade instructions") + + +if __name__ == "__main__": + main() diff --git a/sz_tools/sz_update_project_prior8 b/sz_tools/sz_update_project_prior8 new file mode 100755 index 0000000..cf3bf8c --- /dev/null +++ b/sz_tools/sz_update_project_prior8 @@ -0,0 +1,463 @@ +#! /usr/bin/env python3 +"""Upgrade a V3 or V4 Senzing SDK project to V4.x.x""" + +import argparse +import json +import shutil +import sys +from contextlib import suppress +from dataclasses import dataclass, field +from pathlib import Path +from time import sleep +from typing import Any + +from packaging import version as p_version + +INPUT_CONFS = ("y", "Y", "yEs", "yES", "YES", "yes", "YEs", "yeS", "Yes", "YeS") +MODULE_NAME = Path(__file__).stem +V3_BACKUP_PATH = "v3_to_v4_upgrade_backups" +V3_BUILD = "g2BuildVersion.json" +V4_BUILD = "szBuildVersion.json" +SZ_SYS_PATH = Path("/opt/senzing") +V3_SYS_PATH = SZ_SYS_PATH / "g2" +V4_SYS_PATH = SZ_SYS_PATH / "er" +V4_DATA_PATH = SZ_SYS_PATH / "data" +V3_SYS_BUILD = V3_SYS_PATH / V3_BUILD +V4_SYS_BUILD = V4_SYS_PATH / V4_BUILD +UPGRADE_URL = "https://www.senzing.com/docs/4_beta/index.html" + +V3_BACKUP_PROJ = { + "bin": {"files": ["g2dbencrypt", "g2saltadm", "g2ssadm"], "excludes": []}, + "etc": {"files": ["senzing_governor.py"], "excludes": []}, + "lib": {"files": ["libSpaceTimeBoxStandardizer.so", "libG2Hasher.so", "libG2SSAdm.so"], "excludes": []}, + "python": { + "files": ["*"], + "excludes": [ + "CompressedFile.py", + "DumpStack.py", + "G2Audit.py", + "G2Command.py", + "G2ConfigTables.py", + "G2ConfigTool.py", + "G2Database.py", + "G2Explorer.py", + "G2Export.py", + "G2IniParams.py", + "G2Loader.py", + "G2Paths.py", + "G2Project.py", + "G2S3.py", + "G2SetupConfig.py", + "G2Snapshot.py", + "SenzingGo.py", + "senzing", + ], + }, + "setupEnv": {"files": [], "excludes": []}, + "g2BuildVersion.json": {"files": [], "excludes": []}, +} + +V3_REMOVE_FROM_PROJ = { + "bin": {"files": ["g2configupgrade", "g2dbencrypt", "g2dbupgrade", "g2saltadm", "g2ssadm"], "excludes": ["bin"]}, + "data": {"files": [], "excludes": []}, + "etc": {"files": ["senzing_governor.py"], "excludes": []}, + "lib": { + "files": [ + "g2.jar", + "libG2*.so", + "libG2Hasher.so", + "libG2SSAdm.so", + "libg2CompJavaScoreSet.so", + "libg2DistinctFeatJava.so", + "libg2EFeatJava.so", + "libg2JVMPlugin.so", + "libg2StdJava.so", + "libmariadbplugin.so", + "libSpaceTimeBoxStandardizer.so", + ], + "excludes": ["lib"], + }, + "python": {"files": ["*"], "excludes": []}, + "resources/config": {"files": ["g2core-configuration-upgrade-*.gtc"], "excludes": []}, + "resources/schema": {"files": ["g2core-schema-*-create.sql", "g2core-schema-*-upgrade-*.sql"], "excludes": []}, + "resources/templates": { + "files": [ + "G2C.db*", + "G2Module.ini", + "cfgVariant.json", + "custom*.txt", + "defaultGNRCP.config", + "g2config.json", + "senzing_governor.py", + "setupEnv", + "stb.config", + ], + "excludes": [], + }, + "sdk": {"files": ["*"], "excludes": []}, + "setupEnv": {"files": [], "excludes": []}, +} + +COPY_TO_PROJ = { + "er/": {"files": ["*"], "excludes": ["sz_create_project", "sz_update_project"]}, + "data": {"files": ["*"], "excludes": []}, +} + +V3_RENAME_IN_PROJ = { + "etc": {"from": "G2Module.ini", "to": "sz_engine_config.ini"}, +} + +PERMISSIONS = { + ".": { + "dir_pint": 0, + "file_pint": 0o660, + "files": ["LICENSE", "NOTICES", "README.1ST", "szBuildVersion.json"], + "excludes": ["setupEnv"], + "recursive": False, + }, + "setupEnv": {"dir_pint": 0, "file_pint": 0o770, "files": [], "excludes": [], "recursive": False}, + "bin": { + "dir_pint": 0o770, + "file_pint": 0o770, + "files": ["*"], + "excludes": ["__pycache__", "_sz_database.py", "_tool_helpers.py"], + "recursive": False, + }, + "data": {"dir_pint": 0o770, "file_pint": 0o660, "files": ["*"], "excludes": [], "recursive": True}, + "lib": {"dir_pint": 0o770, "file_pint": 0o660, "files": ["*"], "excludes": [], "recursive": False}, + "resources": {"dir_pint": 0o770, "file_pint": 0o660, "files": ["*"], "excludes": ["setupEnv"], "recursive": True}, + "resources/templates/setupEnv": { + "dir_pint": 0, + "file_pint": 0o770, + "files": [], + "excludes": [], + "recursive": False, + }, + "sdk": {"dir_pint": 0o770, "file_pint": 0o660, "files": ["*"], "excludes": [], "recursive": True}, + V3_BACKUP_PATH: {"dir_pint": 0o770, "file_pint": 0, "files": [], "excludes": [], "recursive": False}, +} +PERMISSIONS_2 = { + "bin": { + "dir_pint": 0o770, + "file_pint": 0o660, + "files": ["_sz_database.py", "_tool_helpers.py"], + "excludes": [], + "recursive": False, + } +} + + +@dataclass() +class SzBuildDetails: + """Build information for a project or Senzing SDK system install""" + + platform: str + version: str + build_version: str + build_number: str + major: int = field(init=False) + minor: int = field(init=False) + micro: int = field(init=False) + + def __post_init__(self) -> None: + self.version_parsed = p_version.parse(self.version) + self.build_version_parsed = p_version.parse(self.build_version) + self.major = self.version_parsed.major + self.minor = self.version_parsed.minor + self.micro = self.version_parsed.micro + + +# pylint: disable=W0106 + + +def parse_cli_args() -> argparse.Namespace: + """Parse the CLI arguments""" + arg_parser = argparse.ArgumentParser( + allow_abbrev=False, + description="Update an existing V3 or V4 Senzing project to the system installed V4 of Senzing.", + ) + arg_parser.add_argument( + "project_path", + metavar="path", + help="Path of the project to update", + ) + arg_parser.add_argument( + "-f", + "--force", + dest="force_mode", + default=False, + action="store_true", + help="Upgrade without prompts. WARNING: Use with caution, ensure you have a backup of the project", + ) + return arg_parser.parse_args() + + +def dir_listing(path: Path) -> list[Path]: + """Get a listing of the specified path to check for files or directories""" + try: + listing = list(path.iterdir()) + except OSError as err: + if type(err).__name__ == "NotADirectoryError": + print(f"\nERROR: {path} is not a directory, expecting one") + print(f"\nERROR: {err}") + sys.exit(1) + + return listing + + +def pre_check(project_path: Path) -> tuple[SzBuildDetails, SzBuildDetails]: + """Check not trying to overwrite the V4 Senzing system install, that path is a project, and versions are correct""" + if not V4_SYS_PATH.exists(): + print(f"\nERROR: Couldn't locate Senzing SDK V4 system install at {V4_SYS_PATH}") + sys.exit(1) + + if str(project_path).startswith(str(SZ_SYS_PATH)): + print(f"\nERROR: {project_path} is the Senzing system installation, not a Senzing project") + sys.exit(1) + + v3_project_build_file = project_path / V3_BUILD + v4_project_build_file = project_path / V4_BUILD + proj_listing = dir_listing(project_path) + if v3_project_build_file not in proj_listing and v4_project_build_file not in proj_listing: + print(f"\nERROR: {project_path} isn't a Senzing project, expected it to contain either:") + print(f"\tExisting V3 project: {v3_project_build_file}") + print(f"\tExisting V4 project: {v4_project_build_file}") + sys.exit(1) + + try: + proj_build_file = v3_project_build_file if v3_project_build_file in proj_listing else v4_project_build_file + proj_build_details = get_build_details(proj_build_file) + sys_build_details = get_build_details(V4_SYS_BUILD) + except OSError as err: + print(f"\nERROR: Trying to read {proj_build_file} or {V4_SYS_BUILD} to collect version information: {err}") + sys.exit(1) + + if proj_build_details.major not in (3, 4) or sys_build_details.major != 4: + print(f"\nERROR: {MODULE_NAME} updates a V3 project to V4, or V4.n.n to a newer release") + print(f"\tProject version: {proj_build_details.version}") + print(f"\tSystem install version: {sys_build_details.version}") + sys.exit(1) + + if sys_build_details.version_parsed < proj_build_details.version_parsed: + print("\nNo update required, project is newer than the system install") + print(f"\tProject version: {proj_build_details.version}") + print(f"\tSystem install version: {sys_build_details.version}") + sys.exit(0) + + if sys_build_details.version_parsed == proj_build_details.version_parsed: + print(f"\nNo update required, project and system install are the same version - {proj_build_details.version}") + sys.exit(0) + + return (proj_build_details, sys_build_details) + + +def get_build_details(path: Path) -> SzBuildDetails: + """Return dataclass with the details from a build file.""" + try: + with open(path, "r", encoding="utf-8") as f: + # Ignore DATA_VERSION, V3 build files had it V4 doesn't + version_dict = {k.lower(): v for k, v in json.load(f).items() if k != "DATA_VERSION"} + return SzBuildDetails(**version_dict) + except (OSError, json.JSONDecodeError) as err: + print(f"\nERROR: Couldn't get the build information from {path}: {err}") + sys.exit(1) + except TypeError as err: + print(f"\nERROR: When creating build information from {path}: {err}") + sys.exit(1) + + +def remove_dir(dir_: Path, excludes: list[Path]) -> None: + """Recursively remove a directory""" + try: + for path in dir_.iterdir(): + if path in excludes: + continue + path.unlink(missing_ok=True) if (path.is_file() or path.is_symlink()) else remove_dir(path, excludes) + + if not excludes: + with suppress(FileNotFoundError): + dir_.rmdir() + except OSError as err: + raise err + + +def remove_files_dirs(to_remove: dict[str, Any], target_dir: Path) -> None: + """Remove files/directories that are no longer required""" + for r_path, r_dict in to_remove.items(): + target: Path = target_dir / r_path + files = r_dict["files"] + excludes = [target / e for e in r_dict["excludes"]] + + try: + if target.is_dir(): + if not files or (files and files[0] == "*"): + remove_dir(target, excludes) + + if files and files[0] != "*": + target_files = [] + for f in files: + target_files.append(target / f) if "*" not in f else target_files.extend(list(target.glob(f))) + + for target_file in target_files: + target_file.unlink(missing_ok=True) + + if target.is_file() or target.is_symlink(): + target.unlink(missing_ok=True) + except OSError as err: + raise OSError(f"ERROR: Couldn't delete a file or directory: {err}") from err + + +def copy_files_dirs(to_copy: dict[str, Any], source_dir: Path, target_dir: Path) -> None: + """Copy files/directories within and to a project""" + for c_path, c_dict in to_copy.items(): + excludes = c_dict["excludes"] + files = c_dict["files"] + source = source_dir / c_path + + try: + if source.is_dir(): + # If the key in to_copy ends with / copy everything in source dir to target_dir + # er/ as the key copies everything from /opt/senzing/er to target_dir + # + # If the key in to_copy doesn't end with / copy source dir and everything in it to target_dir + # data as the key copies /opt/senzing/data to target_dir/data + target = target_dir if c_path.endswith("/") else target_dir / c_path + + # Copy entire contents of the source directory + if not files or (files and files[0] == "*"): + shutil.copytree(source, target, ignore=shutil.ignore_patterns(*excludes), dirs_exist_ok=True) + + # Create the source directory in the target and only copy listed files + if files and files[0] != "*": + target.mkdir(exist_ok=True, parents=True) + for source_file in [source / f for f in files]: + shutil.copy(source_file, target / source_file.name) + + if source.is_file(): + # Single file copy always copies only the file, if the key to to_copy is er/szBuildVersion.json + # szBuildVersion.json is copied to target_dir and not target_dir/er/szBuildVersion.json + target = target_dir / source.name + shutil.copy(source, target) + except OSError as err: + raise OSError(f"ERROR: Couldn't copy a file or directory: {err}") from err + + +def rename_files(to_rename: dict[str, dict[str, str]], target_dir: Path) -> None: + """Rename existing project files that had a name change""" + + try: + for r_path, r_dict in to_rename.items(): + current = target_dir / r_path / r_dict["from"] + new = target_dir / r_path / r_dict["to"] + with suppress(FileNotFoundError): + current.rename(new) + except OSError as err: + raise OSError(f"ERROR: Couldn't rename a file or directory: {err}") from err + + +def setup_env(proj_path: Path) -> None: + """Create a new setupEnv and replace place holders with paths for the project""" + try: + shutil.copy(proj_path / "resources/templates/setupEnv", proj_path) + setup_path = proj_path / "setupEnv" + + with open(setup_path, "r", encoding="utf-8") as in_: + data = in_.read() + + data = data.replace("${SENZING_DIR}", str(proj_path)).replace("${SENZING_CONFIG_PATH}", str(proj_path / "etc")) + + with open(setup_path, "w", encoding="utf-8") as out: + out.write(data) + except OSError as err: + raise OSError(f"ERROR: Couldn't create a new setupEnv file: {err}") from err + + +def set_permissions(proj_path: Path, permissions: dict[str, dict[str, Any]]) -> None: + """Set permissions for files/dirs copied to the project, or dirs removed and replaced completely e.g., data/""" + try: + for p_path, p_dict in permissions.items(): + dir_pint = p_dict["dir_pint"] + file_pint = p_dict["file_pint"] + files = p_dict["files"] + recursive = p_dict["recursive"] + target = proj_path if p_path.startswith(".") else proj_path / p_path + excludes = [target / e for e in p_dict["excludes"]] + + if target.is_dir(): + if dir_pint != 0: + target.chmod(dir_pint) + d_chmods = ( + [d for d in target.glob("*") if d.is_dir() and not d.is_symlink() and d not in excludes] + if not recursive + else [d for d in target.rglob("*") if d.is_dir() and not d.is_symlink() and d not in excludes] + ) + for dir_ in d_chmods: + dir_.chmod(dir_pint) + + if files and files[0] == "*": + f_chmods = ( + [f for f in target.glob("*") if f.is_file() and not f.is_symlink() and f not in excludes] + if not recursive + else [f for f in target.rglob("*") if f.is_file() and not f.is_symlink() and f not in excludes] + ) + + for file in f_chmods: + Path(target / file).chmod(file_pint) + + if files and files[0] != "*": + for file in files: + Path(target / file).chmod(file_pint) + + if target.is_file(): + target.chmod(file_pint) + except OSError as err: + raise OSError(f"ERROR: Couldn't set a permission: {err}") from err + + +def main() -> None: + """main""" + cli_args = parse_cli_args() + proj_path = Path(cli_args.project_path).expanduser().resolve() + proj_build, sys_build = pre_check(proj_path) + proj_is_v3 = bool(proj_build.major == 3) + + if not cli_args.force_mode: + print("\nWARNING: If you don't have a backup of the project, create one before continuing!") + sleep(3) + if ( + input(f"\nUpdate the project from version {proj_build.build_version} to {sys_build.build_version}? (y/n) ") + not in INPUT_CONFS + ): + sys.exit(0) + print("\nUpdating...") + else: + print(f"\nUpdating project from version {proj_build.build_version} to {sys_build.build_version}...") + + try: + if proj_is_v3: + v3_backup_path = proj_path / V3_BACKUP_PATH + v3_backup_path.mkdir(exist_ok=True) + copy_files_dirs(V3_BACKUP_PROJ, proj_path, v3_backup_path) + rename_files(V3_RENAME_IN_PROJ, proj_path) + remove_files_dirs(V3_REMOVE_FROM_PROJ, proj_path) + copy_files_dirs(COPY_TO_PROJ, SZ_SYS_PATH, proj_path) + if proj_is_v3: + setup_env(proj_path) + set_permissions(proj_path, PERMISSIONS) + set_permissions(proj_path, PERMISSIONS_2) + except OSError as err: + shutil.copytree(v3_backup_path, proj_path, dirs_exist_ok=True) + print(f"\n{err}") + print("\nIf the error is file or directory permission related, run again with appropriate privileges") + print( + "\nIf the error is missing file or directory, check senzingsdk-setup, senzingsdk-tools, and senzingsdk-poc are installed" + ) + else: + if proj_is_v3: # Only delete if no errors so re-running can find the file again + proj_path.joinpath(V3_BUILD).unlink(missing_ok=True) + + print(f"\nProject successfully updated. Refer to {UPGRADE_URL} for additional upgrade instructions") + + +if __name__ == "__main__": + main() diff --git a/sz_tools/sz_update_project_prior9 b/sz_tools/sz_update_project_prior9 new file mode 100755 index 0000000..129d272 --- /dev/null +++ b/sz_tools/sz_update_project_prior9 @@ -0,0 +1,496 @@ +#! /usr/bin/env python3 +"""Upgrade a V3 or V4 Senzing SDK project to V4.x.x""" + +import argparse + +# TODO - +# import json +import shutil +import sys +from contextlib import suppress + +# TODO - +# from dataclasses import dataclass, field +from pathlib import Path +from time import sleep +from typing import Any + +from _project_helpers import ( + SZ_SYS_PATH, + V4_BUILD, + V4_SYS_BUILD, + V4_SYS_PATH, + SzBuildDetails, + get_build_details, +) + +# from packaging import version as p_version + +INPUT_CONFS = ("y", "Y", "yEs", "yES", "YES", "yes", "YEs", "yeS", "Yes", "YeS") +MODULE_NAME = Path(__file__).stem +# TODO - +V3_BACKUP_PATH = "v3_to_v4_upgrade_backups" +V3_BUILD = "g2BuildVersion.json" +# V4_BUILD = "szBuildVersion.json" +# SZ_SYS_PATH = Path("/opt/senzing") +V3_SYS_PATH = SZ_SYS_PATH / "g2" +# V4_SYS_PATH = SZ_SYS_PATH / "er" +# V4_DATA_PATH = SZ_SYS_PATH / "data" +V3_SYS_BUILD = V3_SYS_PATH / V3_BUILD +# V4_SYS_BUILD = V4_SYS_PATH / V4_BUILD +UPGRADE_URL = "https://www.senzing.com/docs/4_beta/index.html" + +V3_BACKUP_PROJ = { + "bin": {"files": ["g2dbencrypt", "g2saltadm", "g2ssadm"], "excludes": []}, + "etc": {"files": ["senzing_governor.py"], "excludes": []}, + "lib": {"files": ["libSpaceTimeBoxStandardizer.so", "libG2Hasher.so", "libG2SSAdm.so"], "excludes": []}, + "python": { + "files": ["*"], + "excludes": [ + "CompressedFile.py", + "DumpStack.py", + "G2Audit.py", + "G2Command.py", + "G2ConfigTables.py", + "G2ConfigTool.py", + "G2Database.py", + "G2Explorer.py", + "G2Export.py", + "G2IniParams.py", + "G2Loader.py", + "G2Paths.py", + "G2Project.py", + "G2S3.py", + "G2SetupConfig.py", + "G2Snapshot.py", + "SenzingGo.py", + "senzing", + ], + }, + "setupEnv": {"files": [], "excludes": []}, + "g2BuildVersion.json": {"files": [], "excludes": []}, +} + +V3_REMOVE_FROM_PROJ = { + "bin": {"files": ["g2configupgrade", "g2dbencrypt", "g2dbupgrade", "g2saltadm", "g2ssadm"], "excludes": ["bin"]}, + "data": {"files": [], "excludes": []}, + "etc": {"files": ["senzing_governor.py"], "excludes": []}, + "lib": { + "files": [ + "g2.jar", + "libG2*.so", + "libG2Hasher.so", + "libG2SSAdm.so", + "libg2CompJavaScoreSet.so", + "libg2DistinctFeatJava.so", + "libg2EFeatJava.so", + "libg2JVMPlugin.so", + "libg2StdJava.so", + "libmariadbplugin.so", + "libSpaceTimeBoxStandardizer.so", + ], + "excludes": ["lib"], + }, + "python": {"files": ["*"], "excludes": []}, + "resources/config": {"files": ["g2core-configuration-upgrade-*.gtc"], "excludes": []}, + "resources/schema": {"files": ["g2core-schema-*-create.sql", "g2core-schema-*-upgrade-*.sql"], "excludes": []}, + "resources/templates": { + "files": [ + "G2C.db*", + "G2Module.ini", + "cfgVariant.json", + "custom*.txt", + "defaultGNRCP.config", + "g2config.json", + "senzing_governor.py", + "setupEnv", + "stb.config", + ], + "excludes": [], + }, + "sdk": {"files": ["*"], "excludes": []}, + "setupEnv": {"files": [], "excludes": []}, +} + +COPY_TO_PROJ = { + "er/": {"files": ["*"], "excludes": ["sz_create_project", "sz_update_project"]}, + "data": {"files": ["*"], "excludes": []}, +} + +V3_RENAME_IN_PROJ = { + "etc": {"from": "G2Module.ini", "to": "sz_engine_config.ini"}, +} + +# TODO - +PERMISSIONS = { + ".": { + "dir_pint": 0, + "file_pint": 0o660, + "files": ["LICENSE", "NOTICES", "README.1ST", "szBuildVersion.json"], + "excludes": ["setupEnv"], + "recursive": False, + }, + "setupEnv": {"dir_pint": 0, "file_pint": 0o770, "files": [], "excludes": [], "recursive": False}, + "bin": { + "dir_pint": 0o770, + "file_pint": 0o770, + "files": ["*"], + "excludes": ["__pycache__", "_sz_database.py", "_tool_helpers.py"], + "recursive": False, + }, + "data": {"dir_pint": 0o770, "file_pint": 0o660, "files": ["*"], "excludes": [], "recursive": True}, + "lib": {"dir_pint": 0o770, "file_pint": 0o660, "files": ["*"], "excludes": [], "recursive": False}, + "resources": {"dir_pint": 0o770, "file_pint": 0o660, "files": ["*"], "excludes": ["setupEnv"], "recursive": True}, + "resources/templates/setupEnv": { + "dir_pint": 0, + "file_pint": 0o770, + "files": [], + "excludes": [], + "recursive": False, + }, + "sdk": {"dir_pint": 0o770, "file_pint": 0o660, "files": ["*"], "excludes": [], "recursive": True}, + V3_BACKUP_PATH: {"dir_pint": 0o770, "file_pint": 0, "files": [], "excludes": [], "recursive": False}, +} +PERMISSIONS_2 = { + "bin": { + "dir_pint": 0o770, + "file_pint": 0o660, + "files": ["_sz_database.py", "_tool_helpers.py"], + "excludes": [], + "recursive": False, + } +} + +# TODO - +# @dataclass() +# class SzBuildDetails: +# """Build information for a project or Senzing SDK system install""" + +# platform: str +# version: str +# build_version: str +# build_number: str +# major: int = field(init=False) +# minor: int = field(init=False) +# micro: int = field(init=False) + +# def __post_init__(self) -> None: +# self.version_parsed = p_version.parse(self.version) +# self.build_version_parsed = p_version.parse(self.build_version) +# self.major = self.version_parsed.major +# self.minor = self.version_parsed.minor +# self.micro = self.version_parsed.micro + + +# pylint: disable=W0106 + + +def parse_cli_args() -> argparse.Namespace: + """Parse the CLI arguments""" + arg_parser = argparse.ArgumentParser( + allow_abbrev=False, + description="Update an existing V3 or V4 Senzing project to the system installed V4 of Senzing.", + ) + arg_parser.add_argument( + "project_path", + metavar="path", + help="Path of the project to update", + ) + arg_parser.add_argument( + "-f", + "--force", + dest="force_mode", + default=False, + action="store_true", + help="Upgrade without prompts. WARNING: Use with caution, ensure you have a backup of the project", + ) + return arg_parser.parse_args() + + +def dir_listing(path: Path) -> list[Path]: + """Get a listing of the specified path to check for files or directories""" + try: + listing = list(path.iterdir()) + except OSError as err: + if type(err).__name__ == "NotADirectoryError": + print(f"\nERROR: {path} is not a directory, expecting one") + print(f"\nERROR: {err}") + sys.exit(1) + + return listing + + +def pre_check(project_path: Path) -> tuple[SzBuildDetails, SzBuildDetails]: + """Check not trying to overwrite the V4 Senzing system install, that path is a project, and versions are correct""" + if not V4_SYS_PATH.exists(): + print(f"\nERROR: Couldn't locate Senzing SDK V4 system install at {V4_SYS_PATH}") + sys.exit(1) + + if str(project_path).startswith(str(SZ_SYS_PATH)): + print(f"\nERROR: {project_path} is the Senzing system installation, not a Senzing project") + sys.exit(1) + + v3_project_build_file = project_path / V3_BUILD + v4_project_build_file = project_path / V4_BUILD + proj_listing = dir_listing(project_path) + if v3_project_build_file not in proj_listing and v4_project_build_file not in proj_listing: + print(f"\nERROR: {project_path} isn't a Senzing project, expected it to contain either:") + print(f"\tExisting V3 project: {v3_project_build_file}") + print(f"\tExisting V4 project: {v4_project_build_file}") + sys.exit(1) + + try: + proj_build_file = v3_project_build_file if v3_project_build_file in proj_listing else v4_project_build_file + proj_build_details = get_build_details(proj_build_file) + sys_build_details = get_build_details(V4_SYS_BUILD) + except OSError as err: + print(f"\nERROR: Trying to read {proj_build_file} or {V4_SYS_BUILD} to collect version information: {err}") + sys.exit(1) + # TODO - + except TypeError as err: + print(f"\nERROR: {err}") + sys.exit(1) + + if proj_build_details.major not in (3, 4) or sys_build_details.major != 4: + print(f"\nERROR: {MODULE_NAME} updates a V3 project to V4, or V4.n.n to a newer release") + print(f"\tProject version: {proj_build_details.version}") + print(f"\tSystem install version: {sys_build_details.version}") + sys.exit(1) + + if sys_build_details.version_parsed < proj_build_details.version_parsed: + print("\nNo update required, project is newer than the system install") + print(f"\tProject version: {proj_build_details.version}") + print(f"\tSystem install version: {sys_build_details.version}") + sys.exit(0) + + if sys_build_details.version_parsed == proj_build_details.version_parsed: + print(f"\nNo update required, project and system install are the same version - {proj_build_details.version}") + sys.exit(0) + + return (proj_build_details, sys_build_details) + + +# TODO - +# def get_build_details(path: Path) -> SzBuildDetails: +# """Return dataclass with the details from a build file.""" +# try: +# with open(path, "r", encoding="utf-8") as f: +# # Ignore DATA_VERSION, V3 build files had it V4 doesn't +# version_dict = {k.lower(): v for k, v in json.load(f).items() if k != "DATA_VERSION"} +# return SzBuildDetails(**version_dict) +# except (OSError, json.JSONDecodeError) as err: +# print(f"\nERROR: Couldn't get the build information from {path}: {err}") +# sys.exit(1) +# except TypeError as err: +# print(f"\nERROR: When creating build information from {path}: {err}") +# sys.exit(1) + + +def remove_dir(dir_: Path, excludes: list[Path]) -> None: + """Recursively remove a directory""" + try: + for path in dir_.iterdir(): + if path in excludes: + continue + path.unlink(missing_ok=True) if (path.is_file() or path.is_symlink()) else remove_dir(path, excludes) + + if not excludes: + with suppress(FileNotFoundError): + dir_.rmdir() + except OSError as err: + raise err + + +def remove_files_dirs(to_remove: dict[str, Any], target_dir: Path) -> None: + """Remove files/directories that are no longer required""" + for r_path, r_dict in to_remove.items(): + target: Path = target_dir / r_path + files = r_dict["files"] + excludes = [target / e for e in r_dict["excludes"]] + + try: + if target.is_dir(): + if not files or (files and files[0] == "*"): + remove_dir(target, excludes) + + if files and files[0] != "*": + target_files = [] + for f in files: + target_files.append(target / f) if "*" not in f else target_files.extend(list(target.glob(f))) + + for target_file in target_files: + target_file.unlink(missing_ok=True) + + if target.is_file() or target.is_symlink(): + target.unlink(missing_ok=True) + except OSError as err: + # TODO - + # raise OSError(f"ERROR: Couldn't delete a file or directory: {err}") from err + raise OSError(f"Couldn't delete a file or directory: {err}") from err + + +def copy_files_dirs(to_copy: dict[str, Any], source_dir: Path, target_dir: Path) -> None: + """Copy files/directories within and to a project""" + for c_path, c_dict in to_copy.items(): + excludes = c_dict["excludes"] + files = c_dict["files"] + source = source_dir / c_path + + try: + if source.is_dir(): + # If the key in to_copy ends with / copy everything in source dir to target_dir + # er/ as the key copies everything from /opt/senzing/er to target_dir + # + # If the key in to_copy doesn't end with / copy source dir and everything in it to target_dir + # data as the key copies /opt/senzing/data to target_dir/data + target = target_dir if c_path.endswith("/") else target_dir / c_path + + # Copy entire contents of the source directory + if not files or (files and files[0] == "*"): + shutil.copytree(source, target, ignore=shutil.ignore_patterns(*excludes), dirs_exist_ok=True) + + # Create the source directory in the target and only copy listed files + if files and files[0] != "*": + target.mkdir(exist_ok=True, parents=True) + for source_file in [source / f for f in files]: + shutil.copy(source_file, target / source_file.name) + + if source.is_file(): + # Single file copy always copies only the file, if the key to to_copy is er/szBuildVersion.json + # szBuildVersion.json is copied to target_dir and not target_dir/er/szBuildVersion.json + target = target_dir / source.name + shutil.copy(source, target) + except OSError as err: + # TODO - + # raise OSError(f"ERROR: Couldn't copy a file or directory: {err}") from err + raise OSError(f"Couldn't copy a file or directory: {err}") from err + + +def rename_files(to_rename: dict[str, dict[str, str]], target_dir: Path) -> None: + """Rename existing project files that had a name change""" + + try: + for r_path, r_dict in to_rename.items(): + current = target_dir / r_path / r_dict["from"] + new = target_dir / r_path / r_dict["to"] + with suppress(FileNotFoundError): + current.rename(new) + except OSError as err: + # TODO - + # raise OSError(f"ERROR: Couldn't rename a file or directory: {err}") from err + raise OSError(f"Couldn't rename a file or directory: {err}") from err + + +# TODO - +def setup_env(proj_path: Path) -> None: + """Create a new setupEnv and replace place holders with paths for the project""" + try: + shutil.copy(proj_path / "resources/templates/setupEnv", proj_path) + setup_path = proj_path / "setupEnv" + + with open(setup_path, "r", encoding="utf-8") as in_: + data = in_.read() + + data = data.replace("${SENZING_DIR}", str(proj_path)).replace("${SENZING_CONFIG_PATH}", str(proj_path / "etc")) + + with open(setup_path, "w", encoding="utf-8") as out: + out.write(data) + except OSError as err: + # TODO - + # raise OSError(f"ERROR: Couldn't create a new setupEnv file: {err}") from err + raise OSError(f"Couldn't create a new setupEnv file: {err}") from err + + +# TODO - +def set_permissions(proj_path: Path, permissions: dict[str, dict[str, Any]]) -> None: + """Set permissions for files/dirs copied to the project, or dirs removed and replaced completely e.g., data/""" + try: + for p_path, p_dict in permissions.items(): + dir_pint = p_dict["dir_pint"] + file_pint = p_dict["file_pint"] + files = p_dict["files"] + recursive = p_dict["recursive"] + target = proj_path if p_path.startswith(".") else proj_path / p_path + excludes = [target / e for e in p_dict["excludes"]] + + if target.is_dir(): + if dir_pint != 0: + target.chmod(dir_pint) + d_chmods = ( + [d for d in target.glob("*") if d.is_dir() and not d.is_symlink() and d not in excludes] + if not recursive + else [d for d in target.rglob("*") if d.is_dir() and not d.is_symlink() and d not in excludes] + ) + for dir_ in d_chmods: + dir_.chmod(dir_pint) + + if files and files[0] == "*": + f_chmods = ( + [f for f in target.glob("*") if f.is_file() and not f.is_symlink() and f not in excludes] + if not recursive + else [f for f in target.rglob("*") if f.is_file() and not f.is_symlink() and f not in excludes] + ) + + for file in f_chmods: + Path(target / file).chmod(file_pint) + + if files and files[0] != "*": + for file in files: + Path(target / file).chmod(file_pint) + + if target.is_file(): + target.chmod(file_pint) + except OSError as err: + # TODO - + # raise OSError(f"ERROR: Couldn't set a permission: {err}") from err + raise OSError(f"Couldn't set a permission: {err}") from err + + +def main() -> None: + """main""" + cli_args = parse_cli_args() + proj_path = Path(cli_args.project_path).expanduser().resolve() + proj_build, sys_build = pre_check(proj_path) + proj_is_v3 = bool(proj_build.major == 3) + + if not cli_args.force_mode: + print("\nWARNING: If you don't have a backup of the project, create one before continuing!") + sleep(3) + if ( + input(f"\nUpdate the project from version {proj_build.build_version} to {sys_build.build_version}? (y/n) ") + not in INPUT_CONFS + ): + sys.exit(0) + print("\nUpdating...") + else: + print(f"\nUpdating project from version {proj_build.build_version} to {sys_build.build_version}...") + + try: + if proj_is_v3: + v3_backup_path = proj_path / V3_BACKUP_PATH + v3_backup_path.mkdir(exist_ok=True) + copy_files_dirs(V3_BACKUP_PROJ, proj_path, v3_backup_path) + rename_files(V3_RENAME_IN_PROJ, proj_path) + remove_files_dirs(V3_REMOVE_FROM_PROJ, proj_path) + copy_files_dirs(COPY_TO_PROJ, SZ_SYS_PATH, proj_path) + if proj_is_v3: + setup_env(proj_path) + set_permissions(proj_path, PERMISSIONS) + set_permissions(proj_path, PERMISSIONS_2) + except OSError as err: + shutil.copytree(v3_backup_path, proj_path, dirs_exist_ok=True) + # TODO - Might need ERROR: and change the raise in some places + print(f"\nERROR: {err}") + print("\nIf the error is file or directory permission related, run again with appropriate privileges") + print( + "\nIf the error is missing file or directory, check senzingsdk-setup, senzingsdk-tools, and senzingsdk-poc are installed" + ) + else: + if proj_is_v3: # Only delete if no errors so re-running can find the file again + proj_path.joinpath(V3_BUILD).unlink(missing_ok=True) + + print(f"\nProject successfully updated. Refer to {UPGRADE_URL} for additional upgrade instructions") + + +if __name__ == "__main__": + main() From 94f1e997feda714dd6cfdd966a7cea2bcaf8fed1 Mon Sep 17 00:00:00 2001 From: Ant Date: Thu, 19 Jun 2025 16:28:21 +0100 Subject: [PATCH 2/8] #188 - Save point --- sz_tools/sz_command | 2 +- sz_tools/sz_configtool | 2 +- sz_tools/sz_export | 70 +++++++++++++++++++----------------------- 3 files changed, 34 insertions(+), 40 deletions(-) diff --git a/sz_tools/sz_command b/sz_tools/sz_command index 53c08c0..4a24ef8 100755 --- a/sz_tools/sz_command +++ b/sz_tools/sz_command @@ -660,7 +660,7 @@ class SzCmdShell(cmd.Cmd): if self.debug_reinit["prior_instance_reinitialized"]: print_response("Senzing engines were reinitialized due to a configuration change", color="success") else: - print_info(f"Welcome to {MODULE_NAME}. Type help or ? for help", info_prefix=False) + print_info(f"Type help or ? for help", info_prefix=False) # ------------------------------------------------------------------------- # Non-interactive input methods diff --git a/sz_tools/sz_configtool b/sz_tools/sz_configtool index 11afad4..7b5985e 100755 --- a/sz_tools/sz_configtool +++ b/sz_tools/sz_configtool @@ -626,7 +626,7 @@ class SzCfgShell(cmd.Cmd): def preloop(self): self.load_config() colorize_msg( - "Welcome to the Senzing configuration tool. Type help or ? for help", + "Type help or ? for help", "highlight2", ) diff --git a/sz_tools/sz_export b/sz_tools/sz_export index 6a48d6c..4f8c5ad 100755 --- a/sz_tools/sz_export +++ b/sz_tools/sz_export @@ -257,9 +257,9 @@ if __name__ == "__main__": help=textwrap.dedent( """\ - Path and file name to send output to. + Path and file name to send output to. - """ + """ ), ) @@ -272,9 +272,9 @@ if __name__ == "__main__": help=textwrap.dedent( """\ - Path and file name of optional sz_engine_config.ini to use. + Path and file name of optional sz_engine_config.ini to use. - """ + """ ), ) @@ -286,16 +286,11 @@ if __name__ == "__main__": help=textwrap.dedent( """\ - Space separated list of export flags to apply to the export. - - Valid flags: + Space separated list of export flags to apply to the export. - ... - ... + Default: %(default)s - Default: %(default)s - - """ + """ ), ) @@ -308,11 +303,11 @@ if __name__ == "__main__": help=textwrap.dedent( """\ - Data format to export to, JSON or CSV. + Data format to export to, JSON or CSV. - Default: %(default)s + Default: %(default)s - """ + """ ), ) @@ -324,20 +319,20 @@ if __name__ == "__main__": help=textwrap.dedent( """\ - Return extended details, adds RESOLVED_ENTITY_NAME & JSON_DATA. + Return extended details, adds RESOLVED_ENTITY_NAME & JSON_DATA. - Adding JSON_DATA significantly increases the size of the output and execution time. + Adding JSON_DATA significantly increases the size of the output and execution time. - When used with CSV output, JSON_DATA isn\'t included for the related entities - (RELATED_ENTITY_ID) for each resolved entity (RESOLVED_ENTITY_ID). This reduces - the size of a CSV export by preventing repeating data for related entities. JSON_DATA - for the related entities is still included in the CSV export and is located in the - export record where the RELATED_ENTITY_ID = RESOLVED_ENTITY_ID. + When used with CSV output, JSON_DATA isn\'t included for the related entities + (RELATED_ENTITY_ID) for each resolved entity (RESOLVED_ENTITY_ID). This reduces + the size of a CSV export by preventing repeating data for related entities. JSON_DATA + for the related entities is still included in the CSV export and is located in the + export record where the RELATED_ENTITY_ID = RESOLVED_ENTITY_ID. - WARNING: This is not recommended! To include the JSON_DATA for every CSV record see the - --extendCSVRelates (-xcr) argument. + WARNING: This is not recommended! To include the JSON_DATA for every CSV record see the + --extendCSVRelates (-xcr) argument. - """ + """ ), ) @@ -349,11 +344,11 @@ if __name__ == "__main__": help=textwrap.dedent( """\ - Frequency of export output statistics. + Frequency of export output statistics. - Default: %(default)s + Default: %(default)s - """ + """ ), ) @@ -368,14 +363,14 @@ if __name__ == "__main__": help=textwrap.dedent( f"""\ - Compress output file with gzip. Compression level can be optionally specified. + Compress output file with gzip. Compression level can be optionally specified. - If output file is specified as - (for stdout), use shell redirection instead to compress: - {MODULE_NAME} -o - | gzip -v > export.csv.gz + If output file is specified as - (for stdout), use shell redirection instead to compress: + {MODULE_NAME} -o - | gzip -v > export.csv.gz - Default: %(const)s + Default: %(const)s - """ + """ ), ) @@ -387,13 +382,13 @@ if __name__ == "__main__": help=textwrap.dedent( """\ - WARNING: Use of this argument is not recommended! + WARNING: Use of this argument is not recommended! - Used in addition to --extend (-x), it will include JSON_DATA in CSV output for related entities. + Used in addition to --extend (-x), it will include JSON_DATA in CSV output for related entities. - Only valid for CSV output format. + Only valid for CSV output format. - """ + """ ), ) @@ -489,7 +484,6 @@ if __name__ == "__main__": print(", ".join(invalid_string_flags)) valid_flags = [flag for flag in flags if flag not in invalid_string_flags] - final_flags = combine_engine_flags(valid_flags) # Initialize the export From 801ab2fe27b0c8596f9f4f27fcf1220ac10da6d8 Mon Sep 17 00:00:00 2001 From: Ant Date: Thu, 19 Jun 2025 16:59:52 +0100 Subject: [PATCH 3/8] 188 - Save point --- sz_tools/sz_explorer | 471 +++++++++++++++++++++++++++++-------------- 1 file changed, 318 insertions(+), 153 deletions(-) diff --git a/sz_tools/sz_explorer b/sz_tools/sz_explorer index fe5afd6..d60bc9c 100755 --- a/sz_tools/sz_explorer +++ b/sz_tools/sz_explorer @@ -34,18 +34,16 @@ LF = "\n" LFTAB = "\n\t" try: - from prettytable import ALL, SINGLE_BORDER, PrettyTable -except ImportError: - print("\nPlease install the Python prettytable module (e.g. pip install prettytable)") + import prettytable +except ImportError as ex: + print(f"\n{ex}\nPlease install the Python prettytable module (e.g. pip install prettytable)") sys.exit(1) -def print_exception_info(ex): - print_message(ex, "error") - response = get_char_with_prompt("press T for traceback...") - if response.upper() == "T": +def print_exception(ex): + print_message(f"{ex}, press any key to continue", "error") + if get_char() in "tTdD": print(traceback.format_exc()) - get_char_with_prompt("press any key any key to continue...") print() @@ -205,6 +203,7 @@ class eda_node(object): self.node_data = None self.children = [] self.parents = [] + self.counter1 = 0 def add_child(self, obj): self.children.append(obj) @@ -217,17 +216,17 @@ class eda_node(object): tree += self.node_desc + "\n" if self.node_text: tree += self.node_text + "\n" - parents = [{"node": self, "next_child": 0, "display_children": self.children}] + parents = [{"node": self, "display_children": self.children}] while parents: - if parents[-1]["next_child"] == len(parents[-1]["display_children"]): + if len(parents[-1]["display_children"]) == 0: parents.pop() continue + next_node = parents[-1]["display_children"][0] + parents[-1]["display_children"].pop(0) - next_node = parents[-1]["display_children"][parents[-1]["next_child"]] - parents[-1]["next_child"] += 1 prefix = "" for i, _ in enumerate(parents): - last_child = parents[i]["next_child"] == len(parents[i]["display_children"]) + last_child = len(parents[i]["display_children"]) == 0 if i < len(parents) - 1: # prior level prefix += " " if last_child else "\u2502 " else: @@ -247,15 +246,9 @@ class eda_node(object): tree += prefix + line + "\n" if next_node.children: - prior_parents = [] - for i, _ in enumerate(parents): - prior_parents.append(parents[i]["node"].node_id) - # don't display prior parents in this tree - display_children = [] - for child in next_node.children: - if child.node_id not in prior_parents: - display_children.append(child) - parents.append({"node": next_node, "next_child": 0, "display_children": display_children}) + prior_parents = [x["node"].node_id for x in parents] + display_children = [x for x in next_node.children if x.node_id not in prior_parents] + parents.append({"node": next_node, "display_children": display_children}) return tree @@ -276,16 +269,22 @@ class eda_table: super_header = kwargs.get("super_header") no_lines = kwargs.get("no_lines") side_header = kwargs.get("side_header") - table_object = PrettyTable() + table_object = prettytable.PrettyTable() if org_mode: table_object.horizontal_char = "\u2500" table_object.vertical_char = "\u2502" table_object.junction_char = "\u253c" else: - table_object.set_style(SINGLE_BORDER) + if hasattr(prettytable, "TableStyle"): + table_object.set_style(prettytable.TableStyle.SINGLE_BORDER) + else: + table_object.set_style(prettytable.SINGLE_BORDER) if not no_lines: - table_object.hrules = ALL + if hasattr(prettytable, "HRuleStyle"): + table_object.hrules = prettytable.HRuleStyle.ALL + else: + table_object.hrules = prettytable.ALL field_name_list = [] column_header_list = [] @@ -410,12 +409,13 @@ class EdaReports: # (feature_counts) for ftype_code in feature_counts: ftype_excl = sdk_wrapper.ftype_code_lookup[ftype_code]["FTYPE_EXCL"] + ftype_freq = sdk_wrapper.ftype_code_lookup[ftype_code]["FTYPE_FREQ"] ftype_count = feature_counts[ftype_code] if ftype_count > entity_size: # a single record has reported multiple continue if ftype_excl == "Yes" and ftype_count > 1: review_features.append(ftype_code) - elif ftype_count > review_max: + elif ftype_freq in ("F1", "FF") and ftype_count > review_max: review_features.append(ftype_code) return review_features @@ -560,7 +560,7 @@ class EdaReports: tbl = eda_table() tbl.title = "Review categories..." tbl.columns = [ - {"name": "row", "width": 5, "align": "center"}, + {"name": "Row", "width": 5, "align": "center"}, {"name": "Category", "width": 25, "align": "left"}, {"name": "Count", "width": 25, "align": "right"}, ] @@ -617,7 +617,7 @@ class EdaReports: tbl = eda_table() tbl.title = f"Entity size breakdown from {self.snapshot_file}" tbl.columns = [ - {"name": "row", "width": 5, "align": "center"}, + {"name": "Row", "width": 5, "align": "center"}, {"name": "Size Group", "width": 10, "align": "center"}, {"name": "Entity Count", "width": 10, "align": "right"}, {"name": "Review Count", "width": 10, "align": "right"}, @@ -647,7 +647,7 @@ class EdaReports: tbl = eda_table() tbl.title = f"Data Source Summary from {self.snapshot_file}" tbl.columns = [ - {"name": "\nrow", "width": 5, "align": "center"}, + {"name": "\nRow", "width": 5, "align": "center"}, {"name": "\nData Source", "width": 25, "align": "left"}, {"name": "\nRecords", "width": 15, "align": "right"}, {"name": "\nEntities", "width": 15, "align": "right"}, @@ -701,7 +701,7 @@ class EdaReports: tbl = eda_table() tbl.title = f"Cross Source Summary from {self.snapshot_file}" tbl.columns = [ - {"name": "\nrow", "width": 5, "align": "center"}, + {"name": "\nRow", "width": 5, "align": "center"}, {"name": "From\nData Source", "width": 25, "align": "center"}, {"name": "To\nData Source", "width": 25, "align": "center"}, {"name": "Matched\nRecords", "width": 15, "align": "right"}, @@ -746,25 +746,25 @@ class EdaReports: select_levels = ["DATA_SOURCES", "MATCH_LEVEL", "MATCH_KEY"] self.drill_into(report_table, report_data, select_levels) - def multi_source_summary(self, data_source_filter): + def entity_source_summary(self, data_source_filter): self.check_for_snapshot() report_data = {} tbl = eda_table() - tbl.title = f"Multi-Source Summary from {self.snapshot_file}" + tbl.title = f"Entity Source Summary from {self.snapshot_file}" tbl.columns = [ - {"name": "row", "width": 5, "align": "center"}, + {"name": "Row", "width": 5, "align": "center"}, {"name": "Data Sources", "width": 100, "align": "left"}, - {"name": "Records", "width": 15, "align": "right"}, + {"name": "Entities", "width": 15, "align": "right"}, ] tbl.rows = [] row_num = 0 - _data = self.snapshot_data["MULTI_SOURCES"] + _data = self.snapshot_data["ENTITY_SOURCES"] sorted_data = sorted(_data, key=lambda k: _data[k]["ENTITY_COUNT"], reverse=True) for data_sources in sorted_data: if data_source_filter and data_source_filter.upper() not in data_sources: continue row_num += 1 - report_segment = self.snapshot_data["MULTI_SOURCES"][data_sources] + report_segment = self.snapshot_data["ENTITY_SOURCES"][data_sources] row = [ colorize(row_num, "bold"), colorize(" | ", "dim").join(colorize_dsrc(x) for x in data_sources.split("||")), @@ -804,7 +804,7 @@ class EdaReports: tbl = eda_table() tbl.title = f"Principles Used Report from {self.snapshot_file}" tbl.columns = [ - {"name": "row", "width": 5, "align": "center"}, + {"name": "Row", "width": 5, "align": "center"}, {"name": "Match level", "width": 25, "align": "left"}, {"name": "Count", "width": 15, "align": "right"}, ] @@ -839,7 +839,7 @@ class EdaReports: tbl = eda_table() tbl.title = f"Selected {prior_keys}" tbl.columns = [ - {"name": "row", "width": 5, "align": "center"}, + {"name": "Row", "width": 5, "align": "center"}, {"name": select_level.lower(), "width": 100, "align": "left"}, {"name": "count", "width": 10, "align": "right"}, ] @@ -1001,7 +1001,7 @@ class EdaReports: report = sdk_wrapper.compare_entities(entity_list) view_report(report, no_scroll=True, extra_header=item_header) - valid_responses = ["P", "N", "G", "S", "Q", "E"] + valid_responses = ["P", "N", "G", "S", "Q", "E", "C"] if len(entity_list) == 1: prompt = "Select (P)revious, (N)ext, (G)oto, (H)ow, (E)xport, (S)croll, (Q)uit..." valid_responses += ["D", "H"] @@ -1183,7 +1183,6 @@ class EdaSdkWrapper: parms = parms if isinstance(parms, list) else [parms] caller = inspect.stack()[1].function try: - # flags_int = SzEngineFlags.combine_flags(flags) flags_int = combine_engine_flags(flags) parms.append(flags_int) api_call = getattr(self.sz_engine, api_name) @@ -1590,7 +1589,7 @@ class EdaSdkWrapper: line_cnt = 0 if include_feature_code: if ftype_code in self.senzing_features: - colored_ftype = colorize(ftype_code, "caution") + ": " + colored_ftype = colorize(ftype_code, "bad") + ": " else: colored_ftype = colorize_attr(ftype_code) + ": " else: @@ -1646,7 +1645,7 @@ class EdaSdkWrapper: feat_desc = feature["FEAT_DESC"] stats = self.fmt_feature_stats(feature) feat_color = "" - if feature.get("MATCH_SCORE", 0) > 0: + if feature.get("MATCH_SCORE", -1) >= 0: if feature.get("WAS_CANDIDATE"): # always 100 feat_color = "highlight2" elif feature.get("SCORE_BUCKET") in ("SAME", "CLOSE"): @@ -1656,7 +1655,7 @@ class EdaSdkWrapper: else: feat_color = "caution" elif feature["FTYPE_CODE"] in self.senzing_features: - feat_color = "caution" + feat_color = "bad" if any(x in stats for x in ("~", "!", "#")): feat_color += ",dim" if feat_color else "dim" @@ -1962,7 +1961,7 @@ class EdaSdkWrapper: entity_only_features = [] for feat_data in entity_features.values(): if feat_data["RECORD_COUNT"] == 0: - entity_only_features.append(f"{colorize(feat_data['FTYPE_CODE'], 'caution')}: {feat_data['FEAT_DESC']}") + entity_only_features.append(f"{colorize(feat_data['FTYPE_CODE'], 'bad')}: {feat_data['FEAT_DESC']}") if entity_only_features: tbl.rows.insert(0, [colorize_dsrc("SENZING"), "\n".join(entity_only_features), ""]) @@ -2020,72 +2019,71 @@ class EdaSdkWrapper: raise err from err entities = self.get_resolved_entities(json_data["ENTITIES"]) - if build_out_degree == 1: # categorize and group - category_counts = {} - categories = {} - related_entities = entities[root_entity_id]["RELATIONSHIPS"] - for relation in sorted( - related_entities, key=lambda k: (k["MATCH_CATEGORY_SORT"], k["ERRULE_ID"], k["ENTITY_ID"]) - ): - match_category = relation["MATCH_CATEGORY"] - match_key = relation["MATCH_KEY"] - if match_category not in categories: - categories[match_category] = {} - category_counts[match_category] = 1 - else: - category_counts[match_category] = 1 - match_group = self.fmt_match_group(match_category, match_key) - if match_group not in categories[match_category]: - categories[match_category][match_group] = [relation] - else: - categories[match_category][match_group].append(relation) - - root_node = eda_node(root_entity_id) - root_node.node_desc = self.fmt_entity_desc(entities[root_entity_id]) - for category in categories: - category_node = eda_node(category) - category_desc = f"{self.match_category_desc[category]} ({category_counts[category]})" - category_node.node_desc = category_desc - root_node.add_child(category_node) - for group in categories[category]: - group_node = eda_node(group) - group_desc = f"{group} ({len(categories[category][group])})" - group_node.node_desc = group_desc - category_node.add_child(group_node) - for relation in categories[category][group]: - related_id = relation["ENTITY_ID"] - rel_node = eda_node(related_id) - rel_desc = self.fmt_entity_desc(entities[related_id]) - rel_desc += " " + self.fmt_match_key({"MATCH_KEY": relation["MATCH_KEY"]}) - rel_node.node_desc = rel_desc - group_node.add_child(rel_node) - else: - input("NEVER GETS HERE") - tree_nodes = {} - for entity_id in entities: - if entity_id not in tree_nodes: - tree_nodes[entity_id] = eda_node(entity_id) - tree_nodes[entity_id].node_desc = self.fmt_entity_desc(entities[entity_id]) - related_entities = entities[entity_id]["RELATIONSHIPS"] - for relation in sorted( - related_entities, key=lambda k: (k["MATCH_CATEGORY_SORT"], k["ERRULE_ID"], k["ENTITY_ID"]) - ): - related_id = relation["ENTITY_ID"] - if related_id not in entities: - continue - if related_id not in tree_nodes: - tree_nodes[related_id] = eda_node(related_id) - node_desc = self.fmt_entity_desc(entities[related_id]) - node_desc += " " + self.fmt_match_key({"MATCH_KEY": relation["MATCH_KEY"]}) - tree_nodes[related_id].node_desc = node_desc - - if related_id not in tree_nodes[entity_id].children: - tree_nodes[entity_id].add_child(tree_nodes[related_id]) - else: - print_message("Related_id was already added to child list", "warning") - root_node = tree_nodes[root_entity_id] - return root_node.render_tree() + related_entities = sorted( + entities[root_entity_id]["RELATIONSHIPS"], + key=lambda k: (k["MATCH_CATEGORY_SORT"], k["ERRULE_ID"], k["ENTITY_ID"]), + ) + parents = [ + { + "entity_id": root_entity_id, + "relationships": related_entities, + } + ] + tree_nodes = {} + while parents: + if len(parents[-1]["relationships"]) == 0: + parents.pop() + continue + entity_id = parents[-1]["entity_id"] + if entity_id not in tree_nodes: + tree_nodes[entity_id] = eda_node(entity_id) + tree_nodes[entity_id].node_desc = self.fmt_entity_desc(entities[entity_id]) + + relation = parents[-1]["relationships"][0] + parents[-1]["relationships"].pop(0) + related_id = relation["ENTITY_ID"] + if related_id not in entities or related_id in [x["entity_id"] for x in parents]: + continue + + category = relation["MATCH_CATEGORY"] + group = self.fmt_match_group(category, relation["MATCH_KEY"]) + category_node_id = f"{entity_id}-{category}" + group_node_id = f"{entity_id}-{category}-{group}" + + if category_node_id not in tree_nodes: + tree_nodes[category_node_id] = eda_node(category_node_id) + tree_nodes[entity_id].add_child(tree_nodes[category_node_id]) + tree_nodes[category_node_id].counter1 += 1 + tree_nodes[category_node_id].node_desc = ( + f"{self.match_category_desc[category]} ({tree_nodes[category_node_id].counter1 })" + ) + + if group_node_id not in tree_nodes: + tree_nodes[group_node_id] = eda_node(group_node_id) + tree_nodes[category_node_id].add_child(tree_nodes[group_node_id]) + tree_nodes[group_node_id].counter1 += 1 + tree_nodes[group_node_id].node_desc = f"{group} ({tree_nodes[group_node_id].counter1})" + + if related_id not in tree_nodes: + rel_desc = self.fmt_entity_desc(entities[related_id]) + rel_desc += " " + self.fmt_match_key({"MATCH_KEY": relation["MATCH_KEY"]}) + tree_nodes[related_id] = eda_node(related_id) + tree_nodes[related_id].node_desc = rel_desc + tree_nodes[group_node_id].add_child(tree_nodes[related_id]) + + if len(entities[related_id]["RELATIONSHIPS"]) != 0 and len(parents) < build_out_degree: + related_entities = sorted( + entities[related_id]["RELATIONSHIPS"], + key=lambda k: (k["MATCH_CATEGORY_SORT"], k["ERRULE_ID"], k["ENTITY_ID"]), + ) + parents.append( + { + "entity_id": related_id, + "relationships": related_entities, + } + ) + return tree_nodes[root_entity_id].render_tree() def compare_relationships(self, entity_id, **kwargs): """compare an entity's relationships side by side""" @@ -2252,7 +2250,7 @@ class EdaSdkWrapper: for lib_feat_id in features_by_id: ftype_codes_used.append(features_by_id[lib_feat_id]["FTYPE_CODE"]) if lib_feat_id in scores_by_id: - if scores_by_id[lib_feat_id].get("MATCH_SCORE"): + if "MATCH_SCORE" in scores_by_id[lib_feat_id]: features_by_id[lib_feat_id].update(scores_by_id[lib_feat_id]) entity_data[_id]["FEATURES"] = self.regroup_by_type(features_by_id) @@ -2277,7 +2275,11 @@ class EdaSdkWrapper: for ftype_code in sorted(set(ftype_codes_used), key=lambda k: self.ftype_code_order[k]): feat_row = [ftype_code] for _id in entity_list: - why_key = entity_data[_id]["MATCH_INFO"].get("WHY_KEY", "") + # why not many has multiple match_info_list, must append them altogether to get any minuses + if entity_data[_id].get("MATCH_INFO_LIST"): + why_key = "+".join(x.get("MATCH_KEY", "") for x in entity_data[_id]["MATCH_INFO_LIST"]) + else: + why_key = entity_data[_id]["MATCH_INFO"].get("WHY_KEY", "") feat_list = entity_data[_id]["FEATURES"].get(ftype_code) if feat_list: feat_row.append(self.fmt_why_features(why_key, feat_list, **kwargs)) @@ -2358,25 +2360,25 @@ class EdaSdkWrapper: return tbl.render_table(no_lines=True) def why_search(self, search_json, **kwargs): + search_flag_list = [ + "SZ_SEARCH_INCLUDE_REQUEST_DETAILS", + "SZ_INCLUDE_FEATURE_SCORES", + "SZ_INCLUDE_MATCH_KEY_DETAILS", + "SZ_ENTITY_DEFAULT_FLAGS", + "SZ_ENTITY_INCLUDE_ENTITY_NAME", + "SZ_ENTITY_INCLUDE_INTERNAL_FEATURES", + "SZ_ENTITY_INCLUDE_FEATURE_STATS", + "SZ_ENTITY_INCLUDE_RECORD_FEATURES", + ] + if kwargs["search"] == 0: - search_flag_list = [ - "SZ_SEARCH_INCLUDE_REQUEST_DETAILS", - "SZ_SEARCH_INCLUDE_ALL_CANDIDATES", - "SZ_INCLUDE_MATCH_KEY_DETAILS", - "SZ_WHY_ENTITIES_DEFAULT_FLAGS", - ] + search_flag_list.append("SZ_SEARCH_INCLUDE_ALL_CANDIDATES") try: json_data = self.call_sdk("search_by_attributes", search_flag_list, json.dumps(search_json)) except SzError as err: raise err from err matched_list = sdk_wrapper.reorder_search_results(json_data.get("RESOLVED_ENTITIES", [])) else: - search_flag_list = [ - "SZ_SEARCH_INCLUDE_REQUEST_DETAILS", - "SZ_INCLUDE_MATCH_KEY_DETAILS", - "SZ_INCLUDE_FEATURE_SCORES", - "SZ_WHY_ENTITIES_DEFAULT_FLAGS", - ] try: json_data = self.call_sdk("why_search", search_flag_list, [json.dumps(search_json), kwargs["search"]]) except SzError as err: @@ -2443,14 +2445,17 @@ class EdaSdkWrapper: return self.why_display(entities, **kwargs) def why_records(self, entity_list, **kwargs): - whyFlagList = [ - "SZ_WHY_ENTITIES_DEFAULT_FLAGS", + why_flag_list = [ + "SZ_ENTITY_DEFAULT_FLAGS", + "SZ_ENTITY_INCLUDE_INTERNAL_FEATURES", + "SZ_ENTITY_INCLUDE_FEATURE_STATS", + "SZ_INCLUDE_FEATURE_SCORES", "SZ_ENTITY_INCLUDE_RECORD_FEATURES", "SZ_INCLUDE_MATCH_KEY_DETAILS", ] try: parms = [entity_list[0], entity_list[1], entity_list[2], entity_list[3]] - json_data = self.call_sdk("why_records", whyFlagList, parms) + json_data = self.call_sdk("why_records", why_flag_list, parms) except SzError as err: raise err from err @@ -2502,7 +2507,13 @@ class EdaSdkWrapper: def why_not(self, entity_list, **kwargs): entity_list = [int(x) for x in entity_list] - why_flag_list = ["SZ_WHY_ENTITIES_DEFAULT_FLAGS", "SZ_INCLUDE_MATCH_KEY_DETAILS"] + why_flag_list = [ + "SZ_ENTITY_DEFAULT_FLAGS", + "SZ_ENTITY_INCLUDE_INTERNAL_FEATURES", + "SZ_ENTITY_INCLUDE_FEATURE_STATS", + "SZ_INCLUDE_FEATURE_SCORES", + "SZ_INCLUDE_MATCH_KEY_DETAILS", + ] try: parms = [entity_list[0], entity_list[1]] json_data = self.call_sdk("why_entities", why_flag_list, parms) @@ -2726,9 +2737,10 @@ class EdaSdkWrapper: result_id_str = colorize_entity(result_id) step_node.node_desc = f"Step {step_num}: {step_desc} on {match_key_desc} to create {result_id_str}" step_node.node_text = self.how_tree_display(steps[step_num], **kwargs) + step_node.step_created = step_num current_node.add_child(step_node) if step_data["STEP_TYPE"] == "Combine virtual entities": - current_node.children = sorted(current_node.children, key=lambda k: k.step_created) + # current_node.children = sorted(current_node.children, key=lambda k: k.step_created) parent_nodes.pop() # done with this parent for virtual_entity in steps[step_num]["ENTITY_LIST"]: interim_id = virtual_entity["VIRTUAL_ID"] @@ -2739,7 +2751,7 @@ class EdaSdkWrapper: parent_nodes.append(interim_node) break if step_data["STEP_TYPE"] == "Create virtual entity": - current_node.children = sorted(current_node.children, key=lambda k: k.node_desc) + # current_node.children = sorted(current_node.children, key=lambda k: k.node_desc) parent_nodes.pop() # end of the line break else: # its an add record to virtual ID @@ -2750,7 +2762,7 @@ class EdaSdkWrapper: root_node = root_node.children[0] else: ordered_nodes = [] - for node in sorted(root_node.children, key=lambda k: k.step_created): + for node in root_node.children: # sorted(root_node.children, key=lambda k: k.step_created): node.node_desc += f" {len(ordered_nodes)+1} of {len(final_entities)}" if len(node.children) == 0: node.node_desc += " was orphaned!" @@ -2970,18 +2982,22 @@ class EdaCmd(cmd.Cmd): tree {colorize("- see a tree view of an entity's relationships through 1 or 2 degrees.", 'dim')} export {colorize("- export the json records for an entity for debugging or correcting and reloading.", 'dim')} - {colorize('Snapshot reports:', 'highlight2')} {colorize('(requires a json file created with sz_snapshot)', 'italics')} + {colorize('Snapshot reports:', 'highlight2')} {colorize('(requires a json file generated by sz_snapshot)', 'italics')} data_source_summary {colorize('– shows how many duplicates were detected within each data source, as well as ', 'dim')} {colorize('the possible matches and relationships that were derived. For example, how many duplicate customers ', 'dim')} {colorize('there are, and are any of them related to each other.', 'dim')} cross_source_summary {colorize('– shows how many matches were made across data sources. For example, how many ', 'dim')} {colorize('employees are related to customers.', 'dim')} + entity_source_summary {colorize('– shows the number of entities by the set of data sources they can be found in. For example, ', 'dim')} + {colorize('how many entities are only in one data source, how many are only in these two data sources, etc.', 'dim')} entity_size_breakdown {colorize("– shows how many entities of what size were created. For instance, some entities ", 'dim')} {colorize("are singletons, some might have connected 2 records, some 3, etc. This report is primarily used to", 'dim')} {colorize("ensure there are no instances of over matching. For instance, it’s ok for an entity to have hundreds", 'dim')} {colorize("of records as long as there are not too many different names, addresses, identifiers, etc.", 'dim')} + principles_used {colorize('– shows what principles and match_keys are firing across all data sources. For example, ', 'dim')} + {colorize('how many name and address matches, how many address only, etc.', 'dim')} - {colorize('Audit report:', 'highlight2')} {colorize('(requires a json file created with sz_audit)', 'italics')} + {colorize('Audit report:', 'highlight2')} {colorize('(requires a json file generated by sz_audit)', 'italics')} audit_summary {colorize("- shows the precision, recall and F1 scores with the ability to browse the entities that", 'dim')} {colorize("were split or merged.", 'dim')} @@ -3179,7 +3195,7 @@ class EdaCmd(cmd.Cmd): tbl = eda_table() tbl.title = "Data source counts" tbl.columns = [ - {"name": "id", "width": 5, "align": "center"}, + {"name": "ID", "width": 5, "align": "center"}, {"name": "DataSource", "width": 30, "align": "left"}, {"name": "ActualRecordCount", "width": 20, "align": "right"}, {"name": "DistinctRecordCount", "width": 20, "align": "right"}, @@ -3232,7 +3248,7 @@ class EdaCmd(cmd.Cmd): try: self.eda_reports.audit_summary() except Exception as ex: - print_exception_info(ex) + print_exception(ex) def help_entity_size_breakdown(self): print( @@ -3256,7 +3272,7 @@ class EdaCmd(cmd.Cmd): try: self.eda_reports.entity_size_breakdown() except Exception as ex: - print_exception_info(ex) + print_exception(ex) def complete_data_source_summary(self, text, line, begidx, endidx): possibles = sorted(self.eda_reports.snapshot_data.get("DATA_SOURCES", {}).keys()) @@ -3281,7 +3297,7 @@ class EdaCmd(cmd.Cmd): try: self.eda_reports.data_source_summary(arg) except Exception as ex: - print_exception_info(ex) + print_exception(ex) def help_cross_source_summary(self): print( @@ -3306,9 +3322,9 @@ class EdaCmd(cmd.Cmd): try: self.eda_reports.cross_source_summary(arg) except Exception as ex: - print_exception_info(ex) + print_exception(ex) - def help_multi_source_summary(self): + def help_entity_source_summary(self): print( textwrap.dedent( f"""\ @@ -3320,22 +3336,22 @@ class EdaCmd(cmd.Cmd): can only show the matches, not the possible matches and relationships. {colorize('Syntax:', 'highlight2')} - multi_source_summary [dataSource] + entity_source_summary [dataSource] """ ) ) - def complete_multi_source_summary(self, text, line, begidx, endidx): + def complete_entity_source_summary(self, text, line, begidx, endidx): possibles = sorted(self.eda_reports.snapshot_data.get("DATA_SOURCES", {}).keys()) if text: return [i for i in possibles if i.startswith(text.upper())] return possibles - def do_multi_source_summary(self, arg): + def do_entity_source_summary(self, arg): try: - self.eda_reports.multi_source_summary(arg) + self.eda_reports.entity_source_summary(arg) except Exception as ex: - print_exception_info(ex) + print_exception(ex) def help_principles_used(self): print( @@ -3355,7 +3371,7 @@ class EdaCmd(cmd.Cmd): try: self.eda_reports.principles_used_report() except Exception as ex: - print_exception_info(ex) + print_exception(ex) # adhoc commands @@ -3399,6 +3415,153 @@ class EdaCmd(cmd.Cmd): arg_str = " ".join(remaining_tokens) return arg_str, kwarg_dict + def help_feature_search(self): + print( + textwrap.dedent( + f"""\ + + finds entities by feature description or id + + {colorize('Syntax:', 'highlight2')} + featureSearch id = 123 + featureSearch address = 123 first str + featureSearch name like anderson + + {colorize('Warning:', 'bad')} + featureSearch is intended to be used during smallish POCs to help find entities by feature ID or description when they are not returned by search. + featureSearch can take a long time and return a lot of entities! + + """ + ) + ) + + def do_feature_search(self, arg): + if not arg: + self.help_feature_search() + return + if not sz_dbo: + print_message("Direct database access required", "error") + return + + arg = arg.replace("=", " = ") + ftype = operator = value = "" + for token in arg.split(): + if not ftype: + ftype = token.upper() + elif not operator: + operator = token.upper() + else: + value += token + " " + value = value.strip() + + if not ftype or not operator or not value: + print_message("Invalid syntax", "error") + return + + if operator not in ("=", "LIKE"): + print_message('Operator can only be "=" or "like"', "error") + return + + feat_select = "select LIB_FEAT_ID, FTYPE_ID, FEAT_DESC from LIB_FEAT " + stat_select = "select NUM_RES_ENT, NUM_RES_ENT_OOM, CANDIDATE_CAP_REACHED, SCORING_CAP_REACHED FROM RES_FEAT_STAT WHERE LIB_FEAT_ID = " + + if ftype == "ID": + feat_select += f"where LIB_FEAT_ID = {value}" + elif ftype in sdk_wrapper.ftype_code_lookup: + ftype_id = sdk_wrapper.ftype_code_lookup[ftype]["FTYPE_ID"] + if operator == "LIKE": + if sz_dbo_uri.startswith("postgres"): + operator = "ILIKE" # makes case insensitive + elif sz_dbo_uri.startswith("sqlite"): + sz_dbo.sqlExec("PRAGMA case_sensitive_like=OFF") + if "%" not in value: + value = "%" + value + "%" + feat_select += f"where FTYPE_ID = {ftype_id} and FEAT_DESC {operator} '{value}'" + else: + print_message(f"{ftype} is not a feature", "error") + return + + # print(feat_select) + + tbl = eda_table() + tbl.columns = [ + {"name": "\nID", "width": 5, "align": "center"}, + {"name": "\nFeature", "width": 25, "align": "center"}, + {"name": "\nDescription", "width": 75, "align": "left"}, + {"name": "Estimated\nCount", "width": 10, "align": "center"}, + {"name": "Generic\nCandidate", "width": 20, "align": "center"}, + {"name": "Generic\nScoring", "width": 20, "align": "center"}, + ] + tbl.rows = [] + feat_cache = {} + feat_count = 0 + for row in sz_dbo.fetchAllRows(sz_dbo.sqlExec(feat_select)): + feat_count += 1 + feat_id = row[0] + ftype = sdk_wrapper.ftype_lookup[row[1]]["FTYPE_CODE"] + feat_desc = row[2] + feat_cache[feat_id] = feat_desc + + stat_row = sz_dbo.fetchRow(sz_dbo.sqlExec(stat_select + f" {feat_id}")) + if not stat_row: + stat_row = [0, 0, "U", "U"] + + tbl.rows.append( + [ + colorize(feat_id, "row_title"), + colorize_attr(ftype), + feat_desc, + f"{stat_row[0] * stat_row[1]:,}", + "Yes" if stat_row[2] == "Y" else "No", + "Yes" if stat_row[3] == "Y" else "No", + ] + ) + + if feat_count == 0: + print_message("No features Found!", "error") + return + + tbl.title = f"{feat_count:,} Features found" + report = tbl.render_table(super_header=True) + + tbl1 = eda_table() + tbl1.columns = [ + {"name": "Index", "width": 5, "align": "center"}, + {"name": "Entity ID", "width": 15, "align": "center"}, + {"name": "Entity Name", "width": 75, "align": "left"}, + {"name": "Data Sources", "width": 50, "align": "left"}, + {"name": "Feature", "width": 25, "align": "left"}, + {"name": "Description", "width": 75, "align": "left"}, + ] + get_flag_list = ["SZ_ENTITY_INCLUDE_ENTITY_NAME", "SZ_ENTITY_INCLUDE_RECORD_DATA"] + entity_cnt = 0 + entity_cache = {} + tbl1.rows = [] + for lib_feat_id, feat_desc in feat_cache.items(): + sql = f"select distinct RES_ENT_ID from RES_FEAT_EKEY where LIB_FEAT_ID = {lib_feat_id} order by RES_ENT_ID" + for row in sz_dbo.fetchAllRows(sz_dbo.sqlExec(sql)): + entity_id = row[0] + if entity_id in entity_cache: + continue + entity_cache[entity_id] = True + entity_cnt += 1 + if entity_cnt <= 1000: + json_data = sdk_wrapper.call_sdk("get_entity_by_entity_id", get_flag_list, entity_id) + entity_name = json_data["RESOLVED_ENTITY"]["ENTITY_NAME"] + tbl1.rows.append( + [ + colorize(entity_cnt, "row_title"), + colorize_entity(entity_id), + entity_name, + sdk_wrapper.fmt_record_list(json_data["RESOLVED_ENTITY"]["RECORDS"]), + colorize_attr(ftype), + feat_desc, + ] + ) + tbl1.title = f"{entity_cnt:,} Entities found" + report += "\n" + tbl1.render_table() + view_report(report) + def help_search(self): print( textwrap.dedent( @@ -3612,7 +3775,7 @@ class EdaCmd(cmd.Cmd): print_message(err, "error") return except Exception as ex: - print_exception_info(ex) + print_exception(ex) return view_report(report) if len(arg_tokens) == 1: # supports previous/next @@ -3718,17 +3881,17 @@ class EdaCmd(cmd.Cmd): return root_entity_id = None - build_out_degree = 1 - max_children = 25 + kwargs["build_out_degree"] = 1 + kwargs["max_children"] = 25 arg_list = arg.split() if arg_list[-1].upper() == "ALL": - max_children = 10000 + kwargs["max_children"] = 10000 arg_list.pop(-1) if len(arg_list) in (1, 3): if arg_list[0].isdigit(): root_entity_id = int(arg_list[0]) if len(arg_list) == 3 and arg_list[1].upper() == "DEGREE" and arg_list[2].isdigit(): - build_out_degree = int(arg_list[2]) + kwargs["build_out_degree"] = int(arg_list[2]) if not root_entity_id: print_message("Invalid parameter: expected a numeric entity ID", "warning") return @@ -3747,7 +3910,8 @@ class EdaCmd(cmd.Cmd): why {colorize('actually runs a columnar how instead!', 'dim')} why {colorize('shows why two or more different entities did not resolve', 'dim')} why {colorize('shows if the two data source records could resolve or relate', 'dim')} - + why search [optional entity_id] {colorize('shows the features and keys generated by a search against any candidate entities', 'dim')} + {colorize('Color legend:', 'highlight2')} {colorize('green', 'good')} indicates the values matched and contributed to the overall score {colorize('red', 'bad')} indicates the values did not match and hurt the overall score @@ -4062,12 +4226,13 @@ if __name__ == "__main__": engine_config = get_engine_config(args.config_file_name) try: sdk_wrapper = EdaSdkWrapper(engine_config, debug_trace=args.debug_trace, webapp_url=args.webapp_url) - except SzError: - print_exception_info() + except SzError as ex: + print_message(ex, "error") sys.exit(1) try: - sz_dbo = SzDatabase(json.loads(engine_config)["SQL"]["CONNECTION"]) + sz_dbo_uri = json.loads(engine_config)["SQL"]["CONNECTION"] + sz_dbo = SzDatabase(sz_dbo_uri) except Exception as err: print_message(err, "warning") sz_dbo = None From 03853d19e92993b10a015d0c459f71c01b5fe5fe Mon Sep 17 00:00:00 2001 From: Ant Date: Fri, 20 Jun 2025 13:10:48 +0200 Subject: [PATCH 4/8] 188 - Save point --- pyproject.toml | 3 ++- sz_tools/sz_command | 29 ++++++++++++++--------------- sz_tools/sz_configtool | 4 ++-- sz_tools/sz_file_loader | 2 +- 4 files changed, 19 insertions(+), 19 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index ce839b1..23fe881 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,7 +42,8 @@ disable = [ "line-too-long", "too-many-branches", "too-many-instance-attributes", - "too-many-locals" + "too-many-locals", + "too-many-statements" ] good-names = [ "template-python" diff --git a/sz_tools/sz_command b/sz_tools/sz_command index 4a24ef8..008acc9 100755 --- a/sz_tools/sz_command +++ b/sz_tools/sz_command @@ -45,15 +45,14 @@ _WrappedFunc = TypeVar("_WrappedFunc", bound=Callable[..., Any]) MODULE_NAME = pathlib.Path(__file__).stem -# Per command settings -PER_CMD_SETTINGS = ["json", "jsonl", "color", "colour", "nocolor", "nocolour", "debug", "timer", "page"] +PER_CMD_SETTINGS = ["json", "jsonl", "color", "colour", "nocolor", "nocolour", "debug", "timer", "scroll"] DEFAULT_CONFIG = { "format_json": True, "color_output": True, "timer": False, "theme": "TERMINAL", - "page_output": False, + "scroll_output": False, "debug_sdk_call": False, "debug_sz_engine": False, "history_file": True, @@ -71,7 +70,7 @@ SETTINGS_TO_CONFIG_MAP = { "debug": ["debug_sdk_call", True], "debug_sdk": ["debug_sdk_call", True], "timer": ["timer", True], - "page": ["page_output", True], + "scroll": ["scroll_output", True], } # Don't overwrite last command if we still need to use it @@ -99,7 +98,7 @@ CONFIG_SETTINGS: Dict[str, Dict[str, Any]] = { "values": [c.lower() for c in Colors.AVAILABLE_THEMES], "description": "Set color scheme to use", }, - "page_output": { + "scroll_output": { "values": ["off", "on"], "description": "Use a pager for content larger than the terminal", }, @@ -126,7 +125,7 @@ CONFIG_SETTINGS: Dict[str, Dict[str, Any]] = { def do_methods_decorator(do_method: _WrappedFunc) -> _WrappedFunc: @functools.wraps(do_method) - def wrapper(self, *args, **kwargs) -> Any: + def wrapper(self, *args: Any, **kwargs: Any) -> Any: # Remove do_ from wrapped method sdk_method_name = do_method.__name__[3:] @@ -582,7 +581,7 @@ class SzCmdShell(cmd.Cmd): formatted_response: str = print_response( response, self.per_cmd_config["format_json"], # type: ignore[arg-type] - self.per_cmd_config["page_output"], # type: ignore[arg-type] + self.per_cmd_config["scroll_output"], # type: ignore[arg-type] self.per_cmd_config["color_output"], # type: ignore[arg-type] color=color, ) @@ -781,7 +780,7 @@ class SzCmdShell(cmd.Cmd): - nocolor / nocolour - debug - timer - - page + - scroll {colorize_str('- Examples:', 'dim')} - get_entity_by_entity_id 1001 jsonl @@ -845,21 +844,21 @@ class SzCmdShell(cmd.Cmd): Notes: - Retrieve the active configuration identifier with get_active_config_id - - Retrieve a list of configurations and identifiers with get_configs""" + - Retrieve a list of configurations and identifiers with get_config_registry""" sz_config = self.sz_configmgr.create_config_from_config_id(kwargs["parsed_args"].config_id) response = sz_config.export() self.last_response = self.output_response(response) @do_methods_decorator - def do_get_configs(self) -> None: + def do_get_config_registry(self) -> None: """ - Get a list of current configurations + Get details of current registered configurations Syntax: - get_configs""" + get_config_registry""" - response = self.sz_configmgr.get_configs() + response = self.sz_configmgr.get_config_registry() self.last_response = self.output_response(response) @do_methods_decorator @@ -889,7 +888,7 @@ class SzCmdShell(cmd.Cmd): new_default_config_id = Configuration identifier Notes: - - Retrieve a list of configurations and identifiers with get_configs""" + - Retrieve a list of configurations and identifiers with get_config_registry""" self.sz_configmgr.replace_default_config_id( kwargs["parsed_args"].current_default_config_id, @@ -2297,7 +2296,7 @@ class SzCmdShell(cmd.Cmd): return kwargs["cmd_settings"] @cmd_settings_decorator - def complete_get_configs(self, text: str, line: str, begidx: int, endidx: int, **kwargs) -> List[str]: + def complete_get_config_registry(self, text: str, line: str, begidx: int, endidx: int, **kwargs) -> List[str]: return kwargs["cmd_settings"] @cmd_settings_decorator diff --git a/sz_tools/sz_configtool b/sz_tools/sz_configtool index 59181c9..36ad78b 100755 --- a/sz_tools/sz_configtool +++ b/sz_tools/sz_configtool @@ -754,7 +754,7 @@ class SzCfgShell(cmd.Cmd): except SzError as err: colorize_msg(err, "error") - def do_getConfigList(self, arg): + def do_getConfigRegistry(self, arg): """ Returns the list of all known configurations @@ -763,7 +763,7 @@ class SzCfgShell(cmd.Cmd): """ arg = self.check_arg_for_output_format("record", arg) try: - config_list = self.sz_configmgr.get_configs() + config_list = self.sz_configmgr.get_config_registry() self.print_json_record(json.loads(config_list)["CONFIGS"]) except SzError as err: colorize_msg(err, "error") diff --git a/sz_tools/sz_file_loader b/sz_tools/sz_file_loader index dcf9539..6e9a73c 100755 --- a/sz_tools/sz_file_loader +++ b/sz_tools/sz_file_loader @@ -608,7 +608,7 @@ def startup_info( try: lic_info = _json_loads(product.get_license()) ver_info = _json_loads(product.get_version()) - config_list = _json_loads(configmgr.get_configs()) + config_list = _json_loads(configmgr.get_config_registry()) active_cfg_id = engine.get_active_config_id() ds_info = _json_loads(diag.get_datastore_info()) except SzError as err: From a30b3cccd074548a95c7d55e39cdd16db0fd3b2f Mon Sep 17 00:00:00 2001 From: Ant Date: Fri, 20 Jun 2025 13:24:42 +0200 Subject: [PATCH 5/8] 188 - Save point --- CHANGELOG.md | 15 + requirements.txt | 2 +- setup.cfg | 6 +- sz_tools/sz_update_project_prior1 | 483 ---------------------------- sz_tools/sz_update_project_prior2 | 492 ---------------------------- sz_tools/sz_update_project_prior3 | 478 --------------------------- sz_tools/sz_update_project_prior4 | 456 -------------------------- sz_tools/sz_update_project_prior5 | 515 ------------------------------ sz_tools/sz_update_project_prior6 | 465 --------------------------- sz_tools/sz_update_project_prior7 | 480 ---------------------------- sz_tools/sz_update_project_prior8 | 463 --------------------------- sz_tools/sz_update_project_prior9 | 496 ---------------------------- 12 files changed, 19 insertions(+), 4332 deletions(-) delete mode 100755 sz_tools/sz_update_project_prior1 delete mode 100755 sz_tools/sz_update_project_prior2 delete mode 100755 sz_tools/sz_update_project_prior3 delete mode 100755 sz_tools/sz_update_project_prior4 delete mode 100755 sz_tools/sz_update_project_prior5 delete mode 100755 sz_tools/sz_update_project_prior6 delete mode 100755 sz_tools/sz_update_project_prior7 delete mode 100755 sz_tools/sz_update_project_prior8 delete mode 100755 sz_tools/sz_update_project_prior9 diff --git a/CHANGELOG.md b/CHANGELOG.md index 42c0180..7ef590a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,21 @@ and this project adheres to [Semantic Versioning]. ## [Unreleased] +## [0.0.24] - 2025-06-20 + +### Added in 0.0.24 + +- sz_update_project can now upgrade V3 -> V4 and V4 -> V4 projects + +### Changed in 0.0.24 + +- Update tools from get_configs() to get_config_registry() (SDK change) +- sz_command now uses the setting and command `scroll` instead of `page` + +### Fixed in 0.0.24 + +- sz_create_project shouldn't copy sz_update_project to a new project + ## [0.0.23] - 2025-06-12 ### Fixed in 0.0.23 diff --git a/requirements.txt b/requirements.txt index 6146007..a757902 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,2 @@ -senzing==0.2.2 +senzing==0.2.16 senzing-core==0.3.15 diff --git a/setup.cfg b/setup.cfg index c5ecb03..c3e44b5 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = python-tools -version = 0.0.23 +version = 0.0.24 author = senzing author_email = support@senzing.com description = Python Tools @@ -21,8 +21,8 @@ package_dir = packages = find: python_requires = >=3.9 install_requires = - senzing >= 0.2.12 - senzing-core >= 0.3.11 + senzing >= 0.2.16 + senzing-core >= 0.3.15 [options.packages.find] where = src diff --git a/sz_tools/sz_update_project_prior1 b/sz_tools/sz_update_project_prior1 deleted file mode 100755 index d9e4a22..0000000 --- a/sz_tools/sz_update_project_prior1 +++ /dev/null @@ -1,483 +0,0 @@ -#! /usr/bin/env python3 -# TODO - sz_project_update -# TODO - -# TODO - -# TODO - -# TODO - -# TODO - - -# TODO - -"""Upgrade a V3 or V4 Senzing SDK project""" - -import argparse -import json -import shutil -import sys -from contextlib import suppress -from pathlib import Path -from time import sleep -from typing import Any, NamedTuple - -INPUT_CONFS = ("y", "Y", "yes", "YES") -MODULE_NAME = Path(__file__).stem -# TODO - -# V3_PROJ_BUILD = "g2BuildVersion.json" -# V4_PROJ_BUILD = "szBuildVersion.json" -V3_BUILD = "g2BuildVersion.json" -V4_BUILD = "szBuildVersion.json" -V3_SYS_PATH = Path("/opt/senzing/g2") -V4_SYS_PATH = Path("/opt/senzing/er") -V3_SYS_BUILD = V3_SYS_PATH / V3_BUILD -V4_SYS_BUILD = V4_SYS_PATH / V4_BUILD -V3_BACKUP_PATH = "v3_upgrade_backups" -UPGRADE_URL = "https://www.senzing.com/docs/4_beta/index.html" - -V3_BACKUP_PROJ = { - "bin": {"files": ["g2dbencrypt", "g2saltadm", "g2ssadm"], "excludes": []}, - "etc": {"files": ["senzing_governor.py"], "excludes": []}, - "lib": {"files": ["libSpaceTimeBoxStandardizer.so", "libG2Hasher.so", "libG2SSAdm.so"], "excludes": []}, - "python": {"files": [], "excludes": []}, - "resources/templates": {"files": ["setupEnv"], "excludes": []}, - "setupEnv": {"files": [], "excludes": []}, - "g2BuildVersion.json": {"files": [], "excludes": []}, -} - -V3_REMOVE_FROM_PROJ = { - "bin": {"files": ["*"], "excludes": ["bin"]}, - "data": {"files": [], "excludes": []}, - "etc": {"files": ["senzing_governor.py"], "excludes": []}, - # "g2BuildVersion.json": {"files": [], "excludes": []}, - "lib": { - "files": [ - "g2.jar", - "libG2.so", - "libG2Hasher.so", - "libG2SSAdm.so", - "libg2CompJavaScoreSet.so", - "libg2DistinctFeatJava.so", - "libg2EFeatJava.so", - "libg2JVMPlugin.so", - "libg2StdJava.so", - "libmariadbplugin.so", - "libSpaceTimeBoxStandardizer.so", - ], - "excludes": ["lib"], - }, - "python": {"files": ["*"], "excludes": []}, - "resources/config": {"files": ["g2core-configuration-upgrade-*.gtc"], "excludes": []}, - "resources/schema": {"files": ["g2core-schema-*-create.sql", "g2core-schema-*-upgrade-*.sql"], "excludes": []}, - "resources/templates": { - "files": [ - "G2C.db*", - "G2Module.ini", - "custom*.txt", - "defaultGNRCP.config", - "g2config.json", - "senzing_governor.py", - "setupEnv", - "stb.config", - ], - "excludes": [], - }, - "sdk": {"files": ["*"], "excludes": ["sdk"]}, - "setupEnv": {"files": [], "excludes": []}, -} - -V3_COPY_TO_PROJ = { - "LICENSE": {"files": [], "excludes": []}, - "NOTICES": {"files": [], "excludes": []}, - "README.1ST": {"files": [], "excludes": []}, - "bin": {"files": ["*"], "excludes": ["sz_create_project", "sz_update_project"]}, - "/opt/senzing/data": {"files": ["*"], "excludes": []}, - "lib": {"files": ["*"], "excludes": []}, - "resources": {"files": ["*"], "excludes": []}, - "sdk": {"files": ["*"], "excludes": []}, - "szBuildVersion.json": {"files": [], "excludes": []}, -} - - -V3_RENAME_IN_PROJ = { - "etc": {"from": "G2Module.ini", "to": "sz_engine_config.ini"}, -} - - -V3_RESET_PERMISSIONS = { - ".": { - "dir_pint": 0, - "file_pint": 0o660, - "files": ["LICENSE", "NOTICES", "README.1ST", "szBuildVersion.json"], - "excludes": ["setupEnv"], - "recursive": False, - }, - "setupEnv": {"dir_pint": 0, "file_pint": 0o770, "files": [], "excludes": [], "recursive": False}, - "bin": { - "dir_pint": 0o770, - "file_pint": 0o770, - "files": ["*"], - "excludes": ["__pycache__", "_sz_database.py", "_tool_helpers.py"], - "recursive": False, - }, - "data": {"dir_pint": 0o770, "file_pint": 0o660, "files": ["*"], "excludes": [], "recursive": True}, - "lib": {"dir_pint": 0o770, "file_pint": 0o660, "files": ["*"], "excludes": [], "recursive": False}, - "resources": {"dir_pint": 0o770, "file_pint": 0o660, "files": ["*"], "excludes": ["setupEnv"], "recursive": True}, - "resources/templates/setupEnv": { - "dir_pint": 0, - "file_pint": 0o770, - "files": [], - "excludes": [], - "recursive": False, - }, - "sdk": {"dir_pint": 0o770, "file_pint": 0o660, "files": ["*"], "excludes": [], "recursive": True}, - "v3_upgrade_backups": {"dir_pint": 0o770, "file_pint": 0, "files": [], "excludes": [], "recursive": False}, -} - - -# TODO - -# VersionTokens = namedtuple("VersionTokens", ["major", "minor", "patch"]) -class VersionTokens(NamedTuple): - major: int - minor: int - patch: int - - -# pylint: disable=W0106 - - -def parse_cli_args() -> argparse.Namespace: - """Parse the CLI arguments""" - arg_parser = argparse.ArgumentParser( - description="Update an existing Senzing project to the system installed version of Senzing." - ) - arg_parser.add_argument( - "project_path", - metavar="path", - help="path of the project to update", - ) - arg_parser.add_argument( - "-f", - "--force", - dest="force_mode", - default=False, - action="store_true", - help="upgrade without prompts. WARNING: Use with caution, ensure you have a backup of the project", - ) - return arg_parser.parse_args() - - -def dir_listing(path: Path) -> list[Path]: - """Get a listing of the specified path to check for files or directories""" - try: - listing = list(path.iterdir()) - except OSError as err: - if type(err).__name__ == "NotADirectoryError": - print(f"\nERROR: {path} is not a directory, expecting one") - print(f"\nERROR: {err}") - sys.exit(1) - - return listing - - -# TODO - -# pre_check(proj_path, proj_path / PROJ_BUILD, SZ_SYS_PATH / SYS_BUILD) -# def pre_check(project_path: Path, proj_build_file: Path, sys_build_file: Path, listing: list[Path]) -> tuple[str, str]: -def pre_check(project_path: Path) -> tuple[str, str]: - """Check not trying to overwrite the V4 Senzing system install, that path is a project, and versions are correct""" - if not V4_SYS_PATH.exists(): - print(f"\nERROR: Couldn't locate Senzing SDK system install at {V4_SYS_PATH}") - sys.exit(1) - - # TODO - - # if project_path.samefile(SZ_SYS_PATH): - if str(project_path).startswith(str(V4_SYS_PATH)): - print(f"\nERROR: {project_path} is the Senzing system installation, not a Senzing project") - sys.exit(1) - - # TODO - - # if proj_build_file not in listing: - # if proj_build_file not in dir_listing(project_path): - v3_project_build_file = project_path / V3_BUILD - v4_project_build_file = project_path / V4_BUILD - proj_listing = dir_listing(project_path) - print(f"\n{v3_project_build_file = }", flush=True) - print(f"{v4_project_build_file = }", flush=True) - print(f"{proj_listing = }", flush=True) - # if v3_project_build_file or v4_project_build_file not in dir_listing(project_path): - if v3_project_build_file not in proj_listing and v4_project_build_file not in proj_listing: - print(f"\nERROR: {project_path} isn't a Senzing project, expected it to contain either:") - print(f"\tExisting V3 project - {v3_project_build_file}") - print(f"\tExisting V4 project - {v4_project_build_file}") - sys.exit(1) - - # TODO - Does this work on V3 and V4? - try: - proj_build_file = v3_project_build_file if v3_project_build_file in proj_listing else v4_project_build_file - # TODO - What if don't get an int or error? - # proj_major_version, _, _ = get_build_versions(proj_build_file) - proj_version, proj_version_tokens = get_build_versions(proj_build_file) - # sys_build_file = V3_SYS_BUILD if proj_version_tokens.major == 3 else V4_SYS_BUILD - # sys_build_file = V3_SYS_BUILD if V3_SYS_BUILD.exists() else V4_SYS_BUILD - # sys_major_version, _, _ = get_build_versions(sys_build_file) - sys_version, sys_version_tokens = get_build_versions(V4_SYS_BUILD) - except OSError as err: - print(f"\nERROR: Trying to read {proj_build_file} or {V4_SYS_BUILD} to collect version information: {err}") - sys.exit(1) - - if proj_version_tokens.major not in (3, 4) or sys_version_tokens.major != 4: - print(f"\nERROR: {MODULE_NAME} updates V3 to V4, or V4.n.n to a newer release") - print(f"\tProject version: {proj_version_tokens.major}, system install version: {sys_version_tokens.major}") - sys.exit(1) - - # TODO - - if proj_version_tokens.major == 4 and (sum(sys_version_tokens) == sum(proj_version_tokens)): - print(f"No update required, project and system install are the same version - {proj_version}") - sys.exit(0) - - return (proj_version, sys_version) - - -# TODO - Should this check it looks like a version? If not above needs to check it. -# TODO - Have this return major, etc? -# def get_build_version(path: Path) -> str: -# TODO - Move to helpers? -# def get_build_versions(path: Path) -> list[int]: -# def get_build_versions(path: Path) -> list[int]: -def get_build_versions(path: Path) -> tuple[str, VersionTokens]: - """Return the version string and major, minor, and patch build versions from a build file""" - err_msg = f"ERROR: Couldn't get the version information from {path}" - - try: - with open(path, "r", encoding="utf-8") as f: - # data = json.load(f) - # version = data["VERSION"] - # version_str: str = data["VERSION"] - version_str: str = json.load(f)["VERSION"] - except (OSError, json.JSONDecodeError) as err: - print(f"\n{err_msg}: {err}") - sys.exit(1) - except KeyError as err: - print(f"\n{err_msg}, missing key {err}") - sys.exit(1) - - # TODO - test this - # str_tokens = version_str.split(".") - if not (str_tokens := version_str.split(".")) or (str_tokens and str_tokens[0] == ""): - print(f"\n{err_msg}, VERSION was blank") - sys.exit(1) - # TODO - - print(f"\n{str_tokens = }", flush=True) - - if len(str_tokens) != 3: - print(f"\nERROR: Version information should consist of major, minor, and patch, it is {version_str}") - sys.exit(1) - - try: - # tokens_list = [int(v) for v in vers_tokens] - vers_tokens = VersionTokens._make([int(v) for v in str_tokens]) - # t = [11, 22] - # Point._make(t) - except ValueError: - print(f"\nERROR: Version information should consist of integers, it is {version_str}") - sys.exit(1) - - # TODO - - # return version - return (version_str, vers_tokens) - - -def remove_dir(dir_: Path, excludes: list[Path]): - """Recursively remove a directory""" - try: - for path in dir_.iterdir(): - if path in excludes: - continue - path.unlink(missing_ok=True) if (path.is_file() or path.is_symlink()) else remove_dir(path, excludes) - - if not excludes: - with suppress(FileNotFoundError): - dir_.rmdir() - except OSError as err: - raise err - - -def remove_files_dirs(to_remove: dict[str, Any], target_dir: Path): - """Remove files/directories that are no longer required""" - for r_path, r_dict in to_remove.items(): - target: Path = target_dir / r_path - files = r_dict["files"] - excludes = [target / e for e in r_dict["excludes"]] - - try: - if target.is_file() or target.is_symlink(): - target.unlink(missing_ok=True) - - if target.is_dir() and not files or (files and files[0] == "*"): - remove_dir(target, excludes) - - if target.is_dir() and files and files[0] != "*": - target_files = [] - for f in files: - target_files.append(target / f) if "*" not in f else target_files.extend(list(target.glob(f))) - - for target_file in target_files: - target_file.unlink(missing_ok=True) - except OSError as err: - raise OSError(f"ERROR: Couldn't delete a file or directory: {err}") from err - - -def copy_files_dirs(to_copy: dict[str, Any], source_dir: Path, target_dir: Path): - """Copy files/directories within and to tne project""" - for c_path, c_dict in to_copy.items(): - excludes = c_dict["excludes"] - files = c_dict["files"] - source: Path = source_dir / c_path - target: Path = target_dir / c_path - - if c_path.startswith("/"): - source = Path(c_path) - try: - target = target_dir / source.relative_to(SZ_SYS_PATH) - except ValueError: - target = target_dir / target.name - - try: - if source.is_file(): - shutil.copy( - source, - target, - ) - - if source.is_dir() and not files or (files and files[0] == "*"): - shutil.copytree(source, target, ignore=shutil.ignore_patterns(*excludes), dirs_exist_ok=True) - - if source.is_dir() and files and files[0] != "*": - target.mkdir(exist_ok=True, parents=True) - for source_file in [source / f for f in files]: - shutil.copy(source_file, target / source_file.name) - except OSError as err: - raise OSError(f"ERROR: Couldn't copy a file or directory: {err}") from err - - -def rename_files(to_rename: dict[str, dict[str, str]], target_dir: Path): - """Rename existing project files that had a name change""" - - try: - for r_path, r_dict in to_rename.items(): - current = target_dir / r_path / r_dict["from"] - new = target_dir / r_path / r_dict["to"] - with suppress(FileNotFoundError): - current.rename(new) - except OSError as err: - raise OSError(f"ERROR: Couldn't rename a file or directory: {err}") from err - - -def setup_env(proj_path: Path): - """Create a new setupEnv and replace place holders with paths for the project""" - try: - shutil.copy(proj_path / "resources/templates/setupEnv", proj_path) - setup_path = proj_path / "setupEnv" - - with open(setup_path, "r", encoding="utf-8") as in_: - data = in_.read() - - data = data.replace("${SENZING_DIR}", str(proj_path)).replace("${SENZING_CONFIG_PATH}", str(proj_path / "etc")) - - with open(setup_path, "w", encoding="utf-8") as out: - out.write(data) - except OSError as err: - raise OSError(f"ERROR: Couldn't create a new setupEnv file: {err}") from err - - -def set_permissions(proj_path: Path, permissions: dict[str, dict[str, Any]]): - """ - Reset permissions for files and dirs copied to the projector, or dirs removed and replaced - completely e.g., data/ - """ - try: - for p_path, p_dict in permissions.items(): - dir_pint = p_dict["dir_pint"] - file_pint = p_dict["file_pint"] - files = p_dict["files"] - recursive = p_dict["recursive"] - target = proj_path if p_path.startswith(".") else proj_path / p_path - excludes = [target / e for e in p_dict["excludes"]] - - if target.is_file(): - target.chmod(file_pint) - - if target.is_dir() and dir_pint != 0: - target.chmod(dir_pint) - d_chmods = ( - [d for d in target.glob("*") if d.is_dir() and not d.is_symlink() and d not in excludes] - if not recursive - else [d for d in target.rglob("*") if d.is_dir() and not d.is_symlink() and d not in excludes] - ) - for dir_ in d_chmods: - dir_.chmod(dir_pint) - - if target.is_dir() and (files and files[0] == "*"): - f_chmods = ( - [f for f in target.glob("*") if f.is_file() and not f.is_symlink() and f not in excludes] - if not recursive - else [f for f in target.rglob("*") if f.is_file() and not f.is_symlink() and f not in excludes] - ) - for file in f_chmods: - Path(target / file).chmod(file_pint) - - if target.is_dir() and files and files[0] != "*": - for file in files: - Path(target / file).chmod(file_pint) - except OSError as err: - raise OSError(f"ERROR: Couldn't set a permission: {err}") from err - - -def main() -> None: - """main""" - cli_args = parse_cli_args() - proj_path = Path(cli_args.project_path).resolve() - # TODO - - # proj_ver, sys_ver = pre_check(proj_path, proj_path / PROJ_BUILD, SZ_SYS_PATH / SYS_BUILD, dir_listing(proj_path)) - # proj_ver, sys_ver = pre_check(proj_path, proj_path / PROJ_BUILD, SZ_SYS_PATH / SYS_BUILD) - proj_ver, sys_ver = pre_check(proj_path) - # TODO - - print(f"\n{proj_ver = }", flush=True) - print(f"{sys_ver = }", flush=True) - sys.exit() - - if not cli_args.force_mode: - print(f"\nWARNING: If you don't have a backup of the project ({proj_path}), create one before completing this!") - sleep(3) - if input(f"\nContinue updating the project from version {proj_ver} to {sys_ver}? (y/n) ") not in INPUT_CONFS: - sys.exit(0) - print("\nUpdating...") - else: - print(f"\nUpdating project from version {proj_ver} to {sys_ver}...") - - try: - # Backup some of the V3 project files - if proj_ver[:1] == "3": - v3_backup_path = proj_path / V3_BACKUP_PATH - v3_backup_path.mkdir(exist_ok=True) - copy_files_dirs(V3_BACKUP_PROJ, proj_path, v3_backup_path) - - rename_files(V3_RENAME_IN_PROJ, proj_path) - remove_files_dirs(V3_REMOVE_FROM_PROJ, proj_path) - copy_files_dirs(V3_COPY_TO_PROJ, SZ_SYS_PATH, proj_path) - setup_env(proj_path) - set_permissions(proj_path, V3_RESET_PERMISSIONS) - except OSError as err: - shutil.copytree(v3_backup_path, proj_path, dirs_exist_ok=True) - print(f"\n{err}") - print( - "\nIf the error is file or directory permission related, run again with a user with appropriate privileges" - ) - print( - "\nIf the error is missing file or directory, check senzingsdk-setup, senzingsdk-tools, and senzingsdk-poc are installed" - ) - else: - # Remove if no errors so re-running can find the file - proj_path.joinpath(PROJ_BUILD).unlink(missing_ok=True) - print(f"\nProject successfully updated. Refer to {UPGRADE_URL} for additional upgrade instructions") - - -if __name__ == "__main__": - main() diff --git a/sz_tools/sz_update_project_prior2 b/sz_tools/sz_update_project_prior2 deleted file mode 100755 index 3981ce7..0000000 --- a/sz_tools/sz_update_project_prior2 +++ /dev/null @@ -1,492 +0,0 @@ -#! /usr/bin/env python3 -# TODO - sz_project_update -# TODO - -# TODO - -# TODO - -# TODO - -# TODO - - -# TODO - -"""Upgrade a V3 or V4 Senzing SDK project""" - -import argparse -import json -import shutil -import sys -from contextlib import suppress -from dataclasses import dataclass -from pathlib import Path -from time import sleep -from typing import Any - -INPUT_CONFS = ("y", "Y", "yes", "YES") -MODULE_NAME = Path(__file__).stem -# TODO - -# V3_PROJ_BUILD = "g2BuildVersion.json" -# V4_PROJ_BUILD = "szBuildVersion.json" -V3_BUILD = "g2BuildVersion.json" -V4_BUILD = "szBuildVersion.json" -V3_SYS_PATH = Path("/opt/senzing/g2") -V4_SYS_PATH = Path("/opt/senzing/er") -V3_SYS_BUILD = V3_SYS_PATH / V3_BUILD -V4_SYS_BUILD = V4_SYS_PATH / V4_BUILD -V3_BACKUP_PATH = "v3_upgrade_backups" -UPGRADE_URL = "https://www.senzing.com/docs/4_beta/index.html" - -V3_BACKUP_PROJ = { - "bin": {"files": ["g2dbencrypt", "g2saltadm", "g2ssadm"], "excludes": []}, - "etc": {"files": ["senzing_governor.py"], "excludes": []}, - "lib": {"files": ["libSpaceTimeBoxStandardizer.so", "libG2Hasher.so", "libG2SSAdm.so"], "excludes": []}, - "python": {"files": [], "excludes": []}, - "resources/templates": {"files": ["setupEnv"], "excludes": []}, - "setupEnv": {"files": [], "excludes": []}, - "g2BuildVersion.json": {"files": [], "excludes": []}, -} - -V3_REMOVE_FROM_PROJ = { - "bin": {"files": ["*"], "excludes": ["bin"]}, - "data": {"files": [], "excludes": []}, - "etc": {"files": ["senzing_governor.py"], "excludes": []}, - # "g2BuildVersion.json": {"files": [], "excludes": []}, - "lib": { - "files": [ - "g2.jar", - "libG2.so", - "libG2Hasher.so", - "libG2SSAdm.so", - "libg2CompJavaScoreSet.so", - "libg2DistinctFeatJava.so", - "libg2EFeatJava.so", - "libg2JVMPlugin.so", - "libg2StdJava.so", - "libmariadbplugin.so", - "libSpaceTimeBoxStandardizer.so", - ], - "excludes": ["lib"], - }, - "python": {"files": ["*"], "excludes": []}, - "resources/config": {"files": ["g2core-configuration-upgrade-*.gtc"], "excludes": []}, - "resources/schema": {"files": ["g2core-schema-*-create.sql", "g2core-schema-*-upgrade-*.sql"], "excludes": []}, - "resources/templates": { - "files": [ - "G2C.db*", - "G2Module.ini", - "custom*.txt", - "defaultGNRCP.config", - "g2config.json", - "senzing_governor.py", - "setupEnv", - "stb.config", - ], - "excludes": [], - }, - "sdk": {"files": ["*"], "excludes": ["sdk"]}, - "setupEnv": {"files": [], "excludes": []}, -} - -# TODO - -# V3_COPY_TO_PROJ = { -# "LICENSE": {"files": [], "excludes": []}, -# "NOTICES": {"files": [], "excludes": []}, -# "README.1ST": {"files": [], "excludes": []}, -# "bin": {"files": ["*"], "excludes": ["sz_create_project", "sz_update_project"]}, -# "/opt/senzing/data": {"files": ["*"], "excludes": []}, -# "lib": {"files": ["*"], "excludes": []}, -# "resources": {"files": ["*"], "excludes": []}, -# "sdk": {"files": ["*"], "excludes": []}, -# "szBuildVersion.json": {"files": [], "excludes": []}, -# } -COPY_TO_PROJ = { - "LICENSE": {"files": [], "excludes": []}, - "NOTICES": {"files": [], "excludes": []}, - "README.1ST": {"files": [], "excludes": []}, - "bin": {"files": ["*"], "excludes": ["sz_create_project", "sz_update_project"]}, - "/opt/senzing/data": {"files": ["*"], "excludes": []}, - "lib": {"files": ["*"], "excludes": []}, - "resources": {"files": ["*"], "excludes": []}, - "sdk": {"files": ["*"], "excludes": []}, - "szBuildVersion.json": {"files": [], "excludes": []}, -} - - -V3_RENAME_IN_PROJ = { - "etc": {"from": "G2Module.ini", "to": "sz_engine_config.ini"}, -} - - -V3_RESET_PERMISSIONS = { - ".": { - "dir_pint": 0, - "file_pint": 0o660, - "files": ["LICENSE", "NOTICES", "README.1ST", "szBuildVersion.json"], - "excludes": ["setupEnv"], - "recursive": False, - }, - "setupEnv": {"dir_pint": 0, "file_pint": 0o770, "files": [], "excludes": [], "recursive": False}, - "bin": { - "dir_pint": 0o770, - "file_pint": 0o770, - "files": ["*"], - "excludes": ["__pycache__", "_sz_database.py", "_tool_helpers.py"], - "recursive": False, - }, - "data": {"dir_pint": 0o770, "file_pint": 0o660, "files": ["*"], "excludes": [], "recursive": True}, - "lib": {"dir_pint": 0o770, "file_pint": 0o660, "files": ["*"], "excludes": [], "recursive": False}, - "resources": {"dir_pint": 0o770, "file_pint": 0o660, "files": ["*"], "excludes": ["setupEnv"], "recursive": True}, - "resources/templates/setupEnv": { - "dir_pint": 0, - "file_pint": 0o770, - "files": [], - "excludes": [], - "recursive": False, - }, - "sdk": {"dir_pint": 0o770, "file_pint": 0o660, "files": ["*"], "excludes": [], "recursive": True}, - "v3_upgrade_backups": {"dir_pint": 0o770, "file_pint": 0, "files": [], "excludes": [], "recursive": False}, -} - - -# TODO - -# VersionTokens = namedtuple("VersionTokens", ["major", "minor", "patch"]) -# class VersionDetails(NamedTuple): -@dataclass() -class VersionDetails: - """Version information for a project or Senzing SDK system install""" - - version: str - major: int - minor: int - patch: int - - def __post_init__(self) -> None: - self.sum_mmp = sum((self.major, self.minor, self.patch)) - - -# pylint: disable=W0106 - - -def parse_cli_args() -> argparse.Namespace: - """Parse the CLI arguments""" - arg_parser = argparse.ArgumentParser( - description="Update an existing Senzing project to the system installed version of Senzing." - ) - arg_parser.add_argument( - "project_path", - metavar="path", - help="path of the project to update", - ) - arg_parser.add_argument( - "-f", - "--force", - dest="force_mode", - default=False, - action="store_true", - help="upgrade without prompts. WARNING: Use with caution, ensure you have a backup of the project", - ) - return arg_parser.parse_args() - - -def dir_listing(path: Path) -> list[Path]: - """Get a listing of the specified path to check for files or directories""" - try: - listing = list(path.iterdir()) - except OSError as err: - if type(err).__name__ == "NotADirectoryError": - print(f"\nERROR: {path} is not a directory, expecting one") - print(f"\nERROR: {err}") - sys.exit(1) - - return listing - - -def pre_check(project_path: Path) -> tuple[VersionDetails, VersionDetails]: - """Check not trying to overwrite the V4 Senzing system install, that path is a project, and versions are correct""" - if not V4_SYS_PATH.exists(): - print(f"\nERROR: Couldn't locate Senzing SDK V4 system install at {V4_SYS_PATH}") - sys.exit(1) - - if str(project_path).startswith(str(V4_SYS_PATH.parent)): - print(f"\nERROR: {project_path} is the Senzing system installation, not a Senzing project") - sys.exit(1) - - v3_project_build_file = project_path / V3_BUILD - v4_project_build_file = project_path / V4_BUILD - proj_listing = dir_listing(project_path) - if v3_project_build_file not in proj_listing and v4_project_build_file not in proj_listing: - print(f"\nERROR: {project_path} isn't a Senzing project, expected it to contain either:") - print(f"\tExisting V3 project - {v3_project_build_file}") - print(f"\tExisting V4 project - {v4_project_build_file}") - sys.exit(1) - - try: - proj_build_file = v3_project_build_file if v3_project_build_file in proj_listing else v4_project_build_file - # proj_version, proj_version_tokens = get_build_version(proj_build_file) - # sys_version, sys_version_tokens = get_build_version(V4_SYS_BUILD) - proj_version_details = get_build_version(proj_build_file) - sys_version_details = get_build_version(V4_SYS_BUILD) - except OSError as err: - print(f"\nERROR: Trying to read {proj_build_file} or {V4_SYS_BUILD} to collect version information: {err}") - sys.exit(1) - - # if proj_version_tokens.major not in (3, 4) or sys_version_tokens.major != 4: - if proj_version_details.major not in (3, 4) or sys_version_details.major != 4: - print(f"\nERROR: {MODULE_NAME} updates a V3 project to V4, or V4.n.n to a newer release") - print(f"\tProject version: {proj_version_details.major}, system install version: {sys_version_details.major}") - sys.exit(1) - - if proj_version_details.major == 4 and sys_version_details.sum_mmp == proj_version_details.sum_mmp: - print(f"No update required, project and system install are the same version - {proj_version_details.version}") - sys.exit(0) - - return (proj_version_details, sys_version_details) - - -# TODO - Move to helpers? -def get_build_version(path: Path) -> VersionDetails: - """Return the version string and major, minor, and patch versions from a build file""" - err_msg = f"ERROR: Couldn't get the version information from {path}" - - try: - with open(path, "r", encoding="utf-8") as f: - version_str: str = json.load(f)["VERSION"] - except (OSError, json.JSONDecodeError) as err: - print(f"\n{err_msg}: {err}") - sys.exit(1) - except KeyError as err: - print(f"\n{err_msg}, missing key {err}") - sys.exit(1) - - if not (str_tokens := version_str.split(".")) or (str_tokens and str_tokens[0] == ""): - print(f"\n{err_msg}, VERSION was blank") - sys.exit(1) - - if len(str_tokens) != 3: - print(f"\nERROR: Version information should consist of major, minor, and patch, it is {version_str}") - sys.exit(1) - - try: - # TODO - - # vers_tokens = VersionDetails._make([int(v) for v in str_tokens]) - # int_tokens = [int(v) for v in str_tokens] - return VersionDetails(version_str, *[int(v) for v in str_tokens]) - # vers_tokens = VersionDetails(version_str, str_tokens) - except ValueError: - print(f"\nERROR: Version information should consist of integers, it is {version_str}") - sys.exit(1) - - # TODO - - # return (version_str, vers_details) - - -def remove_dir(dir_: Path, excludes: list[Path]) -> None: - """Recursively remove a directory""" - try: - for path in dir_.iterdir(): - if path in excludes: - continue - path.unlink(missing_ok=True) if (path.is_file() or path.is_symlink()) else remove_dir(path, excludes) - - if not excludes: - with suppress(FileNotFoundError): - dir_.rmdir() - except OSError as err: - raise err - - -def remove_files_dirs(to_remove: dict[str, Any], target_dir: Path) -> None: - """Remove files/directories that are no longer required""" - for r_path, r_dict in to_remove.items(): - target: Path = target_dir / r_path - files = r_dict["files"] - excludes = [target / e for e in r_dict["excludes"]] - - try: - if target.is_file() or target.is_symlink(): - target.unlink(missing_ok=True) - - if target.is_dir() and not files or (files and files[0] == "*"): - remove_dir(target, excludes) - - if target.is_dir() and files and files[0] != "*": - target_files = [] - for f in files: - target_files.append(target / f) if "*" not in f else target_files.extend(list(target.glob(f))) - - for target_file in target_files: - target_file.unlink(missing_ok=True) - except OSError as err: - raise OSError(f"ERROR: Couldn't delete a file or directory: {err}") from err - - -def copy_files_dirs(to_copy: dict[str, Any], source_dir: Path, target_dir: Path) -> None: - # def copy_files_dirs(to_copy: dict[str, Any], source_dir: Path, target_dir: Path, sz_sys_path: Path) -> None: - """Copy files/directories within and to a project""" - # TODO - - print(f"\n{to_copy = }", flush=True) - print(f"\n{source_dir = }", flush=True) - print(f"\n{target_dir = }", flush=True) - # print(f"\n{sz_sys_path = }", flush=True) - for c_path, c_dict in to_copy.items(): - excludes = c_dict["excludes"] - files = c_dict["files"] - source: Path = source_dir / c_path - target: Path = target_dir / c_path - - if c_path.startswith("/"): - source = Path(c_path) - try: - # TODO - - # target = target_dir / source.relative_to(SZ_SYS_PATH) - # target = target_dir / source.relative_to(sz_sys_path) - target = target_dir / source.relative_to(V4_SYS_PATH) - except ValueError: - target = target_dir / target.name - - try: - if source.is_file(): - shutil.copy( - source, - target, - ) - - if source.is_dir() and not files or (files and files[0] == "*"): - shutil.copytree(source, target, ignore=shutil.ignore_patterns(*excludes), dirs_exist_ok=True) - - if source.is_dir() and files and files[0] != "*": - target.mkdir(exist_ok=True, parents=True) - for source_file in [source / f for f in files]: - shutil.copy(source_file, target / source_file.name) - except OSError as err: - raise OSError(f"ERROR: Couldn't copy a file or directory: {err}") from err - - -def rename_files(to_rename: dict[str, dict[str, str]], target_dir: Path) -> None: - """Rename existing project files that had a name change""" - - try: - for r_path, r_dict in to_rename.items(): - current = target_dir / r_path / r_dict["from"] - new = target_dir / r_path / r_dict["to"] - with suppress(FileNotFoundError): - current.rename(new) - except OSError as err: - raise OSError(f"ERROR: Couldn't rename a file or directory: {err}") from err - - -def setup_env(proj_path: Path) -> None: - """Create a new setupEnv and replace place holders with paths for the project""" - try: - shutil.copy(proj_path / "resources/templates/setupEnv", proj_path) - setup_path = proj_path / "setupEnv" - - with open(setup_path, "r", encoding="utf-8") as in_: - data = in_.read() - - data = data.replace("${SENZING_DIR}", str(proj_path)).replace("${SENZING_CONFIG_PATH}", str(proj_path / "etc")) - - with open(setup_path, "w", encoding="utf-8") as out: - out.write(data) - except OSError as err: - raise OSError(f"ERROR: Couldn't create a new setupEnv file: {err}") from err - - -def set_permissions(proj_path: Path, permissions: dict[str, dict[str, Any]]) -> None: - """Set permissions for files/dirs copied to the project, or dirs removed and replaced completely e.g., data/""" - try: - for p_path, p_dict in permissions.items(): - dir_pint = p_dict["dir_pint"] - file_pint = p_dict["file_pint"] - files = p_dict["files"] - recursive = p_dict["recursive"] - target = proj_path if p_path.startswith(".") else proj_path / p_path - excludes = [target / e for e in p_dict["excludes"]] - - if target.is_file(): - target.chmod(file_pint) - - if target.is_dir() and dir_pint != 0: - target.chmod(dir_pint) - d_chmods = ( - [d for d in target.glob("*") if d.is_dir() and not d.is_symlink() and d not in excludes] - if not recursive - else [d for d in target.rglob("*") if d.is_dir() and not d.is_symlink() and d not in excludes] - ) - for dir_ in d_chmods: - dir_.chmod(dir_pint) - - if target.is_dir() and (files and files[0] == "*"): - f_chmods = ( - [f for f in target.glob("*") if f.is_file() and not f.is_symlink() and f not in excludes] - if not recursive - else [f for f in target.rglob("*") if f.is_file() and not f.is_symlink() and f not in excludes] - ) - for file in f_chmods: - Path(target / file).chmod(file_pint) - - if target.is_dir() and files and files[0] != "*": - for file in files: - Path(target / file).chmod(file_pint) - except OSError as err: - raise OSError(f"ERROR: Couldn't set a permission: {err}") from err - - -def main() -> None: - """main""" - cli_args = parse_cli_args() - proj_path = Path(cli_args.project_path).resolve() - proj_versions, sys_versions = pre_check(proj_path) - proj_is_v3 = bool(proj_versions.major == 3) - # sz_sys_path = V3_SYS_PATH if proj_is_v3 else V4_SYS_PATH - - if not cli_args.force_mode: - print(f"\nWARNING: If you don't have a backup of the project ({proj_path}), create one before continuing!") - sleep(3) - if ( - input( - f"\nContinue updating the project from version {proj_versions.version} to {sys_versions.version}? (y/n) " - ) - not in INPUT_CONFS - ): - sys.exit(0) - print("\nUpdating...") - else: - print(f"\nUpdating project from version {proj_versions.version} to {sys_versions.version}...") - - try: - if proj_is_v3: - v3_backup_path = proj_path / V3_BACKUP_PATH - v3_backup_path.mkdir(exist_ok=True) - # copy_files_dirs( - # V3_BACKUP_PROJ, proj_path, v3_backup_path, sz_sys_path - # ) # Backup some of the V3 project files - copy_files_dirs(V3_BACKUP_PROJ, proj_path, v3_backup_path) # Backup some of the V3 project files - rename_files(V3_RENAME_IN_PROJ, proj_path) - remove_files_dirs(V3_REMOVE_FROM_PROJ, proj_path) - - # rename_files(V3_RENAME_IN_PROJ, proj_path) - # remove_files_dirs(V3_REMOVE_FROM_PROJ, proj_path) - # copy_files_dirs(COPY_TO_PROJ, SZ_SYS_PATH, proj_path) - # copy_files_dirs(COPY_TO_PROJ, sz_sys_path, proj_path, sz_sys_path) - # copy_files_dirs(COPY_TO_PROJ, sz_sys_path, proj_path, V4_SYS_PATH) - copy_files_dirs(COPY_TO_PROJ, V4_SYS_PATH, proj_path) - # setup_env(proj_path) - if proj_is_v3: - setup_env(proj_path) - set_permissions(proj_path, V3_RESET_PERMISSIONS) - except OSError as err: - shutil.copytree(v3_backup_path, proj_path, dirs_exist_ok=True) - print(f"\n{err}") - print( - "\nIf the error is file or directory permission related, run again with a user with appropriate privileges" - ) - print( - "\nIf the error is missing file or directory, check senzingsdk-setup, senzingsdk-tools, and senzingsdk-poc are installed" - ) - else: - if proj_is_v3: # Remove if no errors, otherwise re-running can find the file - # proj_path.joinpath(PROJ_BUILD).unlink(missing_ok=True) - proj_path.joinpath(V3_BUILD).unlink(missing_ok=True) - print(f"\nProject successfully updated. Refer to {UPGRADE_URL} for additional upgrade instructions") - - -if __name__ == "__main__": - main() diff --git a/sz_tools/sz_update_project_prior3 b/sz_tools/sz_update_project_prior3 deleted file mode 100755 index 0341c1f..0000000 --- a/sz_tools/sz_update_project_prior3 +++ /dev/null @@ -1,478 +0,0 @@ -#! /usr/bin/env python3 -"""Upgrade a V3 or V4 Senzing SDK project to V4.x.x""" - -import argparse -import json -import shutil -import sys -from contextlib import suppress -from dataclasses import dataclass, field -from pathlib import Path -from time import sleep -from typing import Any - -from packaging import version as p_version - -INPUT_CONFS = ("y", "Y", "yEs", "yES", "YES", "yes", "YEs", "yeS", "Yes", "YeS") -MODULE_NAME = Path(__file__).stem -V3_BACKUP_PATH = "v3_upgrade_backups" -V3_BUILD = "g2BuildVersion.json" -V4_BUILD = "szBuildVersion.json" -V3_SYS_PATH = Path("/opt/senzing/g2") -V4_SYS_PATH = Path("/opt/senzing/er") -V3_SYS_BUILD = V3_SYS_PATH / V3_BUILD -V4_SYS_BUILD = V4_SYS_PATH / V4_BUILD -UPGRADE_URL = "https://www.senzing.com/docs/4_beta/index.html" - -V3_BACKUP_PROJ = { - "bin": {"files": ["g2dbencrypt", "g2saltadm", "g2ssadm"], "excludes": []}, - "etc": {"files": ["senzing_governor.py"], "excludes": []}, - "lib": {"files": ["libSpaceTimeBoxStandardizer.so", "libG2Hasher.so", "libG2SSAdm.so"], "excludes": []}, - "python": {"files": [], "excludes": []}, - "resources/templates": {"files": ["setupEnv"], "excludes": []}, - "setupEnv": {"files": [], "excludes": []}, - "g2BuildVersion.json": {"files": [], "excludes": []}, -} - -V3_REMOVE_FROM_PROJ = { - "bin": {"files": ["*"], "excludes": ["bin"]}, - "data": {"files": [], "excludes": []}, - "etc": {"files": ["senzing_governor.py"], "excludes": []}, - "lib": { - "files": ["*"], - # "files": [ - # "g2.jar", - # "libG2.so", - # "libG2Hasher.so", - # "libG2SSAdm.so", - # "libg2CompJavaScoreSet.so", - # "libg2DistinctFeatJava.so", - # "libg2EFeatJava.so", - # "libg2JVMPlugin.so", - # "libg2StdJava.so", - # "libmariadbplugin.so", - # "libSpaceTimeBoxStandardizer.so", - # ], - "excludes": ["lib"], - }, - "python": {"files": ["*"], "excludes": []}, - "resources/config": {"files": ["g2core-configuration-upgrade-*.gtc"], "excludes": []}, - "resources/schema": {"files": ["g2core-schema-*-create.sql", "g2core-schema-*-upgrade-*.sql"], "excludes": []}, - "resources/templates": { - "files": [ - "G2C.db*", - "G2Module.ini", - "custom*.txt", - "defaultGNRCP.config", - "g2config.json", - "senzing_governor.py", - "setupEnv", - "stb.config", - ], - "excludes": [], - }, - "sdk": {"files": ["*"], "excludes": ["sdk"]}, - "setupEnv": {"files": [], "excludes": []}, -} - -COPY_TO_PROJ = { - "LICENSE": {"files": [], "excludes": []}, - "NOTICES": {"files": [], "excludes": []}, - "README.1ST": {"files": [], "excludes": []}, - "bin": {"files": ["*"], "excludes": ["sz_create_project", "sz_update_project"]}, - "/opt/senzing/data": {"files": ["*"], "excludes": []}, - "lib": {"files": ["*"], "excludes": []}, - "resources": {"files": ["*"], "excludes": []}, - "sdk": {"files": ["*"], "excludes": []}, - "szBuildVersion.json": {"files": [], "excludes": []}, -} - -V3_RENAME_IN_PROJ = { - "etc": {"from": "G2Module.ini", "to": "sz_engine_config.ini"}, -} - -# TODO - Rename and check -SET_PERMISSIONS = { - ".": { - "dir_pint": 0, - "file_pint": 0o660, - "files": ["LICENSE", "NOTICES", "README.1ST", "szBuildVersion.json"], - "excludes": ["setupEnv"], - "recursive": False, - }, - "setupEnv": {"dir_pint": 0, "file_pint": 0o770, "files": [], "excludes": [], "recursive": False}, - "bin": { - "dir_pint": 0o770, - "file_pint": 0o770, - "files": ["*"], - "excludes": ["__pycache__", "_sz_database.py", "_tool_helpers.py"], - "recursive": False, - }, - "data": {"dir_pint": 0o770, "file_pint": 0o660, "files": ["*"], "excludes": [], "recursive": True}, - "lib": {"dir_pint": 0o770, "file_pint": 0o660, "files": ["*"], "excludes": [], "recursive": False}, - "resources": {"dir_pint": 0o770, "file_pint": 0o660, "files": ["*"], "excludes": ["setupEnv"], "recursive": True}, - "resources/templates/setupEnv": { - "dir_pint": 0, - "file_pint": 0o770, - "files": [], - "excludes": [], - "recursive": False, - }, - "sdk": {"dir_pint": 0o770, "file_pint": 0o660, "files": ["*"], "excludes": [], "recursive": True}, - "v3_upgrade_backups": {"dir_pint": 0o770, "file_pint": 0, "files": [], "excludes": [], "recursive": False}, -} - - -@dataclass() -class BuildDetails: - """Build information for a project or Senzing SDK system install""" - - # version: str - # # TODO - - # version_parsed: p_version.Version - # major: int - # minor: int - # patch: int - # build: int - platform: str - version: str - build_version: str - build_number: str - major: int = field(init=False) - minor: int = field(init=False) - micro: int = field(init=False) - - def __post_init__(self) -> None: - # self.sum_mmp = sum((self.major, self.minor, self.patch)) - self.version_parsed = p_version.parse(self.version) - self.build_version_parsed = p_version.parse(self.build_version) - self.major = self.version_parsed.major - self.minor = self.version_parsed.minor - self.micro = self.version_parsed.micro - - -# pylint: disable=W0106 - - -def parse_cli_args() -> argparse.Namespace: - """Parse the CLI arguments""" - arg_parser = argparse.ArgumentParser( - allow_abbrev=False, - description="Update an existing V3 or V4 Senzing project to the system installed V4 of Senzing.", - ) - arg_parser.add_argument( - "project_path", - metavar="path", - help="Path of the project to update", - ) - arg_parser.add_argument( - "-f", - "--force", - dest="force_mode", - default=False, - action="store_true", - help="Upgrade without prompts. WARNING: Use with caution, ensure you have a backup of the project", - ) - return arg_parser.parse_args() - - -def dir_listing(path: Path) -> list[Path]: - """Get a listing of the specified path to check for files or directories""" - try: - listing = list(path.iterdir()) - except OSError as err: - if type(err).__name__ == "NotADirectoryError": - print(f"\nERROR: {path} is not a directory, expecting one") - print(f"\nERROR: {err}") - sys.exit(1) - - return listing - - -def pre_check(project_path: Path) -> tuple[BuildDetails, BuildDetails]: - """Check not trying to overwrite the V4 Senzing system install, that path is a project, and versions are correct""" - if not V4_SYS_PATH.exists(): - print(f"\nERROR: Couldn't locate Senzing SDK V4 system install at {V4_SYS_PATH}") - sys.exit(1) - - if str(project_path).startswith(str(V4_SYS_PATH.parent)): - print(f"\nERROR: {project_path} is the Senzing system installation, not a Senzing project") - sys.exit(1) - - v3_project_build_file = project_path / V3_BUILD - v4_project_build_file = project_path / V4_BUILD - proj_listing = dir_listing(project_path) - if v3_project_build_file not in proj_listing and v4_project_build_file not in proj_listing: - print(f"\nERROR: {project_path} isn't a Senzing project, expected it to contain either:") - print(f"\tExisting V3 project: {v3_project_build_file}") - print(f"\tExisting V4 project: {v4_project_build_file}") - sys.exit(1) - - try: - proj_build_file = v3_project_build_file if v3_project_build_file in proj_listing else v4_project_build_file - proj_version_details = get_build_details(proj_build_file) - sys_version_details = get_build_details(V4_SYS_BUILD) - except OSError as err: - print(f"\nERROR: Trying to read {proj_build_file} or {V4_SYS_BUILD} to collect version information: {err}") - sys.exit(1) - - if proj_version_details.major not in (3, 4) or sys_version_details.major != 4: - print(f"\nERROR: {MODULE_NAME} updates a V3 project to V4, or V4.n.n to a newer release") - print(f"\tProject version: {proj_version_details.version}") - print(f"\tSystem install version: {sys_version_details.version}") - sys.exit(1) - - # TODO - - # if sys_version_details.sum_mmp < proj_version_details.sum_mmp: - if sys_version_details.version_parsed < proj_version_details.version_parsed: - print("\nNo update required, project is newer than the system install") - print(f"\tProject version: {proj_version_details.version}") - print(f"\tSystem install version: {sys_version_details.version}") - sys.exit(0) - - # TODO - - # if proj_version_details.major == 4 and sys_version_details.sum_mmp == proj_version_details.sum_mmp: - if proj_version_details.major == 4 and sys_version_details.version_parsed == proj_version_details.version_parsed: - print(f"\nNo update required, project and system install are the same version - {proj_version_details.version}") - sys.exit(0) - - return (proj_version_details, sys_version_details) - - -# TODO - Move to helpers -def get_build_details(path: Path) -> BuildDetails: - """Return dataclass with version string, major, minor, and patch versions from a build file""" - err_msg = f"ERROR: Couldn't get the version information from {path}" - - try: - with open(path, "r", encoding="utf-8") as f: - # TODO - - # version_str: str = json.load(f)["VERSION"] - # version_str: str = json.load(f)["BUILD_VERSION"] - version_dict: Any = json.load(f) - version_dict = {k.lower(): v for k, v in version_dict.items() if k != "DATA_VERSION"} - # TODO - - print(f"\n{version_dict = }", flush=True) - except (OSError, json.JSONDecodeError) as err: - print(f"\n{err_msg}: {err}") - sys.exit(1) - # except KeyError as err: - # print(f"\n{err_msg}, missing key {err}") - # sys.exit(1) - - # if not (str_tokens := version_str.split(".")) or (str_tokens and str_tokens[0] == ""): - # # TODO - - # # print(f"\n{err_msg}, VERSION was blank") - # print(f"\n{err_msg}, BUILD_VERSION was blank or misformed") - # sys.exit(1) - - # # TODO - - # # if len(str_tokens) != 3: - # if len(str_tokens) != 4: - # print(f"\nERROR: Version information should consist of major, minor, patch, and build, it is {version_str}") - # sys.exit(1) - - try: - # TODO - - # return VersionDetails(version_str, *[int(v) for v in str_tokens]) - # return VersionDetails(version_str, p_version.parse(version_str), *[int(v) for v in str_tokens]) - return BuildDetails(**version_dict) - except ValueError: - # print(f"\nERROR: Version information should consist of integers, it is {version_str}") - sys.exit(1) - - -def remove_dir(dir_: Path, excludes: list[Path]) -> None: - """Recursively remove a directory""" - try: - for path in dir_.iterdir(): - if path in excludes: - continue - path.unlink(missing_ok=True) if (path.is_file() or path.is_symlink()) else remove_dir(path, excludes) - - if not excludes: - with suppress(FileNotFoundError): - dir_.rmdir() - except OSError as err: - raise err - - -def remove_files_dirs(to_remove: dict[str, Any], target_dir: Path) -> None: - """Remove files/directories that are no longer required""" - for r_path, r_dict in to_remove.items(): - target: Path = target_dir / r_path - files = r_dict["files"] - excludes = [target / e for e in r_dict["excludes"]] - - try: - if target.is_file() or target.is_symlink(): - target.unlink(missing_ok=True) - - if target.is_dir() and not files or (files and files[0] == "*"): - remove_dir(target, excludes) - - if target.is_dir() and files and files[0] != "*": - target_files = [] - for f in files: - target_files.append(target / f) if "*" not in f else target_files.extend(list(target.glob(f))) - - for target_file in target_files: - target_file.unlink(missing_ok=True) - except OSError as err: - raise OSError(f"ERROR: Couldn't delete a file or directory: {err}") from err - - -def copy_files_dirs(to_copy: dict[str, Any], source_dir: Path, target_dir: Path) -> None: - """Copy files/directories within and to a project""" - for c_path, c_dict in to_copy.items(): - excludes = c_dict["excludes"] - files = c_dict["files"] - source: Path = source_dir / c_path - target: Path = target_dir / c_path - - if c_path.startswith("/"): - source = Path(c_path) - try: - target = target_dir / source.relative_to(V4_SYS_PATH) - except ValueError: - target = target_dir / target.name - - try: - if source.is_file(): - shutil.copy( - source, - target, - ) - - if source.is_dir() and not files or (files and files[0] == "*"): - shutil.copytree(source, target, ignore=shutil.ignore_patterns(*excludes), dirs_exist_ok=True) - - if source.is_dir() and files and files[0] != "*": - target.mkdir(exist_ok=True, parents=True) - for source_file in [source / f for f in files]: - shutil.copy(source_file, target / source_file.name) - except OSError as err: - raise OSError(f"ERROR: Couldn't copy a file or directory: {err}") from err - - -def rename_files(to_rename: dict[str, dict[str, str]], target_dir: Path) -> None: - """Rename existing project files that had a name change""" - - try: - for r_path, r_dict in to_rename.items(): - current = target_dir / r_path / r_dict["from"] - new = target_dir / r_path / r_dict["to"] - with suppress(FileNotFoundError): - current.rename(new) - except OSError as err: - raise OSError(f"ERROR: Couldn't rename a file or directory: {err}") from err - - -def setup_env(proj_path: Path) -> None: - """Create a new setupEnv and replace place holders with paths for the project""" - try: - shutil.copy(proj_path / "resources/templates/setupEnv", proj_path) - setup_path = proj_path / "setupEnv" - - with open(setup_path, "r", encoding="utf-8") as in_: - data = in_.read() - - data = data.replace("${SENZING_DIR}", str(proj_path)).replace("${SENZING_CONFIG_PATH}", str(proj_path / "etc")) - - with open(setup_path, "w", encoding="utf-8") as out: - out.write(data) - except OSError as err: - raise OSError(f"ERROR: Couldn't create a new setupEnv file: {err}") from err - - -def set_permissions(proj_path: Path, permissions: dict[str, dict[str, Any]]) -> None: - """Set permissions for files/dirs copied to the project, or dirs removed and replaced completely e.g., data/""" - try: - for p_path, p_dict in permissions.items(): - dir_pint = p_dict["dir_pint"] - file_pint = p_dict["file_pint"] - files = p_dict["files"] - recursive = p_dict["recursive"] - target = proj_path if p_path.startswith(".") else proj_path / p_path - excludes = [target / e for e in p_dict["excludes"]] - - if target.is_file(): - target.chmod(file_pint) - - if target.is_dir() and dir_pint != 0: - target.chmod(dir_pint) - d_chmods = ( - [d for d in target.glob("*") if d.is_dir() and not d.is_symlink() and d not in excludes] - if not recursive - else [d for d in target.rglob("*") if d.is_dir() and not d.is_symlink() and d not in excludes] - ) - for dir_ in d_chmods: - dir_.chmod(dir_pint) - - if target.is_dir() and (files and files[0] == "*"): - f_chmods = ( - [f for f in target.glob("*") if f.is_file() and not f.is_symlink() and f not in excludes] - if not recursive - else [f for f in target.rglob("*") if f.is_file() and not f.is_symlink() and f not in excludes] - ) - for file in f_chmods: - Path(target / file).chmod(file_pint) - - if target.is_dir() and files and files[0] != "*": - for file in files: - Path(target / file).chmod(file_pint) - except OSError as err: - raise OSError(f"ERROR: Couldn't set a permission: {err}") from err - - -def main() -> None: - """main""" - cli_args = parse_cli_args() - proj_path = Path(cli_args.project_path).resolve() - proj_build, sys_build = pre_check(proj_path) - print(f"\n{proj_build = }", flush=True) - print(f"\n{proj_build.version_parsed = } - {proj_build.major}", flush=True) - sys.exit() - proj_is_v3 = bool(proj_build.major == 3) - - if not cli_args.force_mode: - print("\nWARNING: If you don't have a backup of the project, create one before continuing!") - sleep(3) - if ( - input(f"\nUpdate the project from version {proj_build.version} to {sys_build.version}? (y/n) ") - not in INPUT_CONFS - ): - sys.exit(0) - print("\nUpdating...") - else: - print(f"\nUpdating project from version {proj_build.version} to {sys_build.version}...") - - try: - if proj_is_v3: - v3_backup_path = proj_path / V3_BACKUP_PATH - v3_backup_path.mkdir(exist_ok=True) - copy_files_dirs(V3_BACKUP_PROJ, proj_path, v3_backup_path) # Backup some of the V3 project files - rename_files(V3_RENAME_IN_PROJ, proj_path) - remove_files_dirs(V3_REMOVE_FROM_PROJ, proj_path) - - copy_files_dirs(COPY_TO_PROJ, V4_SYS_PATH, proj_path) - if proj_is_v3: - setup_env(proj_path) - # TODO - Check V4 permissions are good, is sz_create_project good in the first place when V4 to V4? - set_permissions(proj_path, SET_PERMISSIONS) - except OSError as err: - shutil.copytree(v3_backup_path, proj_path, dirs_exist_ok=True) - print(f"\n{err}") - print("\nIf the error is file or directory permission related, run again with appropriate privileges") - print( - "\nIf the error is missing file or directory, check senzingsdk-setup, senzingsdk-tools, and senzingsdk-poc are installed" - ) - else: - if proj_is_v3: # Only delete if no errors so re-running can find the file again - proj_path.joinpath(V3_BUILD).unlink(missing_ok=True) - - print(f"\nProject successfully updated. Refer to {UPGRADE_URL} for additional upgrade instructions") - - -if __name__ == "__main__": - main() diff --git a/sz_tools/sz_update_project_prior4 b/sz_tools/sz_update_project_prior4 deleted file mode 100755 index 9cf51f8..0000000 --- a/sz_tools/sz_update_project_prior4 +++ /dev/null @@ -1,456 +0,0 @@ -#! /usr/bin/env python3 -"""Upgrade a V3 or V4 Senzing SDK project to V4.x.x""" - -import argparse -import json -import shutil -import sys -from contextlib import suppress -from dataclasses import dataclass, field -from pathlib import Path -from time import sleep -from typing import Any - -from packaging import version as p_version - -INPUT_CONFS = ("y", "Y", "yEs", "yES", "YES", "yes", "YEs", "yeS", "Yes", "YeS") -MODULE_NAME = Path(__file__).stem -V3_BACKUP_PATH = "v3_upgrade_backups" -V3_BUILD = "g2BuildVersion.json" -V4_BUILD = "szBuildVersion.json" -V3_SYS_PATH = Path("/opt/senzing/g2") -V4_SYS_PATH = Path("/opt/senzing/er") -V3_SYS_BUILD = V3_SYS_PATH / V3_BUILD -V4_SYS_BUILD = V4_SYS_PATH / V4_BUILD -UPGRADE_URL = "https://www.senzing.com/docs/4_beta/index.html" - -V3_BACKUP_PROJ = { - "bin": {"files": ["g2dbencrypt", "g2saltadm", "g2ssadm"], "excludes": []}, - "etc": {"files": ["senzing_governor.py"], "excludes": []}, - "lib": {"files": ["libSpaceTimeBoxStandardizer.so", "libG2Hasher.so", "libG2SSAdm.so"], "excludes": []}, - "python": {"files": [], "excludes": []}, - "resources/templates": {"files": ["setupEnv"], "excludes": []}, - "setupEnv": {"files": [], "excludes": []}, - "g2BuildVersion.json": {"files": [], "excludes": []}, -} - -V3_REMOVE_FROM_PROJ = { - "bin": {"files": ["*"], "excludes": ["bin"]}, - "data": {"files": [], "excludes": []}, - "etc": {"files": ["senzing_governor.py"], "excludes": []}, - "lib": { - "files": ["*"], - # TODO - Others simplified? - # "files": [ - # "g2.jar", - # "libG2.so", - # "libG2Hasher.so", - # "libG2SSAdm.so", - # "libg2CompJavaScoreSet.so", - # "libg2DistinctFeatJava.so", - # "libg2EFeatJava.so", - # "libg2JVMPlugin.so", - # "libg2StdJava.so", - # "libmariadbplugin.so", - # "libSpaceTimeBoxStandardizer.so", - # ], - "excludes": ["lib"], - }, - "python": {"files": ["*"], "excludes": []}, - "resources/config": {"files": ["g2core-configuration-upgrade-*.gtc"], "excludes": []}, - "resources/schema": {"files": ["g2core-schema-*-create.sql", "g2core-schema-*-upgrade-*.sql"], "excludes": []}, - "resources/templates": { - "files": [ - "G2C.db*", - "G2Module.ini", - "custom*.txt", - "defaultGNRCP.config", - "g2config.json", - "senzing_governor.py", - "setupEnv", - "stb.config", - ], - "excludes": [], - }, - "sdk": {"files": ["*"], "excludes": ["sdk"]}, - "setupEnv": {"files": [], "excludes": []}, -} - -COPY_TO_PROJ = { - "LICENSE": {"files": [], "excludes": []}, - "NOTICES": {"files": [], "excludes": []}, - "README.1ST": {"files": [], "excludes": []}, - "bin": {"files": ["*"], "excludes": ["sz_create_project", "sz_update_project"]}, - # TODO - - "/opt/senzing/data": {"files": ["*"], "excludes": []}, - "lib": {"files": ["*"], "excludes": []}, - "resources": {"files": ["*"], "excludes": []}, - "sdk": {"files": ["*"], "excludes": []}, - "szBuildVersion.json": {"files": [], "excludes": []}, -} - -V3_RENAME_IN_PROJ = { - "etc": {"from": "G2Module.ini", "to": "sz_engine_config.ini"}, -} - -# TODO - Rename and check -SET_PERMISSIONS = { - ".": { - "dir_pint": 0, - "file_pint": 0o660, - "files": ["LICENSE", "NOTICES", "README.1ST", "szBuildVersion.json"], - "excludes": ["setupEnv"], - "recursive": False, - }, - "setupEnv": {"dir_pint": 0, "file_pint": 0o770, "files": [], "excludes": [], "recursive": False}, - "bin": { - "dir_pint": 0o770, - "file_pint": 0o770, - "files": ["*"], - "excludes": ["__pycache__", "_sz_database.py", "_tool_helpers.py"], - "recursive": False, - }, - "data": {"dir_pint": 0o770, "file_pint": 0o660, "files": ["*"], "excludes": [], "recursive": True}, - "lib": {"dir_pint": 0o770, "file_pint": 0o660, "files": ["*"], "excludes": [], "recursive": False}, - "resources": {"dir_pint": 0o770, "file_pint": 0o660, "files": ["*"], "excludes": ["setupEnv"], "recursive": True}, - "resources/templates/setupEnv": { - "dir_pint": 0, - "file_pint": 0o770, - "files": [], - "excludes": [], - "recursive": False, - }, - "sdk": {"dir_pint": 0o770, "file_pint": 0o660, "files": ["*"], "excludes": [], "recursive": True}, - "v3_upgrade_backups": {"dir_pint": 0o770, "file_pint": 0, "files": [], "excludes": [], "recursive": False}, -} - - -@dataclass() -class SzBuildDetails: - """Build information for a project or Senzing SDK system install""" - - platform: str - version: str - build_version: str - build_number: str - major: int = field(init=False) - minor: int = field(init=False) - micro: int = field(init=False) - - def __post_init__(self) -> None: - self.version_parsed = p_version.parse(self.version) - self.build_version_parsed = p_version.parse(self.build_version) - self.major = self.version_parsed.major - self.minor = self.version_parsed.minor - self.micro = self.version_parsed.micro - - -# pylint: disable=W0106 - - -def parse_cli_args() -> argparse.Namespace: - """Parse the CLI arguments""" - arg_parser = argparse.ArgumentParser( - allow_abbrev=False, - description="Update an existing V3 or V4 Senzing project to the system installed V4 of Senzing.", - ) - arg_parser.add_argument( - "project_path", - metavar="path", - help="Path of the project to update", - ) - arg_parser.add_argument( - "-f", - "--force", - dest="force_mode", - default=False, - action="store_true", - help="Upgrade without prompts. WARNING: Use with caution, ensure you have a backup of the project", - ) - return arg_parser.parse_args() - - -def dir_listing(path: Path) -> list[Path]: - """Get a listing of the specified path to check for files or directories""" - try: - listing = list(path.iterdir()) - except OSError as err: - if type(err).__name__ == "NotADirectoryError": - print(f"\nERROR: {path} is not a directory, expecting one") - print(f"\nERROR: {err}") - sys.exit(1) - - return listing - - -def pre_check(project_path: Path) -> tuple[SzBuildDetails, SzBuildDetails]: - """Check not trying to overwrite the V4 Senzing system install, that path is a project, and versions are correct""" - if not V4_SYS_PATH.exists(): - print(f"\nERROR: Couldn't locate Senzing SDK V4 system install at {V4_SYS_PATH}") - sys.exit(1) - - if str(project_path).startswith(str(V4_SYS_PATH.parent)): - print(f"\nERROR: {project_path} is the Senzing system installation, not a Senzing project") - sys.exit(1) - - v3_project_build_file = project_path / V3_BUILD - v4_project_build_file = project_path / V4_BUILD - proj_listing = dir_listing(project_path) - if v3_project_build_file not in proj_listing and v4_project_build_file not in proj_listing: - print(f"\nERROR: {project_path} isn't a Senzing project, expected it to contain either:") - print(f"\tExisting V3 project: {v3_project_build_file}") - print(f"\tExisting V4 project: {v4_project_build_file}") - sys.exit(1) - - try: - proj_build_file = v3_project_build_file if v3_project_build_file in proj_listing else v4_project_build_file - proj_build_details = get_build_details(proj_build_file) - sys_build_details = get_build_details(V4_SYS_BUILD) - except OSError as err: - print(f"\nERROR: Trying to read {proj_build_file} or {V4_SYS_BUILD} to collect version information: {err}") - sys.exit(1) - - if proj_build_details.major not in (3, 4) or sys_build_details.major != 4: - print(f"\nERROR: {MODULE_NAME} updates a V3 project to V4, or V4.n.n to a newer release") - print(f"\tProject version: {proj_build_details.version}") - print(f"\tSystem install version: {sys_build_details.version}") - sys.exit(1) - - # TODO - - if sys_build_details.version_parsed < proj_build_details.version_parsed: - print("\nNo update required, project is newer than the system install") - print(f"\tProject version: {proj_build_details.version}") - print(f"\tSystem install version: {sys_build_details.version}") - sys.exit(0) - - # TODO - - if sys_build_details.version_parsed == proj_build_details.version_parsed: - print(f"\nNo update required, project and system install are the same version - {proj_build_details.version}") - sys.exit(0) - - return (proj_build_details, sys_build_details) - - -# TODO - Move to helpers -def get_build_details(path: Path) -> SzBuildDetails: - """Return dataclass with the details from a build file.""" - try: - with open(path, "r", encoding="utf-8") as f: - # Ignore DATA_VERSION V3 build files had it, V4 doesn't - version_dict = {k.lower(): v for k, v in json.load(f).items() if k != "DATA_VERSION"} - return SzBuildDetails(**version_dict) - except (OSError, json.JSONDecodeError) as err: - print(f"\nERROR: Couldn't get the build information from {path}: {err}") - sys.exit(1) - except TypeError as err: - print(f"\nERROR: When creating build information from {path}: {err}") - sys.exit(1) - - -def remove_dir(dir_: Path, excludes: list[Path]) -> None: - """Recursively remove a directory""" - try: - for path in dir_.iterdir(): - if path in excludes: - continue - path.unlink(missing_ok=True) if (path.is_file() or path.is_symlink()) else remove_dir(path, excludes) - - if not excludes: - with suppress(FileNotFoundError): - dir_.rmdir() - except OSError as err: - raise err - - -def remove_files_dirs(to_remove: dict[str, Any], target_dir: Path) -> None: - """Remove files/directories that are no longer required""" - for r_path, r_dict in to_remove.items(): - target: Path = target_dir / r_path - files = r_dict["files"] - excludes = [target / e for e in r_dict["excludes"]] - - try: - if target.is_file() or target.is_symlink(): - target.unlink(missing_ok=True) - - if target.is_dir() and not files or (files and files[0] == "*"): - remove_dir(target, excludes) - - if target.is_dir() and files and files[0] != "*": - target_files = [] - for f in files: - target_files.append(target / f) if "*" not in f else target_files.extend(list(target.glob(f))) - - for target_file in target_files: - target_file.unlink(missing_ok=True) - except OSError as err: - raise OSError(f"ERROR: Couldn't delete a file or directory: {err}") from err - - -def copy_files_dirs(to_copy: dict[str, Any], source_dir: Path, target_dir: Path) -> None: - """Copy files/directories within and to a project""" - # TODO - - print(f"\n{to_copy = }", flush=True) - print(f"{source_dir = }", flush=True) - print(f"{target_dir = }", flush=True) - for c_path, c_dict in to_copy.items(): - print(f"\n{c_path = } - {c_dict}", flush=True) - excludes = c_dict["excludes"] - files = c_dict["files"] - source: Path = source_dir / c_path - target: Path = target_dir / c_path - print(f"\n{excludes = }", flush=True) - print(f"{files = }", flush=True) - print(f"{source = }", flush=True) - print(f"{target = }", flush=True) - - if c_path.startswith("/"): - print("\nStarts with/...", flush=True) - source = Path(c_path) - try: - print("here", flush=True) - print(f"\tTrying relative to: {target_dir = } - {source.relative_to(V4_SYS_PATH) = }", flush=True) - target = target_dir / source.relative_to(V4_SYS_PATH) - except ValueError: - print(f"\tTrying target.name: {target = } - {target_dir / target.name}", flush=True) - target = target_dir / target.name - print(f"\tNow: {source = }", flush=True) - print(f"\tNow: {target = }", flush=True) - - try: - if source.is_file(): - # TODO - remove shutil - shutil.copy( - source, - target, - ) - - if source.is_dir() and not files or (files and files[0] == "*"): - shutil.copytree(source, target, ignore=shutil.ignore_patterns(*excludes), dirs_exist_ok=True) - - if source.is_dir() and files and files[0] != "*": - target.mkdir(exist_ok=True, parents=True) - for source_file in [source / f for f in files]: - shutil.copy(source_file, target / source_file.name) - except OSError as err: - raise OSError(f"ERROR: Couldn't copy a file or directory: {err}") from err - - -def rename_files(to_rename: dict[str, dict[str, str]], target_dir: Path) -> None: - """Rename existing project files that had a name change""" - - try: - for r_path, r_dict in to_rename.items(): - current = target_dir / r_path / r_dict["from"] - new = target_dir / r_path / r_dict["to"] - with suppress(FileNotFoundError): - current.rename(new) - except OSError as err: - raise OSError(f"ERROR: Couldn't rename a file or directory: {err}") from err - - -def setup_env(proj_path: Path) -> None: - """Create a new setupEnv and replace place holders with paths for the project""" - try: - shutil.copy(proj_path / "resources/templates/setupEnv", proj_path) - setup_path = proj_path / "setupEnv" - - with open(setup_path, "r", encoding="utf-8") as in_: - data = in_.read() - - data = data.replace("${SENZING_DIR}", str(proj_path)).replace("${SENZING_CONFIG_PATH}", str(proj_path / "etc")) - - with open(setup_path, "w", encoding="utf-8") as out: - out.write(data) - except OSError as err: - raise OSError(f"ERROR: Couldn't create a new setupEnv file: {err}") from err - - -def set_permissions(proj_path: Path, permissions: dict[str, dict[str, Any]]) -> None: - """Set permissions for files/dirs copied to the project, or dirs removed and replaced completely e.g., data/""" - try: - for p_path, p_dict in permissions.items(): - dir_pint = p_dict["dir_pint"] - file_pint = p_dict["file_pint"] - files = p_dict["files"] - recursive = p_dict["recursive"] - target = proj_path if p_path.startswith(".") else proj_path / p_path - excludes = [target / e for e in p_dict["excludes"]] - - if target.is_file(): - target.chmod(file_pint) - - if target.is_dir() and dir_pint != 0: - target.chmod(dir_pint) - d_chmods = ( - [d for d in target.glob("*") if d.is_dir() and not d.is_symlink() and d not in excludes] - if not recursive - else [d for d in target.rglob("*") if d.is_dir() and not d.is_symlink() and d not in excludes] - ) - for dir_ in d_chmods: - dir_.chmod(dir_pint) - - if target.is_dir() and (files and files[0] == "*"): - f_chmods = ( - [f for f in target.glob("*") if f.is_file() and not f.is_symlink() and f not in excludes] - if not recursive - else [f for f in target.rglob("*") if f.is_file() and not f.is_symlink() and f not in excludes] - ) - for file in f_chmods: - Path(target / file).chmod(file_pint) - - if target.is_dir() and files and files[0] != "*": - for file in files: - Path(target / file).chmod(file_pint) - except OSError as err: - raise OSError(f"ERROR: Couldn't set a permission: {err}") from err - - -def main() -> None: - """main""" - cli_args = parse_cli_args() - proj_path = Path(cli_args.project_path).resolve() - proj_build, sys_build = pre_check(proj_path) - proj_is_v3 = bool(proj_build.major == 3) - - if not cli_args.force_mode: - print("\nWARNING: If you don't have a backup of the project, create one before continuing!") - sleep(3) - if ( - input(f"\nUpdate the project from version {proj_build.build_version} to {sys_build.build_version}? (y/n) ") - not in INPUT_CONFS - ): - sys.exit(0) - print("\nUpdating...") - else: - print(f"\nUpdating project from version {proj_build.build_version} to {sys_build.build_version}...") - - try: - if proj_is_v3: - v3_backup_path = proj_path / V3_BACKUP_PATH - v3_backup_path.mkdir(exist_ok=True) - copy_files_dirs(V3_BACKUP_PROJ, proj_path, v3_backup_path) # Backup some of the V3 project files - rename_files(V3_RENAME_IN_PROJ, proj_path) - remove_files_dirs(V3_REMOVE_FROM_PROJ, proj_path) - - copy_files_dirs(COPY_TO_PROJ, V4_SYS_PATH, proj_path) - if proj_is_v3: - setup_env(proj_path) - # TODO - Check V4 permissions are good, is sz_create_project good in the first place when V4 to V4? - set_permissions(proj_path, SET_PERMISSIONS) - except OSError as err: - shutil.copytree(v3_backup_path, proj_path, dirs_exist_ok=True) - print(f"\n{err}") - print("\nIf the error is file or directory permission related, run again with appropriate privileges") - print( - "\nIf the error is missing file or directory, check senzingsdk-setup, senzingsdk-tools, and senzingsdk-poc are installed" - ) - else: - if proj_is_v3: # Only delete if no errors so re-running can find the file again - proj_path.joinpath(V3_BUILD).unlink(missing_ok=True) - - print(f"\nProject successfully updated. Refer to {UPGRADE_URL} for additional upgrade instructions") - - -if __name__ == "__main__": - main() diff --git a/sz_tools/sz_update_project_prior5 b/sz_tools/sz_update_project_prior5 deleted file mode 100755 index cb4b3ef..0000000 --- a/sz_tools/sz_update_project_prior5 +++ /dev/null @@ -1,515 +0,0 @@ -#! /usr/bin/env python3 -"""Upgrade a V3 or V4 Senzing SDK project to V4.x.x""" - -import argparse -import json -import shutil -import sys -from contextlib import suppress -from dataclasses import dataclass, field -from pathlib import Path -from time import sleep -from typing import Any - -from packaging import version as p_version - -INPUT_CONFS = ("y", "Y", "yEs", "yES", "YES", "yes", "YEs", "yeS", "Yes", "YeS") -MODULE_NAME = Path(__file__).stem -V3_BACKUP_PATH = "v3_upgrade_backups" -V3_BUILD = "g2BuildVersion.json" -V4_BUILD = "szBuildVersion.json" -# TODO - -SZ_SYS_PATH = Path("/opt/senzing") -V3_SYS_PATH = SZ_SYS_PATH / "g2" -V4_SYS_PATH = SZ_SYS_PATH / "er" -V4_DATA_PATH = SZ_SYS_PATH / "data" - -# V3_SYS_PATH = Path("/opt/senzing/g2") -# V4_SYS_PATH = Path("/opt/senzing/er") -V3_SYS_BUILD = V3_SYS_PATH / V3_BUILD -V4_SYS_BUILD = V4_SYS_PATH / V4_BUILD -UPGRADE_URL = "https://www.senzing.com/docs/4_beta/index.html" - -V3_BACKUP_PROJ = { - "bin": {"files": ["g2dbencrypt", "g2saltadm", "g2ssadm"], "excludes": []}, - "etc": {"files": ["senzing_governor.py"], "excludes": []}, - "lib": {"files": ["libSpaceTimeBoxStandardizer.so", "libG2Hasher.so", "libG2SSAdm.so"], "excludes": []}, - "python": {"files": [], "excludes": []}, - "resources/templates": {"files": ["setupEnv"], "excludes": []}, - "setupEnv": {"files": [], "excludes": []}, - "g2BuildVersion.json": {"files": [], "excludes": []}, -} - -V3_REMOVE_FROM_PROJ = { - "bin": {"files": ["*"], "excludes": ["bin"]}, - "data": {"files": [], "excludes": []}, - "etc": {"files": ["senzing_governor.py"], "excludes": []}, - "lib": { - "files": ["*"], - # TODO - Others simplified? - # "files": [ - # "g2.jar", - # "libG2.so", - # "libG2Hasher.so", - # "libG2SSAdm.so", - # "libg2CompJavaScoreSet.so", - # "libg2DistinctFeatJava.so", - # "libg2EFeatJava.so", - # "libg2JVMPlugin.so", - # "libg2StdJava.so", - # "libmariadbplugin.so", - # "libSpaceTimeBoxStandardizer.so", - # ], - "excludes": ["lib"], - }, - "python": {"files": ["*"], "excludes": []}, - "resources/config": {"files": ["g2core-configuration-upgrade-*.gtc"], "excludes": []}, - "resources/schema": {"files": ["g2core-schema-*-create.sql", "g2core-schema-*-upgrade-*.sql"], "excludes": []}, - "resources/templates": { - "files": [ - "G2C.db*", - "G2Module.ini", - "custom*.txt", - "defaultGNRCP.config", - "g2config.json", - "senzing_governor.py", - "setupEnv", - "stb.config", - ], - "excludes": [], - }, - "sdk": {"files": ["*"], "excludes": ["sdk"]}, - "setupEnv": {"files": [], "excludes": []}, -} - -# COPY_TO_PROJ = { -# "LICENSE": {"files": [], "excludes": []}, -# "NOTICES": {"files": [], "excludes": []}, -# "README.1ST": {"files": [], "excludes": []}, -# "bin": {"files": ["*"], "excludes": ["sz_create_project", "sz_update_project"]}, -# # TODO - -# "/opt/senzing/data": {"files": ["*"], "excludes": []}, -# "lib": {"files": ["*"], "excludes": []}, -# "resources": {"files": ["*"], "excludes": []}, -# "sdk": {"files": ["*"], "excludes": []}, -# "szBuildVersion.json": {"files": [], "excludes": []}, -# } -# TODO - Change files to objects? - -# TODO - diff between dir and dir/ -COPY_TO_PROJ = { - # V4_SYS_PATH: {"files": ["*"], "excludes": ["sz_create_project", "sz_update_project"]}, - "er/": {"files": ["*"], "excludes": ["sz_create_project", "sz_update_project"]}, - "data": {"files": ["*"], "excludes": []}, - # TODO - - # V4_DATA_PATH: {"files": ["data"], "excludes": []}, - # "LICENSE": {"files": [], "excludes": []}, - # "NOTICES": {"files": [], "excludes": []}, - # "README.1ST": {"files": [], "excludes": []}, - # "bin": {"files": ["*"], "excludes": ["sz_create_project", "sz_update_project"]}, - # # TODO - - # "/opt/senzing/data": {"files": ["*"], "excludes": []}, - # "lib": {"files": ["*"], "excludes": []}, - # "resources": {"files": ["*"], "excludes": []}, - # "sdk": {"files": ["*"], "excludes": []}, - # "szBuildVersion.json": {"files": [], "excludes": []}, -} -COPY_TO_PROJ_2 = { - # V4_DATA_PATH: {"files": ["data"], "excludes": []}, - V4_DATA_PATH: {"files": ["*"], "excludes": []}, -} - -V3_RENAME_IN_PROJ = { - "etc": {"from": "G2Module.ini", "to": "sz_engine_config.ini"}, -} - -# TODO - Rename and check -SET_PERMISSIONS = { - ".": { - "dir_pint": 0, - "file_pint": 0o660, - "files": ["LICENSE", "NOTICES", "README.1ST", "szBuildVersion.json"], - "excludes": ["setupEnv"], - "recursive": False, - }, - "setupEnv": {"dir_pint": 0, "file_pint": 0o770, "files": [], "excludes": [], "recursive": False}, - "bin": { - "dir_pint": 0o770, - "file_pint": 0o770, - "files": ["*"], - "excludes": ["__pycache__", "_sz_database.py", "_tool_helpers.py"], - "recursive": False, - }, - "data": {"dir_pint": 0o770, "file_pint": 0o660, "files": ["*"], "excludes": [], "recursive": True}, - "lib": {"dir_pint": 0o770, "file_pint": 0o660, "files": ["*"], "excludes": [], "recursive": False}, - "resources": {"dir_pint": 0o770, "file_pint": 0o660, "files": ["*"], "excludes": ["setupEnv"], "recursive": True}, - "resources/templates/setupEnv": { - "dir_pint": 0, - "file_pint": 0o770, - "files": [], - "excludes": [], - "recursive": False, - }, - "sdk": {"dir_pint": 0o770, "file_pint": 0o660, "files": ["*"], "excludes": [], "recursive": True}, - "v3_upgrade_backups": {"dir_pint": 0o770, "file_pint": 0, "files": [], "excludes": [], "recursive": False}, -} - - -@dataclass() -class SzBuildDetails: - """Build information for a project or Senzing SDK system install""" - - platform: str - version: str - build_version: str - build_number: str - major: int = field(init=False) - minor: int = field(init=False) - micro: int = field(init=False) - - def __post_init__(self) -> None: - self.version_parsed = p_version.parse(self.version) - self.build_version_parsed = p_version.parse(self.build_version) - self.major = self.version_parsed.major - self.minor = self.version_parsed.minor - self.micro = self.version_parsed.micro - - -# pylint: disable=W0106 - - -def parse_cli_args() -> argparse.Namespace: - """Parse the CLI arguments""" - arg_parser = argparse.ArgumentParser( - allow_abbrev=False, - description="Update an existing V3 or V4 Senzing project to the system installed V4 of Senzing.", - ) - arg_parser.add_argument( - "project_path", - metavar="path", - help="Path of the project to update", - ) - arg_parser.add_argument( - "-f", - "--force", - dest="force_mode", - default=False, - action="store_true", - help="Upgrade without prompts. WARNING: Use with caution, ensure you have a backup of the project", - ) - return arg_parser.parse_args() - - -def dir_listing(path: Path) -> list[Path]: - """Get a listing of the specified path to check for files or directories""" - try: - listing = list(path.iterdir()) - except OSError as err: - if type(err).__name__ == "NotADirectoryError": - print(f"\nERROR: {path} is not a directory, expecting one") - print(f"\nERROR: {err}") - sys.exit(1) - - return listing - - -def pre_check(project_path: Path) -> tuple[SzBuildDetails, SzBuildDetails]: - """Check not trying to overwrite the V4 Senzing system install, that path is a project, and versions are correct""" - if not V4_SYS_PATH.exists(): - print(f"\nERROR: Couldn't locate Senzing SDK V4 system install at {V4_SYS_PATH}") - sys.exit(1) - - # TODO - - # if str(project_path).startswith(str(V4_SYS_PATH.parent)): - if str(project_path).startswith(str(SZ_SYS_PATH)): - print(f"\nERROR: {project_path} is the Senzing system installation, not a Senzing project") - sys.exit(1) - - v3_project_build_file = project_path / V3_BUILD - v4_project_build_file = project_path / V4_BUILD - proj_listing = dir_listing(project_path) - if v3_project_build_file not in proj_listing and v4_project_build_file not in proj_listing: - print(f"\nERROR: {project_path} isn't a Senzing project, expected it to contain either:") - print(f"\tExisting V3 project: {v3_project_build_file}") - print(f"\tExisting V4 project: {v4_project_build_file}") - sys.exit(1) - - try: - proj_build_file = v3_project_build_file if v3_project_build_file in proj_listing else v4_project_build_file - proj_build_details = get_build_details(proj_build_file) - sys_build_details = get_build_details(V4_SYS_BUILD) - except OSError as err: - print(f"\nERROR: Trying to read {proj_build_file} or {V4_SYS_BUILD} to collect version information: {err}") - sys.exit(1) - - if proj_build_details.major not in (3, 4) or sys_build_details.major != 4: - print(f"\nERROR: {MODULE_NAME} updates a V3 project to V4, or V4.n.n to a newer release") - print(f"\tProject version: {proj_build_details.version}") - print(f"\tSystem install version: {sys_build_details.version}") - sys.exit(1) - - # TODO - - if sys_build_details.version_parsed < proj_build_details.version_parsed: - print("\nNo update required, project is newer than the system install") - print(f"\tProject version: {proj_build_details.version}") - print(f"\tSystem install version: {sys_build_details.version}") - sys.exit(0) - - # TODO - - if sys_build_details.version_parsed == proj_build_details.version_parsed: - print(f"\nNo update required, project and system install are the same version - {proj_build_details.version}") - sys.exit(0) - - return (proj_build_details, sys_build_details) - - -# TODO - Move to helpers -def get_build_details(path: Path) -> SzBuildDetails: - """Return dataclass with the details from a build file.""" - try: - with open(path, "r", encoding="utf-8") as f: - # Ignore DATA_VERSION V3 build files had it, V4 doesn't - version_dict = {k.lower(): v for k, v in json.load(f).items() if k != "DATA_VERSION"} - return SzBuildDetails(**version_dict) - except (OSError, json.JSONDecodeError) as err: - print(f"\nERROR: Couldn't get the build information from {path}: {err}") - sys.exit(1) - except TypeError as err: - print(f"\nERROR: When creating build information from {path}: {err}") - sys.exit(1) - - -def remove_dir(dir_: Path, excludes: list[Path]) -> None: - """Recursively remove a directory""" - try: - for path in dir_.iterdir(): - if path in excludes: - continue - path.unlink(missing_ok=True) if (path.is_file() or path.is_symlink()) else remove_dir(path, excludes) - - if not excludes: - with suppress(FileNotFoundError): - dir_.rmdir() - except OSError as err: - raise err - - -def remove_files_dirs(to_remove: dict[str, Any], target_dir: Path) -> None: - """Remove files/directories that are no longer required""" - for r_path, r_dict in to_remove.items(): - target: Path = target_dir / r_path - files = r_dict["files"] - excludes = [target / e for e in r_dict["excludes"]] - - try: - if target.is_file() or target.is_symlink(): - target.unlink(missing_ok=True) - - if target.is_dir() and not files or (files and files[0] == "*"): - remove_dir(target, excludes) - - if target.is_dir() and files and files[0] != "*": - target_files = [] - for f in files: - target_files.append(target / f) if "*" not in f else target_files.extend(list(target.glob(f))) - - for target_file in target_files: - target_file.unlink(missing_ok=True) - except OSError as err: - raise OSError(f"ERROR: Couldn't delete a file or directory: {err}") from err - - -def copy_files_dirs(to_copy: dict[str, Any], source_dir: Path, target_dir: Path) -> None: - """Copy files/directories within and to a project""" - # TODO - - print(f"\n{to_copy = }", flush=True) - print(f"{source_dir = }", flush=True) - print(f"{target_dir = }", flush=True) - for c_path, c_dict in to_copy.items(): - print(f"{Path(c_path).exists() = }", flush=True) - print(f"\n{c_path = } - {c_dict} ", flush=True) - excludes = c_dict["excludes"] - files = c_dict["files"] - - source: Path = source_dir / c_path - # TODO - - # target: Path = target_dir / c_path - target = target_dir if c_path.endswith("/") else target_dir / c_path - - # target: Path = target_dir - # if Path(c_path).exists(): - # source = Path(c_path) - # target = Path - - print(f"\n{excludes = }", flush=True) - print(f"{files = }", flush=True) - print(f"{source = }", flush=True) - print(f"{target = }", flush=True) - - # if c_path.startswith("/"): - # print("\nStarts with/...", flush=True) - # source = Path(c_path) - # try: - # print("here", flush=True) - # print(f"\tTrying relative to: {target_dir = } - {source.relative_to(V4_SYS_PATH) = }", flush=True) - # target = target_dir / source.relative_to(V4_SYS_PATH) - # except ValueError: - # print(f"\tTrying target.name: {target = } - {target_dir / target.name}", flush=True) - # target = target_dir / target.name - # print(f"\tNow: {source = }", flush=True) - # print(f"\tNow: {target = }", flush=True) - - try: - if source.is_file(): - # TODO - remove shutil - shutil.copy( - source, - target, - ) - - # # TODO - # if source_dir.is_dir() and files and source_dir.name == files[0]: - # print("Yes in...", flush=True) - # source = source_dir - # target = target_dir / source_dir.name - # print(f"\n{source = }", flush=True) - # print(f"\n{target = }", flush=True) - # target.mkdir(exist_ok=True, parents=True) - # shutil.copytree(source, target, ignore=shutil.ignore_patterns(*excludes), dirs_exist_ok=True) - - # Copy entire contents of the source directory, but not the directory - if source.is_dir() and not files or (files and files[0] == "*"): - print("Yes in 2...", flush=True) - shutil.copytree(source, target, ignore=shutil.ignore_patterns(*excludes), dirs_exist_ok=True) - - # Create the source directory in the target and only copy listed files - if source.is_dir() and files and files[0] != "*": - print("Yes in 3...", flush=True) - target.mkdir(exist_ok=True, parents=True) - for source_file in [source / f for f in files]: - shutil.copy(source_file, target / source_file.name) - except OSError as err: - raise OSError(f"ERROR: Couldn't copy a file or directory: {err}") from err - - -def rename_files(to_rename: dict[str, dict[str, str]], target_dir: Path) -> None: - """Rename existing project files that had a name change""" - - try: - for r_path, r_dict in to_rename.items(): - current = target_dir / r_path / r_dict["from"] - new = target_dir / r_path / r_dict["to"] - with suppress(FileNotFoundError): - current.rename(new) - except OSError as err: - raise OSError(f"ERROR: Couldn't rename a file or directory: {err}") from err - - -def setup_env(proj_path: Path) -> None: - """Create a new setupEnv and replace place holders with paths for the project""" - try: - shutil.copy(proj_path / "resources/templates/setupEnv", proj_path) - setup_path = proj_path / "setupEnv" - - with open(setup_path, "r", encoding="utf-8") as in_: - data = in_.read() - - data = data.replace("${SENZING_DIR}", str(proj_path)).replace("${SENZING_CONFIG_PATH}", str(proj_path / "etc")) - - with open(setup_path, "w", encoding="utf-8") as out: - out.write(data) - except OSError as err: - raise OSError(f"ERROR: Couldn't create a new setupEnv file: {err}") from err - - -def set_permissions(proj_path: Path, permissions: dict[str, dict[str, Any]]) -> None: - """Set permissions for files/dirs copied to the project, or dirs removed and replaced completely e.g., data/""" - try: - for p_path, p_dict in permissions.items(): - dir_pint = p_dict["dir_pint"] - file_pint = p_dict["file_pint"] - files = p_dict["files"] - recursive = p_dict["recursive"] - target = proj_path if p_path.startswith(".") else proj_path / p_path - excludes = [target / e for e in p_dict["excludes"]] - - if target.is_file(): - target.chmod(file_pint) - - if target.is_dir() and dir_pint != 0: - target.chmod(dir_pint) - d_chmods = ( - [d for d in target.glob("*") if d.is_dir() and not d.is_symlink() and d not in excludes] - if not recursive - else [d for d in target.rglob("*") if d.is_dir() and not d.is_symlink() and d not in excludes] - ) - for dir_ in d_chmods: - dir_.chmod(dir_pint) - - if target.is_dir() and (files and files[0] == "*"): - f_chmods = ( - [f for f in target.glob("*") if f.is_file() and not f.is_symlink() and f not in excludes] - if not recursive - else [f for f in target.rglob("*") if f.is_file() and not f.is_symlink() and f not in excludes] - ) - for file in f_chmods: - Path(target / file).chmod(file_pint) - - if target.is_dir() and files and files[0] != "*": - for file in files: - Path(target / file).chmod(file_pint) - except OSError as err: - raise OSError(f"ERROR: Couldn't set a permission: {err}") from err - - -def main() -> None: - """main""" - cli_args = parse_cli_args() - # TODO - expanduser - proj_path = Path(cli_args.project_path).resolve() - proj_build, sys_build = pre_check(proj_path) - proj_is_v3 = bool(proj_build.major == 3) - - if not cli_args.force_mode: - print("\nWARNING: If you don't have a backup of the project, create one before continuing!") - sleep(3) - if ( - input(f"\nUpdate the project from version {proj_build.build_version} to {sys_build.build_version}? (y/n) ") - not in INPUT_CONFS - ): - sys.exit(0) - print("\nUpdating...") - else: - print(f"\nUpdating project from version {proj_build.build_version} to {sys_build.build_version}...") - - try: - if proj_is_v3: - v3_backup_path = proj_path / V3_BACKUP_PATH - v3_backup_path.mkdir(exist_ok=True) - copy_files_dirs(V3_BACKUP_PROJ, proj_path, v3_backup_path) # Backup some of the V3 project files - rename_files(V3_RENAME_IN_PROJ, proj_path) - remove_files_dirs(V3_REMOVE_FROM_PROJ, proj_path) - # TODO - - # input("...") - # copy_files_dirs(COPY_TO_PROJ, V4_SYS_PATH, proj_path) - copy_files_dirs(COPY_TO_PROJ, SZ_SYS_PATH, proj_path) - if proj_is_v3: - setup_env(proj_path) - # TODO - Check V4 permissions are good, is sz_create_project good in the first place when V4 to V4? - set_permissions(proj_path, SET_PERMISSIONS) - except OSError as err: - shutil.copytree(v3_backup_path, proj_path, dirs_exist_ok=True) - print(f"\n{err}") - print("\nIf the error is file or directory permission related, run again with appropriate privileges") - print( - "\nIf the error is missing file or directory, check senzingsdk-setup, senzingsdk-tools, and senzingsdk-poc are installed" - ) - else: - if proj_is_v3: # Only delete if no errors so re-running can find the file again - proj_path.joinpath(V3_BUILD).unlink(missing_ok=True) - - print(f"\nProject successfully updated. Refer to {UPGRADE_URL} for additional upgrade instructions") - - -if __name__ == "__main__": - main() diff --git a/sz_tools/sz_update_project_prior6 b/sz_tools/sz_update_project_prior6 deleted file mode 100755 index f011fdd..0000000 --- a/sz_tools/sz_update_project_prior6 +++ /dev/null @@ -1,465 +0,0 @@ -#! /usr/bin/env python3 -"""Upgrade a V3 or V4 Senzing SDK project to V4.x.x""" - -import argparse -import json -import shutil -import sys -from contextlib import suppress -from dataclasses import dataclass, field -from pathlib import Path -from time import sleep -from typing import Any - -from packaging import version as p_version - -INPUT_CONFS = ("y", "Y", "yEs", "yES", "YES", "yes", "YEs", "yeS", "Yes", "YeS") -MODULE_NAME = Path(__file__).stem -V3_BACKUP_PATH = "v3_upgrade_backups" -V3_BUILD = "g2BuildVersion.json" -V4_BUILD = "szBuildVersion.json" -SZ_SYS_PATH = Path("/opt/senzing") -V3_SYS_PATH = SZ_SYS_PATH / "g2" -V4_SYS_PATH = SZ_SYS_PATH / "er" -V4_DATA_PATH = SZ_SYS_PATH / "data" -V3_SYS_BUILD = V3_SYS_PATH / V3_BUILD -V4_SYS_BUILD = V4_SYS_PATH / V4_BUILD -UPGRADE_URL = "https://www.senzing.com/docs/4_beta/index.html" - -# TODO - files -> objects? -# TODO - "*" -V3_BACKUP_PROJ = { - "bin": {"files": ["g2dbencrypt", "g2saltadm", "g2ssadm"], "excludes": []}, - "etc": {"files": ["senzing_governor.py"], "excludes": []}, - "lib": {"files": ["libSpaceTimeBoxStandardizer.so", "libG2Hasher.so", "libG2SSAdm.so"], "excludes": []}, - "python": {"files": [], "excludes": []}, - "resources/templates": {"files": ["setupEnv"], "excludes": []}, - "setupEnv": {"files": [], "excludes": []}, - "g2BuildVersion.json": {"files": [], "excludes": []}, -} - -V3_REMOVE_FROM_PROJ = { - "bin": {"files": ["*"], "excludes": ["bin"]}, - "data": {"files": [], "excludes": []}, - "etc": {"files": ["senzing_governor.py"], "excludes": []}, - "lib": { - "files": ["*"], - # TODO - Others simplified? - # "files": [ - # "g2.jar", - # "libG2.so", - # "libG2Hasher.so", - # "libG2SSAdm.so", - # "libg2CompJavaScoreSet.so", - # "libg2DistinctFeatJava.so", - # "libg2EFeatJava.so", - # "libg2JVMPlugin.so", - # "libg2StdJava.so", - # "libmariadbplugin.so", - # "libSpaceTimeBoxStandardizer.so", - # ], - "excludes": ["lib"], - }, - "python": {"files": ["*"], "excludes": []}, - "resources/config": {"files": ["g2core-configuration-upgrade-*.gtc"], "excludes": []}, - "resources/schema": {"files": ["g2core-schema-*-create.sql", "g2core-schema-*-upgrade-*.sql"], "excludes": []}, - "resources/templates": { - "files": [ - "G2C.db*", - "G2Module.ini", - "custom*.txt", - "defaultGNRCP.config", - "g2config.json", - "senzing_governor.py", - "setupEnv", - "stb.config", - ], - "excludes": [], - }, - "sdk": {"files": ["*"], "excludes": ["sdk"]}, - "setupEnv": {"files": [], "excludes": []}, -} - -# TODO - diff between dir and dir/ -# er/ copies everything in /opt/senzing/er into the target (minus excludes) -# data copies everything in /opt/senzing/er into the target -COPY_TO_PROJ = { - "er/": {"files": ["*"], "excludes": ["sz_create_project", "sz_update_project"]}, - "data": {"files": ["*"], "excludes": []}, - "er/szBuildVersion.json": {"files": ["*"], "excludes": []}, -} - -V3_RENAME_IN_PROJ = { - "etc": {"from": "G2Module.ini", "to": "sz_engine_config.ini"}, -} - -# TODO - Rename and check -SET_PERMISSIONS = { - ".": { - "dir_pint": 0, - "file_pint": 0o660, - "files": ["LICENSE", "NOTICES", "README.1ST", "szBuildVersion.json"], - "excludes": ["setupEnv"], - "recursive": False, - }, - "setupEnv": {"dir_pint": 0, "file_pint": 0o770, "files": [], "excludes": [], "recursive": False}, - "bin": { - "dir_pint": 0o770, - "file_pint": 0o770, - "files": ["*"], - "excludes": ["__pycache__", "_sz_database.py", "_tool_helpers.py"], - "recursive": False, - }, - "data": {"dir_pint": 0o770, "file_pint": 0o660, "files": ["*"], "excludes": [], "recursive": True}, - "lib": {"dir_pint": 0o770, "file_pint": 0o660, "files": ["*"], "excludes": [], "recursive": False}, - "resources": {"dir_pint": 0o770, "file_pint": 0o660, "files": ["*"], "excludes": ["setupEnv"], "recursive": True}, - "resources/templates/setupEnv": { - "dir_pint": 0, - "file_pint": 0o770, - "files": [], - "excludes": [], - "recursive": False, - }, - "sdk": {"dir_pint": 0o770, "file_pint": 0o660, "files": ["*"], "excludes": [], "recursive": True}, - "v3_upgrade_backups": {"dir_pint": 0o770, "file_pint": 0, "files": [], "excludes": [], "recursive": False}, -} - - -@dataclass() -class SzBuildDetails: - """Build information for a project or Senzing SDK system install""" - - platform: str - version: str - build_version: str - build_number: str - major: int = field(init=False) - minor: int = field(init=False) - micro: int = field(init=False) - - def __post_init__(self) -> None: - self.version_parsed = p_version.parse(self.version) - self.build_version_parsed = p_version.parse(self.build_version) - self.major = self.version_parsed.major - self.minor = self.version_parsed.minor - self.micro = self.version_parsed.micro - - -# pylint: disable=W0106 - - -def parse_cli_args() -> argparse.Namespace: - """Parse the CLI arguments""" - arg_parser = argparse.ArgumentParser( - allow_abbrev=False, - description="Update an existing V3 or V4 Senzing project to the system installed V4 of Senzing.", - ) - arg_parser.add_argument( - "project_path", - metavar="path", - help="Path of the project to update", - ) - arg_parser.add_argument( - "-f", - "--force", - dest="force_mode", - default=False, - action="store_true", - help="Upgrade without prompts. WARNING: Use with caution, ensure you have a backup of the project", - ) - return arg_parser.parse_args() - - -def dir_listing(path: Path) -> list[Path]: - """Get a listing of the specified path to check for files or directories""" - try: - listing = list(path.iterdir()) - except OSError as err: - if type(err).__name__ == "NotADirectoryError": - print(f"\nERROR: {path} is not a directory, expecting one") - print(f"\nERROR: {err}") - sys.exit(1) - - return listing - - -def pre_check(project_path: Path) -> tuple[SzBuildDetails, SzBuildDetails]: - """Check not trying to overwrite the V4 Senzing system install, that path is a project, and versions are correct""" - if not V4_SYS_PATH.exists(): - print(f"\nERROR: Couldn't locate Senzing SDK V4 system install at {V4_SYS_PATH}") - sys.exit(1) - - if str(project_path).startswith(str(SZ_SYS_PATH)): - print(f"\nERROR: {project_path} is the Senzing system installation, not a Senzing project") - sys.exit(1) - - v3_project_build_file = project_path / V3_BUILD - v4_project_build_file = project_path / V4_BUILD - proj_listing = dir_listing(project_path) - if v3_project_build_file not in proj_listing and v4_project_build_file not in proj_listing: - print(f"\nERROR: {project_path} isn't a Senzing project, expected it to contain either:") - print(f"\tExisting V3 project: {v3_project_build_file}") - print(f"\tExisting V4 project: {v4_project_build_file}") - sys.exit(1) - - try: - proj_build_file = v3_project_build_file if v3_project_build_file in proj_listing else v4_project_build_file - proj_build_details = get_build_details(proj_build_file) - sys_build_details = get_build_details(V4_SYS_BUILD) - except OSError as err: - print(f"\nERROR: Trying to read {proj_build_file} or {V4_SYS_BUILD} to collect version information: {err}") - sys.exit(1) - - if proj_build_details.major not in (3, 4) or sys_build_details.major != 4: - print(f"\nERROR: {MODULE_NAME} updates a V3 project to V4, or V4.n.n to a newer release") - print(f"\tProject version: {proj_build_details.version}") - print(f"\tSystem install version: {sys_build_details.version}") - sys.exit(1) - - # TODO - - if sys_build_details.version_parsed < proj_build_details.version_parsed: - print("\nNo update required, project is newer than the system install") - print(f"\tProject version: {proj_build_details.version}") - print(f"\tSystem install version: {sys_build_details.version}") - sys.exit(0) - - # TODO - - if sys_build_details.version_parsed == proj_build_details.version_parsed: - print(f"\nNo update required, project and system install are the same version - {proj_build_details.version}") - sys.exit(0) - - return (proj_build_details, sys_build_details) - - -# TODO - Move to helpers -def get_build_details(path: Path) -> SzBuildDetails: - """Return dataclass with the details from a build file.""" - try: - with open(path, "r", encoding="utf-8") as f: - # Ignore DATA_VERSION, V3 build files had it V4 doesn't - version_dict = {k.lower(): v for k, v in json.load(f).items() if k != "DATA_VERSION"} - return SzBuildDetails(**version_dict) - except (OSError, json.JSONDecodeError) as err: - print(f"\nERROR: Couldn't get the build information from {path}: {err}") - sys.exit(1) - except TypeError as err: - print(f"\nERROR: When creating build information from {path}: {err}") - sys.exit(1) - - -def remove_dir(dir_: Path, excludes: list[Path]) -> None: - """Recursively remove a directory""" - try: - for path in dir_.iterdir(): - if path in excludes: - continue - path.unlink(missing_ok=True) if (path.is_file() or path.is_symlink()) else remove_dir(path, excludes) - - if not excludes: - with suppress(FileNotFoundError): - dir_.rmdir() - except OSError as err: - raise err - - -def remove_files_dirs(to_remove: dict[str, Any], target_dir: Path) -> None: - """Remove files/directories that are no longer required""" - for r_path, r_dict in to_remove.items(): - target: Path = target_dir / r_path - files = r_dict["files"] - excludes = [target / e for e in r_dict["excludes"]] - - try: - if target.is_file() or target.is_symlink(): - target.unlink(missing_ok=True) - - if target.is_dir() and not files or (files and files[0] == "*"): - remove_dir(target, excludes) - - if target.is_dir() and files and files[0] != "*": - target_files = [] - for f in files: - target_files.append(target / f) if "*" not in f else target_files.extend(list(target.glob(f))) - - for target_file in target_files: - target_file.unlink(missing_ok=True) - except OSError as err: - raise OSError(f"ERROR: Couldn't delete a file or directory: {err}") from err - - -def copy_files_dirs(to_copy: dict[str, Any], source_dir: Path, target_dir: Path) -> None: - """Copy files/directories within and to a project""" - print(f"\n{to_copy = }", flush=True) - print(f"{source_dir = }", flush=True) - print(f"{target_dir = }", flush=True) - for c_path, c_dict in to_copy.items(): - excludes = c_dict["excludes"] - files = c_dict["files"] - source = source_dir / c_path - # TODO - - target = target_dir - - # TODO - - # TODO - diff between dir and dir/ - # If the key in to_copy - # er/ copies everything in /opt/senzing/er into the target (minus excludes) - # data copies everything in /opt/senzing/er into the target - # target = target_dir if c_path.endswith("/") else target_dir / c_path - - if source.is_dir(): - target = target_dir if c_path.endswith("/") else target_dir / c_path - - if source.is_file(): - target = target_dir / source.name - - print("\n\n---------------------------------------------------") - print(f"{c_path = }", flush=True) - print(f"{source = }", flush=True) - print(f"{target = }", flush=True) - - try: - if source.is_file(): - # TODO - remove shutil - print(f"\nYes file: {source = }", flush=True) - # TODO - - - shutil.copy( - source, - target, - ) - - # Copy entire contents of the source directory - # if source.is_dir() and not files or (files and files[0] == "*"): - if source.is_dir() and (not files or (files and files[0] == "*")): - print("\nhere1...") - shutil.copytree(source, target, ignore=shutil.ignore_patterns(*excludes), dirs_exist_ok=True) - - # Create the source directory in the target and only copy listed files - # if source.is_dir() and files and files[0] != "*": - if source.is_dir() and (files and files[0] != "*"): - print("\nhere2...") - target.mkdir(exist_ok=True, parents=True) - for source_file in [source / f for f in files]: - shutil.copy(source_file, target / source_file.name) - except OSError as err: - raise OSError(f"ERROR: Couldn't copy a file or directory: {err}") from err - - -def rename_files(to_rename: dict[str, dict[str, str]], target_dir: Path) -> None: - """Rename existing project files that had a name change""" - - try: - for r_path, r_dict in to_rename.items(): - current = target_dir / r_path / r_dict["from"] - new = target_dir / r_path / r_dict["to"] - with suppress(FileNotFoundError): - current.rename(new) - except OSError as err: - raise OSError(f"ERROR: Couldn't rename a file or directory: {err}") from err - - -def setup_env(proj_path: Path) -> None: - """Create a new setupEnv and replace place holders with paths for the project""" - try: - shutil.copy(proj_path / "resources/templates/setupEnv", proj_path) - setup_path = proj_path / "setupEnv" - - with open(setup_path, "r", encoding="utf-8") as in_: - data = in_.read() - - data = data.replace("${SENZING_DIR}", str(proj_path)).replace("${SENZING_CONFIG_PATH}", str(proj_path / "etc")) - - with open(setup_path, "w", encoding="utf-8") as out: - out.write(data) - except OSError as err: - raise OSError(f"ERROR: Couldn't create a new setupEnv file: {err}") from err - - -def set_permissions(proj_path: Path, permissions: dict[str, dict[str, Any]]) -> None: - """Set permissions for files/dirs copied to the project, or dirs removed and replaced completely e.g., data/""" - try: - for p_path, p_dict in permissions.items(): - dir_pint = p_dict["dir_pint"] - file_pint = p_dict["file_pint"] - files = p_dict["files"] - recursive = p_dict["recursive"] - target = proj_path if p_path.startswith(".") else proj_path / p_path - excludes = [target / e for e in p_dict["excludes"]] - - if target.is_file(): - target.chmod(file_pint) - - if target.is_dir() and dir_pint != 0: - target.chmod(dir_pint) - d_chmods = ( - [d for d in target.glob("*") if d.is_dir() and not d.is_symlink() and d not in excludes] - if not recursive - else [d for d in target.rglob("*") if d.is_dir() and not d.is_symlink() and d not in excludes] - ) - for dir_ in d_chmods: - dir_.chmod(dir_pint) - - if target.is_dir() and (files and files[0] == "*"): - f_chmods = ( - [f for f in target.glob("*") if f.is_file() and not f.is_symlink() and f not in excludes] - if not recursive - else [f for f in target.rglob("*") if f.is_file() and not f.is_symlink() and f not in excludes] - ) - for file in f_chmods: - Path(target / file).chmod(file_pint) - - if target.is_dir() and files and files[0] != "*": - for file in files: - Path(target / file).chmod(file_pint) - except OSError as err: - raise OSError(f"ERROR: Couldn't set a permission: {err}") from err - - -def main() -> None: - """main""" - cli_args = parse_cli_args() - # TODO - expanduser - proj_path = Path(cli_args.project_path).resolve() - proj_build, sys_build = pre_check(proj_path) - proj_is_v3 = bool(proj_build.major == 3) - - if not cli_args.force_mode: - print("\nWARNING: If you don't have a backup of the project, create one before continuing!") - sleep(3) - if ( - input(f"\nUpdate the project from version {proj_build.build_version} to {sys_build.build_version}? (y/n) ") - not in INPUT_CONFS - ): - sys.exit(0) - print("\nUpdating...") - else: - print(f"\nUpdating project from version {proj_build.build_version} to {sys_build.build_version}...") - - try: - if proj_is_v3: - v3_backup_path = proj_path / V3_BACKUP_PATH - v3_backup_path.mkdir(exist_ok=True) - copy_files_dirs(V3_BACKUP_PROJ, proj_path, v3_backup_path) # Backup some of the V3 project files - rename_files(V3_RENAME_IN_PROJ, proj_path) - remove_files_dirs(V3_REMOVE_FROM_PROJ, proj_path) - copy_files_dirs(COPY_TO_PROJ, SZ_SYS_PATH, proj_path) - if proj_is_v3: - setup_env(proj_path) - # TODO - Check V4 permissions are good, is sz_create_project good in the first place when V4 to V4? - set_permissions(proj_path, SET_PERMISSIONS) - except OSError as err: - shutil.copytree(v3_backup_path, proj_path, dirs_exist_ok=True) - print(f"\n{err}") - print("\nIf the error is file or directory permission related, run again with appropriate privileges") - print( - "\nIf the error is missing file or directory, check senzingsdk-setup, senzingsdk-tools, and senzingsdk-poc are installed" - ) - else: - if proj_is_v3: # Only delete if no errors so re-running can find the file again - proj_path.joinpath(V3_BUILD).unlink(missing_ok=True) - - print(f"\nProject successfully updated. Refer to {UPGRADE_URL} for additional upgrade instructions") - - -if __name__ == "__main__": - main() diff --git a/sz_tools/sz_update_project_prior7 b/sz_tools/sz_update_project_prior7 deleted file mode 100755 index edb9274..0000000 --- a/sz_tools/sz_update_project_prior7 +++ /dev/null @@ -1,480 +0,0 @@ -#! /usr/bin/env python3 -"""Upgrade a V3 or V4 Senzing SDK project to V4.x.x""" - -import argparse -import json -import shutil -import sys -from contextlib import suppress -from dataclasses import dataclass, field -from pathlib import Path -from time import sleep -from typing import Any - -from packaging import version as p_version - -INPUT_CONFS = ("y", "Y", "yEs", "yES", "YES", "yes", "YEs", "yeS", "Yes", "YeS") -MODULE_NAME = Path(__file__).stem -V3_BACKUP_PATH = "v3_upgrade_backups" -V3_BUILD = "g2BuildVersion.json" -V4_BUILD = "szBuildVersion.json" -SZ_SYS_PATH = Path("/opt/senzing") -V3_SYS_PATH = SZ_SYS_PATH / "g2" -V4_SYS_PATH = SZ_SYS_PATH / "er" -V4_DATA_PATH = SZ_SYS_PATH / "data" -V3_SYS_BUILD = V3_SYS_PATH / V3_BUILD -V4_SYS_BUILD = V4_SYS_PATH / V4_BUILD -UPGRADE_URL = "https://www.senzing.com/docs/4_beta/index.html" - -# TODO - files -> objects? -# TODO - "*" -V3_BACKUP_PROJ = { - "bin": {"files": ["g2dbencrypt", "g2saltadm", "g2ssadm"], "excludes": []}, - "etc": {"files": ["senzing_governor.py"], "excludes": []}, - "lib": {"files": ["libSpaceTimeBoxStandardizer.so", "libG2Hasher.so", "libG2SSAdm.so"], "excludes": []}, - "python": {"files": [], "excludes": []}, - "resources/templates": {"files": ["setupEnv"], "excludes": []}, - "setupEnv": {"files": [], "excludes": []}, - "g2BuildVersion.json": {"files": [], "excludes": []}, -} - -V3_REMOVE_FROM_PROJ = { - "bin": {"files": ["*"], "excludes": ["bin"]}, - "data": {"files": [], "excludes": []}, - "etc": {"files": ["senzing_governor.py"], "excludes": []}, - "lib": { - "files": ["*"], - # TODO - Others simplified? - # "files": [ - # "g2.jar", - # "libG2.so", - # "libG2Hasher.so", - # "libG2SSAdm.so", - # "libg2CompJavaScoreSet.so", - # "libg2DistinctFeatJava.so", - # "libg2EFeatJava.so", - # "libg2JVMPlugin.so", - # "libg2StdJava.so", - # "libmariadbplugin.so", - # "libSpaceTimeBoxStandardizer.so", - # ], - "excludes": ["lib"], - }, - "python": {"files": ["*"], "excludes": []}, - "resources/config": {"files": ["g2core-configuration-upgrade-*.gtc"], "excludes": []}, - "resources/schema": {"files": ["g2core-schema-*-create.sql", "g2core-schema-*-upgrade-*.sql"], "excludes": []}, - "resources/templates": { - "files": [ - "G2C.db*", - "G2Module.ini", - "custom*.txt", - "defaultGNRCP.config", - "g2config.json", - "senzing_governor.py", - "setupEnv", - "stb.config", - ], - "excludes": [], - }, - "sdk": {"files": ["*"], "excludes": ["sdk"]}, - "setupEnv": {"files": [], "excludes": []}, -} - -# TODO - diff between dir and dir/ -# er/ copies everything in /opt/senzing/er into the target (minus excludes) -# data copies everything in /opt/senzing/er into the target -COPY_TO_PROJ = { - "er/": {"files": ["*"], "excludes": ["sz_create_project", "sz_update_project"]}, - "data": {"files": ["*"], "excludes": []}, - "er/szBuildVersion.json": {"files": ["*"], "excludes": []}, -} - -V3_RENAME_IN_PROJ = { - "etc": {"from": "G2Module.ini", "to": "sz_engine_config.ini"}, -} - -# TODO - Rename and check -SET_PERMISSIONS = { - ".": { - "dir_pint": 0, - "file_pint": 0o660, - "files": ["LICENSE", "NOTICES", "README.1ST", "szBuildVersion.json"], - "excludes": ["setupEnv"], - "recursive": False, - }, - "setupEnv": {"dir_pint": 0, "file_pint": 0o770, "files": [], "excludes": [], "recursive": False}, - "bin": { - "dir_pint": 0o770, - "file_pint": 0o770, - "files": ["*"], - "excludes": ["__pycache__", "_sz_database.py", "_tool_helpers.py"], - "recursive": False, - }, - "data": {"dir_pint": 0o770, "file_pint": 0o660, "files": ["*"], "excludes": [], "recursive": True}, - "lib": {"dir_pint": 0o770, "file_pint": 0o660, "files": ["*"], "excludes": [], "recursive": False}, - "resources": {"dir_pint": 0o770, "file_pint": 0o660, "files": ["*"], "excludes": ["setupEnv"], "recursive": True}, - "resources/templates/setupEnv": { - "dir_pint": 0, - "file_pint": 0o770, - "files": [], - "excludes": [], - "recursive": False, - }, - "sdk": {"dir_pint": 0o770, "file_pint": 0o660, "files": ["*"], "excludes": [], "recursive": True}, - "v3_upgrade_backups": {"dir_pint": 0o770, "file_pint": 0, "files": [], "excludes": [], "recursive": False}, -} - - -@dataclass() -class SzBuildDetails: - """Build information for a project or Senzing SDK system install""" - - platform: str - version: str - build_version: str - build_number: str - major: int = field(init=False) - minor: int = field(init=False) - micro: int = field(init=False) - - def __post_init__(self) -> None: - self.version_parsed = p_version.parse(self.version) - self.build_version_parsed = p_version.parse(self.build_version) - self.major = self.version_parsed.major - self.minor = self.version_parsed.minor - self.micro = self.version_parsed.micro - - -# pylint: disable=W0106 - - -def parse_cli_args() -> argparse.Namespace: - """Parse the CLI arguments""" - arg_parser = argparse.ArgumentParser( - allow_abbrev=False, - description="Update an existing V3 or V4 Senzing project to the system installed V4 of Senzing.", - ) - arg_parser.add_argument( - "project_path", - metavar="path", - help="Path of the project to update", - ) - arg_parser.add_argument( - "-f", - "--force", - dest="force_mode", - default=False, - action="store_true", - help="Upgrade without prompts. WARNING: Use with caution, ensure you have a backup of the project", - ) - return arg_parser.parse_args() - - -def dir_listing(path: Path) -> list[Path]: - """Get a listing of the specified path to check for files or directories""" - try: - listing = list(path.iterdir()) - except OSError as err: - if type(err).__name__ == "NotADirectoryError": - print(f"\nERROR: {path} is not a directory, expecting one") - print(f"\nERROR: {err}") - sys.exit(1) - - return listing - - -def pre_check(project_path: Path) -> tuple[SzBuildDetails, SzBuildDetails]: - """Check not trying to overwrite the V4 Senzing system install, that path is a project, and versions are correct""" - if not V4_SYS_PATH.exists(): - print(f"\nERROR: Couldn't locate Senzing SDK V4 system install at {V4_SYS_PATH}") - sys.exit(1) - - if str(project_path).startswith(str(SZ_SYS_PATH)): - print(f"\nERROR: {project_path} is the Senzing system installation, not a Senzing project") - sys.exit(1) - - v3_project_build_file = project_path / V3_BUILD - v4_project_build_file = project_path / V4_BUILD - proj_listing = dir_listing(project_path) - if v3_project_build_file not in proj_listing and v4_project_build_file not in proj_listing: - print(f"\nERROR: {project_path} isn't a Senzing project, expected it to contain either:") - print(f"\tExisting V3 project: {v3_project_build_file}") - print(f"\tExisting V4 project: {v4_project_build_file}") - sys.exit(1) - - try: - proj_build_file = v3_project_build_file if v3_project_build_file in proj_listing else v4_project_build_file - proj_build_details = get_build_details(proj_build_file) - sys_build_details = get_build_details(V4_SYS_BUILD) - except OSError as err: - print(f"\nERROR: Trying to read {proj_build_file} or {V4_SYS_BUILD} to collect version information: {err}") - sys.exit(1) - - if proj_build_details.major not in (3, 4) or sys_build_details.major != 4: - print(f"\nERROR: {MODULE_NAME} updates a V3 project to V4, or V4.n.n to a newer release") - print(f"\tProject version: {proj_build_details.version}") - print(f"\tSystem install version: {sys_build_details.version}") - sys.exit(1) - - # TODO - - if sys_build_details.version_parsed < proj_build_details.version_parsed: - print("\nNo update required, project is newer than the system install") - print(f"\tProject version: {proj_build_details.version}") - print(f"\tSystem install version: {sys_build_details.version}") - sys.exit(0) - - # TODO - - if sys_build_details.version_parsed == proj_build_details.version_parsed: - print(f"\nNo update required, project and system install are the same version - {proj_build_details.version}") - sys.exit(0) - - return (proj_build_details, sys_build_details) - - -# TODO - Move to helpers -def get_build_details(path: Path) -> SzBuildDetails: - """Return dataclass with the details from a build file.""" - try: - with open(path, "r", encoding="utf-8") as f: - # Ignore DATA_VERSION, V3 build files had it V4 doesn't - version_dict = {k.lower(): v for k, v in json.load(f).items() if k != "DATA_VERSION"} - return SzBuildDetails(**version_dict) - except (OSError, json.JSONDecodeError) as err: - print(f"\nERROR: Couldn't get the build information from {path}: {err}") - sys.exit(1) - except TypeError as err: - print(f"\nERROR: When creating build information from {path}: {err}") - sys.exit(1) - - -def remove_dir(dir_: Path, excludes: list[Path]) -> None: - """Recursively remove a directory""" - try: - for path in dir_.iterdir(): - if path in excludes: - continue - path.unlink(missing_ok=True) if (path.is_file() or path.is_symlink()) else remove_dir(path, excludes) - - if not excludes: - with suppress(FileNotFoundError): - dir_.rmdir() - except OSError as err: - raise err - - -def remove_files_dirs(to_remove: dict[str, Any], target_dir: Path) -> None: - """Remove files/directories that are no longer required""" - for r_path, r_dict in to_remove.items(): - target: Path = target_dir / r_path - files = r_dict["files"] - excludes = [target / e for e in r_dict["excludes"]] - - try: - if target.is_file() or target.is_symlink(): - target.unlink(missing_ok=True) - - if target.is_dir() and not files or (files and files[0] == "*"): - remove_dir(target, excludes) - - if target.is_dir() and files and files[0] != "*": - target_files = [] - for f in files: - target_files.append(target / f) if "*" not in f else target_files.extend(list(target.glob(f))) - - for target_file in target_files: - target_file.unlink(missing_ok=True) - except OSError as err: - raise OSError(f"ERROR: Couldn't delete a file or directory: {err}") from err - - -def copy_files_dirs(to_copy: dict[str, Any], source_dir: Path, target_dir: Path) -> None: - """Copy files/directories within and to a project""" - print(f"\n{to_copy = }", flush=True) - print(f"{source_dir = }", flush=True) - print(f"{target_dir = }", flush=True) - for c_path, c_dict in to_copy.items(): - excludes = c_dict["excludes"] - files = c_dict["files"] - source = source_dir / c_path - # TODO - - target = target_dir - - # TODO - - # TODO - diff between dir and dir/ - # If the key in to_copy - # er/ copies everything in /opt/senzing/er into the target (minus excludes) - # data copies everything in /opt/senzing/er into the target - # target = target_dir if c_path.endswith("/") else target_dir / c_path - - try: - if source.is_dir(): - target = target_dir if c_path.endswith("/") else target_dir / c_path - - # Copy entire contents of the source directory - if not files or (files and files[0] == "*"): - print("\nhere1...") - shutil.copytree(source, target, ignore=shutil.ignore_patterns(*excludes), dirs_exist_ok=True) - - # Create the source directory in the target and only copy listed files - if files and files[0] != "*": - print("\nhere2...") - target.mkdir(exist_ok=True, parents=True) - for source_file in [source / f for f in files]: - shutil.copy(source_file, target / source_file.name) - - if source.is_file(): - target = target_dir / source.name - except OSError as err: - raise OSError(f"ERROR: Couldn't copy a file or directory: {err}") from err - - # print("\n\n---------------------------------------------------") - # print(f"{c_path = }", flush=True) - # print(f"{source = }", flush=True) - # print(f"{target = }", flush=True) - - # try: - # if source.is_file(): - # # TODO - remove shutil - # print(f"\nYes file: {source = }", flush=True) - # # TODO - - - # shutil.copy( - # source, - # target, - # ) - - # # Copy entire contents of the source directory - # # if source.is_dir() and not files or (files and files[0] == "*"): - # if source.is_dir() and (not files or (files and files[0] == "*")): - # print("\nhere1...") - # shutil.copytree(source, target, ignore=shutil.ignore_patterns(*excludes), dirs_exist_ok=True) - - # # Create the source directory in the target and only copy listed files - # # if source.is_dir() and files and files[0] != "*": - # if source.is_dir() and (files and files[0] != "*"): - # print("\nhere2...") - # target.mkdir(exist_ok=True, parents=True) - # for source_file in [source / f for f in files]: - # shutil.copy(source_file, target / source_file.name) - # except OSError as err: - # raise OSError(f"ERROR: Couldn't copy a file or directory: {err}") from err - - -def rename_files(to_rename: dict[str, dict[str, str]], target_dir: Path) -> None: - """Rename existing project files that had a name change""" - - try: - for r_path, r_dict in to_rename.items(): - current = target_dir / r_path / r_dict["from"] - new = target_dir / r_path / r_dict["to"] - with suppress(FileNotFoundError): - current.rename(new) - except OSError as err: - raise OSError(f"ERROR: Couldn't rename a file or directory: {err}") from err - - -def setup_env(proj_path: Path) -> None: - """Create a new setupEnv and replace place holders with paths for the project""" - try: - shutil.copy(proj_path / "resources/templates/setupEnv", proj_path) - setup_path = proj_path / "setupEnv" - - with open(setup_path, "r", encoding="utf-8") as in_: - data = in_.read() - - data = data.replace("${SENZING_DIR}", str(proj_path)).replace("${SENZING_CONFIG_PATH}", str(proj_path / "etc")) - - with open(setup_path, "w", encoding="utf-8") as out: - out.write(data) - except OSError as err: - raise OSError(f"ERROR: Couldn't create a new setupEnv file: {err}") from err - - -def set_permissions(proj_path: Path, permissions: dict[str, dict[str, Any]]) -> None: - """Set permissions for files/dirs copied to the project, or dirs removed and replaced completely e.g., data/""" - try: - for p_path, p_dict in permissions.items(): - dir_pint = p_dict["dir_pint"] - file_pint = p_dict["file_pint"] - files = p_dict["files"] - recursive = p_dict["recursive"] - target = proj_path if p_path.startswith(".") else proj_path / p_path - excludes = [target / e for e in p_dict["excludes"]] - - if target.is_file(): - target.chmod(file_pint) - - if target.is_dir() and dir_pint != 0: - target.chmod(dir_pint) - d_chmods = ( - [d for d in target.glob("*") if d.is_dir() and not d.is_symlink() and d not in excludes] - if not recursive - else [d for d in target.rglob("*") if d.is_dir() and not d.is_symlink() and d not in excludes] - ) - for dir_ in d_chmods: - dir_.chmod(dir_pint) - - if target.is_dir() and (files and files[0] == "*"): - f_chmods = ( - [f for f in target.glob("*") if f.is_file() and not f.is_symlink() and f not in excludes] - if not recursive - else [f for f in target.rglob("*") if f.is_file() and not f.is_symlink() and f not in excludes] - ) - for file in f_chmods: - Path(target / file).chmod(file_pint) - - if target.is_dir() and files and files[0] != "*": - for file in files: - Path(target / file).chmod(file_pint) - except OSError as err: - raise OSError(f"ERROR: Couldn't set a permission: {err}") from err - - -def main() -> None: - """main""" - cli_args = parse_cli_args() - # TODO - expanduser - proj_path = Path(cli_args.project_path).resolve() - proj_build, sys_build = pre_check(proj_path) - proj_is_v3 = bool(proj_build.major == 3) - - if not cli_args.force_mode: - print("\nWARNING: If you don't have a backup of the project, create one before continuing!") - sleep(3) - if ( - input(f"\nUpdate the project from version {proj_build.build_version} to {sys_build.build_version}? (y/n) ") - not in INPUT_CONFS - ): - sys.exit(0) - print("\nUpdating...") - else: - print(f"\nUpdating project from version {proj_build.build_version} to {sys_build.build_version}...") - - try: - if proj_is_v3: - v3_backup_path = proj_path / V3_BACKUP_PATH - v3_backup_path.mkdir(exist_ok=True) - copy_files_dirs(V3_BACKUP_PROJ, proj_path, v3_backup_path) # Backup some of the V3 project files - rename_files(V3_RENAME_IN_PROJ, proj_path) - remove_files_dirs(V3_REMOVE_FROM_PROJ, proj_path) - copy_files_dirs(COPY_TO_PROJ, SZ_SYS_PATH, proj_path) - if proj_is_v3: - setup_env(proj_path) - # TODO - Check V4 permissions are good, is sz_create_project good in the first place when V4 to V4? - set_permissions(proj_path, SET_PERMISSIONS) - except OSError as err: - shutil.copytree(v3_backup_path, proj_path, dirs_exist_ok=True) - print(f"\n{err}") - print("\nIf the error is file or directory permission related, run again with appropriate privileges") - print( - "\nIf the error is missing file or directory, check senzingsdk-setup, senzingsdk-tools, and senzingsdk-poc are installed" - ) - else: - if proj_is_v3: # Only delete if no errors so re-running can find the file again - proj_path.joinpath(V3_BUILD).unlink(missing_ok=True) - - print(f"\nProject successfully updated. Refer to {UPGRADE_URL} for additional upgrade instructions") - - -if __name__ == "__main__": - main() diff --git a/sz_tools/sz_update_project_prior8 b/sz_tools/sz_update_project_prior8 deleted file mode 100755 index cf3bf8c..0000000 --- a/sz_tools/sz_update_project_prior8 +++ /dev/null @@ -1,463 +0,0 @@ -#! /usr/bin/env python3 -"""Upgrade a V3 or V4 Senzing SDK project to V4.x.x""" - -import argparse -import json -import shutil -import sys -from contextlib import suppress -from dataclasses import dataclass, field -from pathlib import Path -from time import sleep -from typing import Any - -from packaging import version as p_version - -INPUT_CONFS = ("y", "Y", "yEs", "yES", "YES", "yes", "YEs", "yeS", "Yes", "YeS") -MODULE_NAME = Path(__file__).stem -V3_BACKUP_PATH = "v3_to_v4_upgrade_backups" -V3_BUILD = "g2BuildVersion.json" -V4_BUILD = "szBuildVersion.json" -SZ_SYS_PATH = Path("/opt/senzing") -V3_SYS_PATH = SZ_SYS_PATH / "g2" -V4_SYS_PATH = SZ_SYS_PATH / "er" -V4_DATA_PATH = SZ_SYS_PATH / "data" -V3_SYS_BUILD = V3_SYS_PATH / V3_BUILD -V4_SYS_BUILD = V4_SYS_PATH / V4_BUILD -UPGRADE_URL = "https://www.senzing.com/docs/4_beta/index.html" - -V3_BACKUP_PROJ = { - "bin": {"files": ["g2dbencrypt", "g2saltadm", "g2ssadm"], "excludes": []}, - "etc": {"files": ["senzing_governor.py"], "excludes": []}, - "lib": {"files": ["libSpaceTimeBoxStandardizer.so", "libG2Hasher.so", "libG2SSAdm.so"], "excludes": []}, - "python": { - "files": ["*"], - "excludes": [ - "CompressedFile.py", - "DumpStack.py", - "G2Audit.py", - "G2Command.py", - "G2ConfigTables.py", - "G2ConfigTool.py", - "G2Database.py", - "G2Explorer.py", - "G2Export.py", - "G2IniParams.py", - "G2Loader.py", - "G2Paths.py", - "G2Project.py", - "G2S3.py", - "G2SetupConfig.py", - "G2Snapshot.py", - "SenzingGo.py", - "senzing", - ], - }, - "setupEnv": {"files": [], "excludes": []}, - "g2BuildVersion.json": {"files": [], "excludes": []}, -} - -V3_REMOVE_FROM_PROJ = { - "bin": {"files": ["g2configupgrade", "g2dbencrypt", "g2dbupgrade", "g2saltadm", "g2ssadm"], "excludes": ["bin"]}, - "data": {"files": [], "excludes": []}, - "etc": {"files": ["senzing_governor.py"], "excludes": []}, - "lib": { - "files": [ - "g2.jar", - "libG2*.so", - "libG2Hasher.so", - "libG2SSAdm.so", - "libg2CompJavaScoreSet.so", - "libg2DistinctFeatJava.so", - "libg2EFeatJava.so", - "libg2JVMPlugin.so", - "libg2StdJava.so", - "libmariadbplugin.so", - "libSpaceTimeBoxStandardizer.so", - ], - "excludes": ["lib"], - }, - "python": {"files": ["*"], "excludes": []}, - "resources/config": {"files": ["g2core-configuration-upgrade-*.gtc"], "excludes": []}, - "resources/schema": {"files": ["g2core-schema-*-create.sql", "g2core-schema-*-upgrade-*.sql"], "excludes": []}, - "resources/templates": { - "files": [ - "G2C.db*", - "G2Module.ini", - "cfgVariant.json", - "custom*.txt", - "defaultGNRCP.config", - "g2config.json", - "senzing_governor.py", - "setupEnv", - "stb.config", - ], - "excludes": [], - }, - "sdk": {"files": ["*"], "excludes": []}, - "setupEnv": {"files": [], "excludes": []}, -} - -COPY_TO_PROJ = { - "er/": {"files": ["*"], "excludes": ["sz_create_project", "sz_update_project"]}, - "data": {"files": ["*"], "excludes": []}, -} - -V3_RENAME_IN_PROJ = { - "etc": {"from": "G2Module.ini", "to": "sz_engine_config.ini"}, -} - -PERMISSIONS = { - ".": { - "dir_pint": 0, - "file_pint": 0o660, - "files": ["LICENSE", "NOTICES", "README.1ST", "szBuildVersion.json"], - "excludes": ["setupEnv"], - "recursive": False, - }, - "setupEnv": {"dir_pint": 0, "file_pint": 0o770, "files": [], "excludes": [], "recursive": False}, - "bin": { - "dir_pint": 0o770, - "file_pint": 0o770, - "files": ["*"], - "excludes": ["__pycache__", "_sz_database.py", "_tool_helpers.py"], - "recursive": False, - }, - "data": {"dir_pint": 0o770, "file_pint": 0o660, "files": ["*"], "excludes": [], "recursive": True}, - "lib": {"dir_pint": 0o770, "file_pint": 0o660, "files": ["*"], "excludes": [], "recursive": False}, - "resources": {"dir_pint": 0o770, "file_pint": 0o660, "files": ["*"], "excludes": ["setupEnv"], "recursive": True}, - "resources/templates/setupEnv": { - "dir_pint": 0, - "file_pint": 0o770, - "files": [], - "excludes": [], - "recursive": False, - }, - "sdk": {"dir_pint": 0o770, "file_pint": 0o660, "files": ["*"], "excludes": [], "recursive": True}, - V3_BACKUP_PATH: {"dir_pint": 0o770, "file_pint": 0, "files": [], "excludes": [], "recursive": False}, -} -PERMISSIONS_2 = { - "bin": { - "dir_pint": 0o770, - "file_pint": 0o660, - "files": ["_sz_database.py", "_tool_helpers.py"], - "excludes": [], - "recursive": False, - } -} - - -@dataclass() -class SzBuildDetails: - """Build information for a project or Senzing SDK system install""" - - platform: str - version: str - build_version: str - build_number: str - major: int = field(init=False) - minor: int = field(init=False) - micro: int = field(init=False) - - def __post_init__(self) -> None: - self.version_parsed = p_version.parse(self.version) - self.build_version_parsed = p_version.parse(self.build_version) - self.major = self.version_parsed.major - self.minor = self.version_parsed.minor - self.micro = self.version_parsed.micro - - -# pylint: disable=W0106 - - -def parse_cli_args() -> argparse.Namespace: - """Parse the CLI arguments""" - arg_parser = argparse.ArgumentParser( - allow_abbrev=False, - description="Update an existing V3 or V4 Senzing project to the system installed V4 of Senzing.", - ) - arg_parser.add_argument( - "project_path", - metavar="path", - help="Path of the project to update", - ) - arg_parser.add_argument( - "-f", - "--force", - dest="force_mode", - default=False, - action="store_true", - help="Upgrade without prompts. WARNING: Use with caution, ensure you have a backup of the project", - ) - return arg_parser.parse_args() - - -def dir_listing(path: Path) -> list[Path]: - """Get a listing of the specified path to check for files or directories""" - try: - listing = list(path.iterdir()) - except OSError as err: - if type(err).__name__ == "NotADirectoryError": - print(f"\nERROR: {path} is not a directory, expecting one") - print(f"\nERROR: {err}") - sys.exit(1) - - return listing - - -def pre_check(project_path: Path) -> tuple[SzBuildDetails, SzBuildDetails]: - """Check not trying to overwrite the V4 Senzing system install, that path is a project, and versions are correct""" - if not V4_SYS_PATH.exists(): - print(f"\nERROR: Couldn't locate Senzing SDK V4 system install at {V4_SYS_PATH}") - sys.exit(1) - - if str(project_path).startswith(str(SZ_SYS_PATH)): - print(f"\nERROR: {project_path} is the Senzing system installation, not a Senzing project") - sys.exit(1) - - v3_project_build_file = project_path / V3_BUILD - v4_project_build_file = project_path / V4_BUILD - proj_listing = dir_listing(project_path) - if v3_project_build_file not in proj_listing and v4_project_build_file not in proj_listing: - print(f"\nERROR: {project_path} isn't a Senzing project, expected it to contain either:") - print(f"\tExisting V3 project: {v3_project_build_file}") - print(f"\tExisting V4 project: {v4_project_build_file}") - sys.exit(1) - - try: - proj_build_file = v3_project_build_file if v3_project_build_file in proj_listing else v4_project_build_file - proj_build_details = get_build_details(proj_build_file) - sys_build_details = get_build_details(V4_SYS_BUILD) - except OSError as err: - print(f"\nERROR: Trying to read {proj_build_file} or {V4_SYS_BUILD} to collect version information: {err}") - sys.exit(1) - - if proj_build_details.major not in (3, 4) or sys_build_details.major != 4: - print(f"\nERROR: {MODULE_NAME} updates a V3 project to V4, or V4.n.n to a newer release") - print(f"\tProject version: {proj_build_details.version}") - print(f"\tSystem install version: {sys_build_details.version}") - sys.exit(1) - - if sys_build_details.version_parsed < proj_build_details.version_parsed: - print("\nNo update required, project is newer than the system install") - print(f"\tProject version: {proj_build_details.version}") - print(f"\tSystem install version: {sys_build_details.version}") - sys.exit(0) - - if sys_build_details.version_parsed == proj_build_details.version_parsed: - print(f"\nNo update required, project and system install are the same version - {proj_build_details.version}") - sys.exit(0) - - return (proj_build_details, sys_build_details) - - -def get_build_details(path: Path) -> SzBuildDetails: - """Return dataclass with the details from a build file.""" - try: - with open(path, "r", encoding="utf-8") as f: - # Ignore DATA_VERSION, V3 build files had it V4 doesn't - version_dict = {k.lower(): v for k, v in json.load(f).items() if k != "DATA_VERSION"} - return SzBuildDetails(**version_dict) - except (OSError, json.JSONDecodeError) as err: - print(f"\nERROR: Couldn't get the build information from {path}: {err}") - sys.exit(1) - except TypeError as err: - print(f"\nERROR: When creating build information from {path}: {err}") - sys.exit(1) - - -def remove_dir(dir_: Path, excludes: list[Path]) -> None: - """Recursively remove a directory""" - try: - for path in dir_.iterdir(): - if path in excludes: - continue - path.unlink(missing_ok=True) if (path.is_file() or path.is_symlink()) else remove_dir(path, excludes) - - if not excludes: - with suppress(FileNotFoundError): - dir_.rmdir() - except OSError as err: - raise err - - -def remove_files_dirs(to_remove: dict[str, Any], target_dir: Path) -> None: - """Remove files/directories that are no longer required""" - for r_path, r_dict in to_remove.items(): - target: Path = target_dir / r_path - files = r_dict["files"] - excludes = [target / e for e in r_dict["excludes"]] - - try: - if target.is_dir(): - if not files or (files and files[0] == "*"): - remove_dir(target, excludes) - - if files and files[0] != "*": - target_files = [] - for f in files: - target_files.append(target / f) if "*" not in f else target_files.extend(list(target.glob(f))) - - for target_file in target_files: - target_file.unlink(missing_ok=True) - - if target.is_file() or target.is_symlink(): - target.unlink(missing_ok=True) - except OSError as err: - raise OSError(f"ERROR: Couldn't delete a file or directory: {err}") from err - - -def copy_files_dirs(to_copy: dict[str, Any], source_dir: Path, target_dir: Path) -> None: - """Copy files/directories within and to a project""" - for c_path, c_dict in to_copy.items(): - excludes = c_dict["excludes"] - files = c_dict["files"] - source = source_dir / c_path - - try: - if source.is_dir(): - # If the key in to_copy ends with / copy everything in source dir to target_dir - # er/ as the key copies everything from /opt/senzing/er to target_dir - # - # If the key in to_copy doesn't end with / copy source dir and everything in it to target_dir - # data as the key copies /opt/senzing/data to target_dir/data - target = target_dir if c_path.endswith("/") else target_dir / c_path - - # Copy entire contents of the source directory - if not files or (files and files[0] == "*"): - shutil.copytree(source, target, ignore=shutil.ignore_patterns(*excludes), dirs_exist_ok=True) - - # Create the source directory in the target and only copy listed files - if files and files[0] != "*": - target.mkdir(exist_ok=True, parents=True) - for source_file in [source / f for f in files]: - shutil.copy(source_file, target / source_file.name) - - if source.is_file(): - # Single file copy always copies only the file, if the key to to_copy is er/szBuildVersion.json - # szBuildVersion.json is copied to target_dir and not target_dir/er/szBuildVersion.json - target = target_dir / source.name - shutil.copy(source, target) - except OSError as err: - raise OSError(f"ERROR: Couldn't copy a file or directory: {err}") from err - - -def rename_files(to_rename: dict[str, dict[str, str]], target_dir: Path) -> None: - """Rename existing project files that had a name change""" - - try: - for r_path, r_dict in to_rename.items(): - current = target_dir / r_path / r_dict["from"] - new = target_dir / r_path / r_dict["to"] - with suppress(FileNotFoundError): - current.rename(new) - except OSError as err: - raise OSError(f"ERROR: Couldn't rename a file or directory: {err}") from err - - -def setup_env(proj_path: Path) -> None: - """Create a new setupEnv and replace place holders with paths for the project""" - try: - shutil.copy(proj_path / "resources/templates/setupEnv", proj_path) - setup_path = proj_path / "setupEnv" - - with open(setup_path, "r", encoding="utf-8") as in_: - data = in_.read() - - data = data.replace("${SENZING_DIR}", str(proj_path)).replace("${SENZING_CONFIG_PATH}", str(proj_path / "etc")) - - with open(setup_path, "w", encoding="utf-8") as out: - out.write(data) - except OSError as err: - raise OSError(f"ERROR: Couldn't create a new setupEnv file: {err}") from err - - -def set_permissions(proj_path: Path, permissions: dict[str, dict[str, Any]]) -> None: - """Set permissions for files/dirs copied to the project, or dirs removed and replaced completely e.g., data/""" - try: - for p_path, p_dict in permissions.items(): - dir_pint = p_dict["dir_pint"] - file_pint = p_dict["file_pint"] - files = p_dict["files"] - recursive = p_dict["recursive"] - target = proj_path if p_path.startswith(".") else proj_path / p_path - excludes = [target / e for e in p_dict["excludes"]] - - if target.is_dir(): - if dir_pint != 0: - target.chmod(dir_pint) - d_chmods = ( - [d for d in target.glob("*") if d.is_dir() and not d.is_symlink() and d not in excludes] - if not recursive - else [d for d in target.rglob("*") if d.is_dir() and not d.is_symlink() and d not in excludes] - ) - for dir_ in d_chmods: - dir_.chmod(dir_pint) - - if files and files[0] == "*": - f_chmods = ( - [f for f in target.glob("*") if f.is_file() and not f.is_symlink() and f not in excludes] - if not recursive - else [f for f in target.rglob("*") if f.is_file() and not f.is_symlink() and f not in excludes] - ) - - for file in f_chmods: - Path(target / file).chmod(file_pint) - - if files and files[0] != "*": - for file in files: - Path(target / file).chmod(file_pint) - - if target.is_file(): - target.chmod(file_pint) - except OSError as err: - raise OSError(f"ERROR: Couldn't set a permission: {err}") from err - - -def main() -> None: - """main""" - cli_args = parse_cli_args() - proj_path = Path(cli_args.project_path).expanduser().resolve() - proj_build, sys_build = pre_check(proj_path) - proj_is_v3 = bool(proj_build.major == 3) - - if not cli_args.force_mode: - print("\nWARNING: If you don't have a backup of the project, create one before continuing!") - sleep(3) - if ( - input(f"\nUpdate the project from version {proj_build.build_version} to {sys_build.build_version}? (y/n) ") - not in INPUT_CONFS - ): - sys.exit(0) - print("\nUpdating...") - else: - print(f"\nUpdating project from version {proj_build.build_version} to {sys_build.build_version}...") - - try: - if proj_is_v3: - v3_backup_path = proj_path / V3_BACKUP_PATH - v3_backup_path.mkdir(exist_ok=True) - copy_files_dirs(V3_BACKUP_PROJ, proj_path, v3_backup_path) - rename_files(V3_RENAME_IN_PROJ, proj_path) - remove_files_dirs(V3_REMOVE_FROM_PROJ, proj_path) - copy_files_dirs(COPY_TO_PROJ, SZ_SYS_PATH, proj_path) - if proj_is_v3: - setup_env(proj_path) - set_permissions(proj_path, PERMISSIONS) - set_permissions(proj_path, PERMISSIONS_2) - except OSError as err: - shutil.copytree(v3_backup_path, proj_path, dirs_exist_ok=True) - print(f"\n{err}") - print("\nIf the error is file or directory permission related, run again with appropriate privileges") - print( - "\nIf the error is missing file or directory, check senzingsdk-setup, senzingsdk-tools, and senzingsdk-poc are installed" - ) - else: - if proj_is_v3: # Only delete if no errors so re-running can find the file again - proj_path.joinpath(V3_BUILD).unlink(missing_ok=True) - - print(f"\nProject successfully updated. Refer to {UPGRADE_URL} for additional upgrade instructions") - - -if __name__ == "__main__": - main() diff --git a/sz_tools/sz_update_project_prior9 b/sz_tools/sz_update_project_prior9 deleted file mode 100755 index 129d272..0000000 --- a/sz_tools/sz_update_project_prior9 +++ /dev/null @@ -1,496 +0,0 @@ -#! /usr/bin/env python3 -"""Upgrade a V3 or V4 Senzing SDK project to V4.x.x""" - -import argparse - -# TODO - -# import json -import shutil -import sys -from contextlib import suppress - -# TODO - -# from dataclasses import dataclass, field -from pathlib import Path -from time import sleep -from typing import Any - -from _project_helpers import ( - SZ_SYS_PATH, - V4_BUILD, - V4_SYS_BUILD, - V4_SYS_PATH, - SzBuildDetails, - get_build_details, -) - -# from packaging import version as p_version - -INPUT_CONFS = ("y", "Y", "yEs", "yES", "YES", "yes", "YEs", "yeS", "Yes", "YeS") -MODULE_NAME = Path(__file__).stem -# TODO - -V3_BACKUP_PATH = "v3_to_v4_upgrade_backups" -V3_BUILD = "g2BuildVersion.json" -# V4_BUILD = "szBuildVersion.json" -# SZ_SYS_PATH = Path("/opt/senzing") -V3_SYS_PATH = SZ_SYS_PATH / "g2" -# V4_SYS_PATH = SZ_SYS_PATH / "er" -# V4_DATA_PATH = SZ_SYS_PATH / "data" -V3_SYS_BUILD = V3_SYS_PATH / V3_BUILD -# V4_SYS_BUILD = V4_SYS_PATH / V4_BUILD -UPGRADE_URL = "https://www.senzing.com/docs/4_beta/index.html" - -V3_BACKUP_PROJ = { - "bin": {"files": ["g2dbencrypt", "g2saltadm", "g2ssadm"], "excludes": []}, - "etc": {"files": ["senzing_governor.py"], "excludes": []}, - "lib": {"files": ["libSpaceTimeBoxStandardizer.so", "libG2Hasher.so", "libG2SSAdm.so"], "excludes": []}, - "python": { - "files": ["*"], - "excludes": [ - "CompressedFile.py", - "DumpStack.py", - "G2Audit.py", - "G2Command.py", - "G2ConfigTables.py", - "G2ConfigTool.py", - "G2Database.py", - "G2Explorer.py", - "G2Export.py", - "G2IniParams.py", - "G2Loader.py", - "G2Paths.py", - "G2Project.py", - "G2S3.py", - "G2SetupConfig.py", - "G2Snapshot.py", - "SenzingGo.py", - "senzing", - ], - }, - "setupEnv": {"files": [], "excludes": []}, - "g2BuildVersion.json": {"files": [], "excludes": []}, -} - -V3_REMOVE_FROM_PROJ = { - "bin": {"files": ["g2configupgrade", "g2dbencrypt", "g2dbupgrade", "g2saltadm", "g2ssadm"], "excludes": ["bin"]}, - "data": {"files": [], "excludes": []}, - "etc": {"files": ["senzing_governor.py"], "excludes": []}, - "lib": { - "files": [ - "g2.jar", - "libG2*.so", - "libG2Hasher.so", - "libG2SSAdm.so", - "libg2CompJavaScoreSet.so", - "libg2DistinctFeatJava.so", - "libg2EFeatJava.so", - "libg2JVMPlugin.so", - "libg2StdJava.so", - "libmariadbplugin.so", - "libSpaceTimeBoxStandardizer.so", - ], - "excludes": ["lib"], - }, - "python": {"files": ["*"], "excludes": []}, - "resources/config": {"files": ["g2core-configuration-upgrade-*.gtc"], "excludes": []}, - "resources/schema": {"files": ["g2core-schema-*-create.sql", "g2core-schema-*-upgrade-*.sql"], "excludes": []}, - "resources/templates": { - "files": [ - "G2C.db*", - "G2Module.ini", - "cfgVariant.json", - "custom*.txt", - "defaultGNRCP.config", - "g2config.json", - "senzing_governor.py", - "setupEnv", - "stb.config", - ], - "excludes": [], - }, - "sdk": {"files": ["*"], "excludes": []}, - "setupEnv": {"files": [], "excludes": []}, -} - -COPY_TO_PROJ = { - "er/": {"files": ["*"], "excludes": ["sz_create_project", "sz_update_project"]}, - "data": {"files": ["*"], "excludes": []}, -} - -V3_RENAME_IN_PROJ = { - "etc": {"from": "G2Module.ini", "to": "sz_engine_config.ini"}, -} - -# TODO - -PERMISSIONS = { - ".": { - "dir_pint": 0, - "file_pint": 0o660, - "files": ["LICENSE", "NOTICES", "README.1ST", "szBuildVersion.json"], - "excludes": ["setupEnv"], - "recursive": False, - }, - "setupEnv": {"dir_pint": 0, "file_pint": 0o770, "files": [], "excludes": [], "recursive": False}, - "bin": { - "dir_pint": 0o770, - "file_pint": 0o770, - "files": ["*"], - "excludes": ["__pycache__", "_sz_database.py", "_tool_helpers.py"], - "recursive": False, - }, - "data": {"dir_pint": 0o770, "file_pint": 0o660, "files": ["*"], "excludes": [], "recursive": True}, - "lib": {"dir_pint": 0o770, "file_pint": 0o660, "files": ["*"], "excludes": [], "recursive": False}, - "resources": {"dir_pint": 0o770, "file_pint": 0o660, "files": ["*"], "excludes": ["setupEnv"], "recursive": True}, - "resources/templates/setupEnv": { - "dir_pint": 0, - "file_pint": 0o770, - "files": [], - "excludes": [], - "recursive": False, - }, - "sdk": {"dir_pint": 0o770, "file_pint": 0o660, "files": ["*"], "excludes": [], "recursive": True}, - V3_BACKUP_PATH: {"dir_pint": 0o770, "file_pint": 0, "files": [], "excludes": [], "recursive": False}, -} -PERMISSIONS_2 = { - "bin": { - "dir_pint": 0o770, - "file_pint": 0o660, - "files": ["_sz_database.py", "_tool_helpers.py"], - "excludes": [], - "recursive": False, - } -} - -# TODO - -# @dataclass() -# class SzBuildDetails: -# """Build information for a project or Senzing SDK system install""" - -# platform: str -# version: str -# build_version: str -# build_number: str -# major: int = field(init=False) -# minor: int = field(init=False) -# micro: int = field(init=False) - -# def __post_init__(self) -> None: -# self.version_parsed = p_version.parse(self.version) -# self.build_version_parsed = p_version.parse(self.build_version) -# self.major = self.version_parsed.major -# self.minor = self.version_parsed.minor -# self.micro = self.version_parsed.micro - - -# pylint: disable=W0106 - - -def parse_cli_args() -> argparse.Namespace: - """Parse the CLI arguments""" - arg_parser = argparse.ArgumentParser( - allow_abbrev=False, - description="Update an existing V3 or V4 Senzing project to the system installed V4 of Senzing.", - ) - arg_parser.add_argument( - "project_path", - metavar="path", - help="Path of the project to update", - ) - arg_parser.add_argument( - "-f", - "--force", - dest="force_mode", - default=False, - action="store_true", - help="Upgrade without prompts. WARNING: Use with caution, ensure you have a backup of the project", - ) - return arg_parser.parse_args() - - -def dir_listing(path: Path) -> list[Path]: - """Get a listing of the specified path to check for files or directories""" - try: - listing = list(path.iterdir()) - except OSError as err: - if type(err).__name__ == "NotADirectoryError": - print(f"\nERROR: {path} is not a directory, expecting one") - print(f"\nERROR: {err}") - sys.exit(1) - - return listing - - -def pre_check(project_path: Path) -> tuple[SzBuildDetails, SzBuildDetails]: - """Check not trying to overwrite the V4 Senzing system install, that path is a project, and versions are correct""" - if not V4_SYS_PATH.exists(): - print(f"\nERROR: Couldn't locate Senzing SDK V4 system install at {V4_SYS_PATH}") - sys.exit(1) - - if str(project_path).startswith(str(SZ_SYS_PATH)): - print(f"\nERROR: {project_path} is the Senzing system installation, not a Senzing project") - sys.exit(1) - - v3_project_build_file = project_path / V3_BUILD - v4_project_build_file = project_path / V4_BUILD - proj_listing = dir_listing(project_path) - if v3_project_build_file not in proj_listing and v4_project_build_file not in proj_listing: - print(f"\nERROR: {project_path} isn't a Senzing project, expected it to contain either:") - print(f"\tExisting V3 project: {v3_project_build_file}") - print(f"\tExisting V4 project: {v4_project_build_file}") - sys.exit(1) - - try: - proj_build_file = v3_project_build_file if v3_project_build_file in proj_listing else v4_project_build_file - proj_build_details = get_build_details(proj_build_file) - sys_build_details = get_build_details(V4_SYS_BUILD) - except OSError as err: - print(f"\nERROR: Trying to read {proj_build_file} or {V4_SYS_BUILD} to collect version information: {err}") - sys.exit(1) - # TODO - - except TypeError as err: - print(f"\nERROR: {err}") - sys.exit(1) - - if proj_build_details.major not in (3, 4) or sys_build_details.major != 4: - print(f"\nERROR: {MODULE_NAME} updates a V3 project to V4, or V4.n.n to a newer release") - print(f"\tProject version: {proj_build_details.version}") - print(f"\tSystem install version: {sys_build_details.version}") - sys.exit(1) - - if sys_build_details.version_parsed < proj_build_details.version_parsed: - print("\nNo update required, project is newer than the system install") - print(f"\tProject version: {proj_build_details.version}") - print(f"\tSystem install version: {sys_build_details.version}") - sys.exit(0) - - if sys_build_details.version_parsed == proj_build_details.version_parsed: - print(f"\nNo update required, project and system install are the same version - {proj_build_details.version}") - sys.exit(0) - - return (proj_build_details, sys_build_details) - - -# TODO - -# def get_build_details(path: Path) -> SzBuildDetails: -# """Return dataclass with the details from a build file.""" -# try: -# with open(path, "r", encoding="utf-8") as f: -# # Ignore DATA_VERSION, V3 build files had it V4 doesn't -# version_dict = {k.lower(): v for k, v in json.load(f).items() if k != "DATA_VERSION"} -# return SzBuildDetails(**version_dict) -# except (OSError, json.JSONDecodeError) as err: -# print(f"\nERROR: Couldn't get the build information from {path}: {err}") -# sys.exit(1) -# except TypeError as err: -# print(f"\nERROR: When creating build information from {path}: {err}") -# sys.exit(1) - - -def remove_dir(dir_: Path, excludes: list[Path]) -> None: - """Recursively remove a directory""" - try: - for path in dir_.iterdir(): - if path in excludes: - continue - path.unlink(missing_ok=True) if (path.is_file() or path.is_symlink()) else remove_dir(path, excludes) - - if not excludes: - with suppress(FileNotFoundError): - dir_.rmdir() - except OSError as err: - raise err - - -def remove_files_dirs(to_remove: dict[str, Any], target_dir: Path) -> None: - """Remove files/directories that are no longer required""" - for r_path, r_dict in to_remove.items(): - target: Path = target_dir / r_path - files = r_dict["files"] - excludes = [target / e for e in r_dict["excludes"]] - - try: - if target.is_dir(): - if not files or (files and files[0] == "*"): - remove_dir(target, excludes) - - if files and files[0] != "*": - target_files = [] - for f in files: - target_files.append(target / f) if "*" not in f else target_files.extend(list(target.glob(f))) - - for target_file in target_files: - target_file.unlink(missing_ok=True) - - if target.is_file() or target.is_symlink(): - target.unlink(missing_ok=True) - except OSError as err: - # TODO - - # raise OSError(f"ERROR: Couldn't delete a file or directory: {err}") from err - raise OSError(f"Couldn't delete a file or directory: {err}") from err - - -def copy_files_dirs(to_copy: dict[str, Any], source_dir: Path, target_dir: Path) -> None: - """Copy files/directories within and to a project""" - for c_path, c_dict in to_copy.items(): - excludes = c_dict["excludes"] - files = c_dict["files"] - source = source_dir / c_path - - try: - if source.is_dir(): - # If the key in to_copy ends with / copy everything in source dir to target_dir - # er/ as the key copies everything from /opt/senzing/er to target_dir - # - # If the key in to_copy doesn't end with / copy source dir and everything in it to target_dir - # data as the key copies /opt/senzing/data to target_dir/data - target = target_dir if c_path.endswith("/") else target_dir / c_path - - # Copy entire contents of the source directory - if not files or (files and files[0] == "*"): - shutil.copytree(source, target, ignore=shutil.ignore_patterns(*excludes), dirs_exist_ok=True) - - # Create the source directory in the target and only copy listed files - if files and files[0] != "*": - target.mkdir(exist_ok=True, parents=True) - for source_file in [source / f for f in files]: - shutil.copy(source_file, target / source_file.name) - - if source.is_file(): - # Single file copy always copies only the file, if the key to to_copy is er/szBuildVersion.json - # szBuildVersion.json is copied to target_dir and not target_dir/er/szBuildVersion.json - target = target_dir / source.name - shutil.copy(source, target) - except OSError as err: - # TODO - - # raise OSError(f"ERROR: Couldn't copy a file or directory: {err}") from err - raise OSError(f"Couldn't copy a file or directory: {err}") from err - - -def rename_files(to_rename: dict[str, dict[str, str]], target_dir: Path) -> None: - """Rename existing project files that had a name change""" - - try: - for r_path, r_dict in to_rename.items(): - current = target_dir / r_path / r_dict["from"] - new = target_dir / r_path / r_dict["to"] - with suppress(FileNotFoundError): - current.rename(new) - except OSError as err: - # TODO - - # raise OSError(f"ERROR: Couldn't rename a file or directory: {err}") from err - raise OSError(f"Couldn't rename a file or directory: {err}") from err - - -# TODO - -def setup_env(proj_path: Path) -> None: - """Create a new setupEnv and replace place holders with paths for the project""" - try: - shutil.copy(proj_path / "resources/templates/setupEnv", proj_path) - setup_path = proj_path / "setupEnv" - - with open(setup_path, "r", encoding="utf-8") as in_: - data = in_.read() - - data = data.replace("${SENZING_DIR}", str(proj_path)).replace("${SENZING_CONFIG_PATH}", str(proj_path / "etc")) - - with open(setup_path, "w", encoding="utf-8") as out: - out.write(data) - except OSError as err: - # TODO - - # raise OSError(f"ERROR: Couldn't create a new setupEnv file: {err}") from err - raise OSError(f"Couldn't create a new setupEnv file: {err}") from err - - -# TODO - -def set_permissions(proj_path: Path, permissions: dict[str, dict[str, Any]]) -> None: - """Set permissions for files/dirs copied to the project, or dirs removed and replaced completely e.g., data/""" - try: - for p_path, p_dict in permissions.items(): - dir_pint = p_dict["dir_pint"] - file_pint = p_dict["file_pint"] - files = p_dict["files"] - recursive = p_dict["recursive"] - target = proj_path if p_path.startswith(".") else proj_path / p_path - excludes = [target / e for e in p_dict["excludes"]] - - if target.is_dir(): - if dir_pint != 0: - target.chmod(dir_pint) - d_chmods = ( - [d for d in target.glob("*") if d.is_dir() and not d.is_symlink() and d not in excludes] - if not recursive - else [d for d in target.rglob("*") if d.is_dir() and not d.is_symlink() and d not in excludes] - ) - for dir_ in d_chmods: - dir_.chmod(dir_pint) - - if files and files[0] == "*": - f_chmods = ( - [f for f in target.glob("*") if f.is_file() and not f.is_symlink() and f not in excludes] - if not recursive - else [f for f in target.rglob("*") if f.is_file() and not f.is_symlink() and f not in excludes] - ) - - for file in f_chmods: - Path(target / file).chmod(file_pint) - - if files and files[0] != "*": - for file in files: - Path(target / file).chmod(file_pint) - - if target.is_file(): - target.chmod(file_pint) - except OSError as err: - # TODO - - # raise OSError(f"ERROR: Couldn't set a permission: {err}") from err - raise OSError(f"Couldn't set a permission: {err}") from err - - -def main() -> None: - """main""" - cli_args = parse_cli_args() - proj_path = Path(cli_args.project_path).expanduser().resolve() - proj_build, sys_build = pre_check(proj_path) - proj_is_v3 = bool(proj_build.major == 3) - - if not cli_args.force_mode: - print("\nWARNING: If you don't have a backup of the project, create one before continuing!") - sleep(3) - if ( - input(f"\nUpdate the project from version {proj_build.build_version} to {sys_build.build_version}? (y/n) ") - not in INPUT_CONFS - ): - sys.exit(0) - print("\nUpdating...") - else: - print(f"\nUpdating project from version {proj_build.build_version} to {sys_build.build_version}...") - - try: - if proj_is_v3: - v3_backup_path = proj_path / V3_BACKUP_PATH - v3_backup_path.mkdir(exist_ok=True) - copy_files_dirs(V3_BACKUP_PROJ, proj_path, v3_backup_path) - rename_files(V3_RENAME_IN_PROJ, proj_path) - remove_files_dirs(V3_REMOVE_FROM_PROJ, proj_path) - copy_files_dirs(COPY_TO_PROJ, SZ_SYS_PATH, proj_path) - if proj_is_v3: - setup_env(proj_path) - set_permissions(proj_path, PERMISSIONS) - set_permissions(proj_path, PERMISSIONS_2) - except OSError as err: - shutil.copytree(v3_backup_path, proj_path, dirs_exist_ok=True) - # TODO - Might need ERROR: and change the raise in some places - print(f"\nERROR: {err}") - print("\nIf the error is file or directory permission related, run again with appropriate privileges") - print( - "\nIf the error is missing file or directory, check senzingsdk-setup, senzingsdk-tools, and senzingsdk-poc are installed" - ) - else: - if proj_is_v3: # Only delete if no errors so re-running can find the file again - proj_path.joinpath(V3_BUILD).unlink(missing_ok=True) - - print(f"\nProject successfully updated. Refer to {UPGRADE_URL} for additional upgrade instructions") - - -if __name__ == "__main__": - main() From c38d3682aa95ad2594b5521d59f96abcefa3ba32 Mon Sep 17 00:00:00 2001 From: Ant Date: Fri, 20 Jun 2025 13:28:14 +0200 Subject: [PATCH 6/8] #188 - Save point --- sz_tools/sz_create_project_prior1 | 268 ------------------------------ 1 file changed, 268 deletions(-) delete mode 100755 sz_tools/sz_create_project_prior1 diff --git a/sz_tools/sz_create_project_prior1 b/sz_tools/sz_create_project_prior1 deleted file mode 100755 index 352f1ee..0000000 --- a/sz_tools/sz_create_project_prior1 +++ /dev/null @@ -1,268 +0,0 @@ -#! /usr/bin/env python3 -"""Create a Senzing SDK project""" - -import argparse -import sys -from pathlib import Path -from shutil import copyfile, copytree, ignore_patterns - -# TODO - -from typing import List, Union - -from _project_helpers import ( - COPY_TO_PROJ, - PERMISSIONS, - PERMISSIONS_2, - SZ_SYS_PATH, - V4_SYS_BUILD, - copy_files_dirs, - get_build_details, - set_permissions, - setup_env, -) - -# sz_path on normal rpm/deb install = /opt/senzing/g2 -# sz_install_root would then = /opt/senzing -# TODO Put back when in API package -# SZ_SYS_PATH = Path(__file__).resolve().parents[1] -# sz_path = Path("/opt/senzing/er").resolve() -SZ_SYS_ROOT = Path(__file__).resolve().parents[2] -# V4_BUILD = SZ_SYS_PATH / "szBuildVersion.json" -COPY_TO_ETC = {"er/resources/templates/": {"files": ["*"], "excludes": ["G2C.*", "setupEnv", "g2config.json"]}} - -COPY_TO_VAR = {"er/resources/templates/G2C.db": {"files": [], "excludes": []}} - - -def parse_cli_args() -> argparse.Namespace: - """Parse the CLI arguments""" - arg_parser = argparse.ArgumentParser( - allow_abbrev=False, - description=" Create a new instance of a Senzing project", - formatter_class=argparse.RawTextHelpFormatter, - ) - arg_parser.add_argument( - "project_path", - metavar="path", - help="Path to create new Senzing project in, it must not exist", - ) - - return arg_parser.parse_args() - - -# TODO - -def pre_check(project_path: Path) -> None: - # TODO - - """Check not trying to overwrite the V4 Senzing system install, that path is a project, and versions are correct""" - # if proj_path.exists() and proj_path.samefile(SZ_SYS_ROOT): - # print(f"\nProject cannot be created in {SZ_SYS_ROOT}. Please specify a different path.") - # sys.exit(1) - if str(project_path).startswith(str(SZ_SYS_ROOT)): - print(f"\nProject cannot be created in {SZ_SYS_ROOT}") - sys.exit(1) - - if project_path.exists(): - print(f"\n{project_path} exists, specify a different path") - sys.exit(1) - - -def replace_in_file(filename: Path, old_string: str, new_string: str) -> None: - """Replace strings in new project files""" - - try: - with open(filename, encoding="utf-8") as fr: - data = fr.read() - with open(filename, "w", encoding="utf-8") as fw: - fw.write(data.replace(old_string, new_string)) - except IOError as err: - raise err - - -def set_folder_permissions(path: Path, permissions: int, folders_to_ignore: Union[List[str], None] = None) -> None: - """Set permissions recursively on a folder, optionally ignore specific folders""" - if folders_to_ignore is None: - folders_to_ignore = [] - - path.chmod(permissions) - - dirs: List[Path] = [d for d in path.rglob("*") if d.is_dir() and not d.is_symlink() and d not in folders_to_ignore] - for dir_ in dirs: - dir_.chmod(permissions) - - -def set_file_permissions( - path: Path, - permissions: int, - files_to_ignore: Union[List[str], None] = None, - recursive: bool = False, -) -> None: - """Set permissions on files in a folder, optionally do recursively""" - if files_to_ignore is None: - files_to_ignore = [] - - files: List[Path] = [] - if recursive: - files = [f for f in path.rglob("*") if f.is_file() and f not in files_to_ignore] - else: - files = [f for f in path.iterdir() if f.is_file() and f not in files_to_ignore] - - for file in files: - file.chmod(permissions) - - -# TODO - -def update_sz_engine_config(config_file: Path, project_path: Path) -> None: - """TODO""" - try: - with open(config_file, "r", encoding="utf-8") as in_: - data = in_.read() - - data = ( - data.replace("${SENZING_DATA_DIR}", str(project_path / "data")) - .replace("${SENZING_CONFIG_PATH}", str(project_path / "etc")) - .replace("${SENZING_RESOURCES_DIR}", str(project_path / "resources")) - .replace("${SENZING_VAR_DIR}", str(project_path / "var")) - ) - - with open(config_file, "w", encoding="utf-8") as out: - out.write(data) - except OSError as err: - raise OSError(f"ERROR: Couldn't update new {config_file}: {err}") from err - - -def main() -> None: - """main""" - - cli_args = parse_cli_args() - - # TODO - - # # sz_path on normal rpm/deb install = /opt/senzing/g2 - # # sz_install_root would then = /opt/senzing - # # TODO Put back when in API package - # sz_path = Path(__file__).resolve().parents[1] - # # sz_path = Path("/opt/senzing/er").resolve() - # sz_path_root = Path(__file__).resolve().parents[2] - proj_path = Path(cli_args.project_path).expanduser().resolve() - - bin_path = proj_path.joinpath("bin") - data_path = proj_path.joinpath("data") - etc_path = proj_path.joinpath("etc") - lib_path = proj_path.joinpath("lib") - resources_path = proj_path.joinpath("resources") - sdk_path = proj_path.joinpath("sdk") - var_path = proj_path.joinpath("var") - - # if proj_path.exists() and proj_path.samefile(SZ_SYS_ROOT): - # print(f"\nProject cannot be created in {SZ_SYS_ROOT}. Please specify a different path.") - # sys.exit(1) - - # if proj_path.exists(): - # print(f"\n{proj_path} exists, please specify a different path.") - # sys.exit(1) - pre_check(proj_path) - - # TODO - Add try/except and check all functions are raising not printing and being caught - - # version_details = get_version_details(SZ_SYS_PATH) - build_details = get_build_details(V4_SYS_BUILD) - # print(f"\nSenzing version: {build_details[0]}\n") - print(f"\nSenzing version: {build_details.version}\n") - print(f"\nSenzing version: {build_details}\n") - - # TODO - - # ignore_files = ["sz_create_project", "sz_update_project"] - # # Example: ignore_paths = [sz_path.joinpath('python')] - # ignore_paths: List[str] = [] - # excludes = ignore_files + ignore_paths - - # # Copy sz_path to new project path - # copytree(SZ_SYS_PATH, proj_path, ignore=ignore_patterns(*excludes), symlinks=True) - copy_files_dirs(COPY_TO_PROJ, SZ_SYS_PATH, proj_path) - # Copy resources/templates to etc - # ignore_files = ["G2C.db", "setupEnv", "*.template", "g2config.json"] - # copytree( - # SZ_SYS_PATH.joinpath("resources", "templates"), - # etc_path, - # ignore=ignore_patterns(*ignore_files), - # ) - copy_files_dirs(COPY_TO_ETC, SZ_SYS_PATH, proj_path / "etc") - - # Copy setupEnv - # copyfile( - # SZ_SYS_PATH.joinpath("resources", "templates", "setupEnv"), - # proj_path.joinpath("setupEnv"), - # ) - setup_env(proj_path) - - # Copy G2C.db to runtime location - # Path.mkdir(proj_path.joinpath("var", "sqlite"), parents=True) - # copyfile( - # SZ_SYS_PATH.joinpath("resources", "templates", "G2C.db"), - # var_path.joinpath("sqlite", "G2C.db"), - # ) - copy_files_dirs(COPY_TO_VAR, SZ_SYS_PATH, proj_path / "var" / "sqlite") - - # Copy data - # copytree( - # SZ_SYS_ROOT.joinpath("data"), - # data_path, - # ignore=ignore_patterns(*excludes), - # symlinks=True, - # ) - - # # Files & strings to modify - # update_files = [ - # proj_path.joinpath("setupEnv"), - # etc_path.joinpath("sz_engine_config.ini"), - # ] - - # path_subs = [ - # ("${SENZING_DIR}", proj_path), - # ("${SENZING_CONFIG_PATH}", etc_path), - # ("${SENZING_DATA_DIR}", data_path), - # ("${SENZING_RESOURCES_DIR}", resources_path), - # ("${SENZING_VAR_DIR}", var_path), - # ] - - # for file in update_files: - # for path in path_subs: - # replace_in_file(file, path[0], str(path[1])) - - update_sz_engine_config(proj_path / "etc" / "sz_engine_config.ini", proj_path) - - set_permissions(proj_path, PERMISSIONS) - set_permissions(proj_path, PERMISSIONS_2) - # # Folder permissions - # set_folder_permissions(proj_path, 0o770) - - # # Root of project - # set_file_permissions(proj_path, 0o660) - # proj_path.joinpath("setupEnv").chmod(0o770) - - # # bin - # set_file_permissions(bin_path, 0o770, recursive=True) - - # # etc - # set_file_permissions(etc_path, 0o660) - - # # lib - # set_file_permissions( - # lib_path, - # 0o660, - # files_to_ignore=["g2.jar"], - # ) - - # # resources - # set_file_permissions(resources_path, 0o660, recursive=True) - # resources_path.joinpath("templates", "setupEnv").chmod(0o770) - - # # sdk - # set_file_permissions(sdk_path, 0o664, recursive=True) - - # # var - # set_file_permissions(var_path, 0o660, recursive=True) - - print("Successfully created.") - - -if __name__ == "__main__": - main() From d6f4669dc261486751a1b806472d41c0f4357bfd Mon Sep 17 00:00:00 2001 From: Ant Date: Fri, 20 Jun 2025 13:56:35 +0200 Subject: [PATCH 7/8] #188 - Save point --- sz_tools/_tool_helpers.py | 7 +++++-- sz_tools/sz_command | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/sz_tools/_tool_helpers.py b/sz_tools/_tool_helpers.py index 63a8297..3a7afbf 100644 --- a/sz_tools/_tool_helpers.py +++ b/sz_tools/_tool_helpers.py @@ -417,8 +417,11 @@ def combine_engine_flags(flags: Union[List[TSzEngineFlags], List[str]]) -> int: return result -def get_engine_flag_names() -> List[str]: - return list(SzEngineFlags.__members__.keys()) +def get_engine_flag_names(filter_: List[str] = None) -> List[str]: # type: ignore[assignment] + """Return flag names and optionally filter""" + filter_ = [] if filter_ is None else filter_ + + return [f for f in SzEngineFlags.__members__ if f not in filter_] def get_engine_flags_as_int(flags: List[str]) -> int: diff --git a/sz_tools/sz_command b/sz_tools/sz_command index 008acc9..c2f1b02 100755 --- a/sz_tools/sz_command +++ b/sz_tools/sz_command @@ -310,7 +310,7 @@ class SzCmdShell(cmd.Cmd): self.attrs = self.get_config_attr_codes() # Get Senzing engine flag names for use in auto completion - self.engine_flags_list = get_engine_flag_names() + self.engine_flags_list = get_engine_flag_names(["_SZ_WITHOUT_INFO"]) except SzError as err: print_error(err) sys.exit(1) From f4ba4084ef5cd90cc497a47695d35e520dca3aa2 Mon Sep 17 00:00:00 2001 From: Ant Date: Fri, 20 Jun 2025 14:05:06 +0200 Subject: [PATCH 8/8] #188 - Save point --- sz_tools/sz_file_loader | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/sz_tools/sz_file_loader b/sz_tools/sz_file_loader index 6e9a73c..7ec699d 100755 --- a/sz_tools/sz_file_loader +++ b/sz_tools/sz_file_loader @@ -123,7 +123,7 @@ shutdown = Event() def files_clean_up(errors_file: Path, with_info_file: Path) -> None: - """TODO""" + """Remove error and with_info files if they weren't used""" if check_file_exists(errors_file) and errors_file.stat().st_size == 0: errors_file.unlink() @@ -132,7 +132,7 @@ def files_clean_up(errors_file: Path, with_info_file: Path) -> None: def parse_cli_args() -> argparse.Namespace: - """TODO""" + """Parse command line arguments""" arg_parser = argparse.ArgumentParser( allow_abbrev=False, description="Utility to load Senzing mapped JSON records and process redo records", @@ -374,7 +374,7 @@ def parse_cli_args() -> argparse.Namespace: def check_ingest_files(files: list[str]): - """TODO""" + """Basic test files to load to catch early JSON issues""" for file in files: json_errors = 0 json_good = 0 @@ -406,12 +406,8 @@ def check_ingest_files(files: list[str]): def docker_redirects( files_list: list[str], redirects, - # redirect_path: str, - # errors_path: str, - # withinfo_path: str, - # shuffle_path: str, ) -> tuple[Any, Any, Any]: - """TODO""" + """Modify paths in docker""" errors_path = redirects.errors_path withinfo_path = redirects.withinfo_path shuffle_path = redirects.shuffle_path @@ -467,8 +463,8 @@ def docker_redirects( return (errors_path, withinfo_path, shuffle_path) -def check_redirect_paths(paths: Iterable[str]): - """TODO""" +def check_redirect_paths(paths: Iterable[str]) -> None: + """Check redirect paths are writable""" try: for p in paths: if not p: @@ -562,7 +558,7 @@ def shuffle_ingest_file( def get_sz_engines( sz_factory: SzAbstractFactoryCore, ) -> Tuple[SzEngine, SzDiagnostic, SzProduct, SzConfigManager]: - """TODO""" + """Create required engines""" try: sz_engine = sz_factory.create_engine() sz_diag = sz_factory.create_diagnostic() @@ -575,7 +571,7 @@ def get_sz_engines( def prime_sz_engine(sz_engine: SzEngine) -> None: - """TODO""" + """Prime sz_engine""" logger.info("") logger.info("Priming Senzing engine...") try: @@ -998,7 +994,7 @@ def load_and_redo( def per_result(cli_args: argparse.Namespace, result: dict[str, Any]) -> None: - """TODO""" + """Results for each ingested file""" logger.info("") logger.info("Results") logger.info("-------") @@ -1048,7 +1044,7 @@ def summary_results( cli_args: argparse.Namespace, overall_results: dict[str, dict[str, Any]], ) -> None: - """TODO""" + """Overall results""" elapsed_time_total = 0 load_blank_lines_total = 0 load_error_total = 0