From 1d9acdd65b4c3adb545eec5eafe15b0f88498e2c Mon Sep 17 00:00:00 2001 From: Chris Mahoney <44449504+chrimaho@users.noreply.github.com> Date: Sun, 3 Aug 2025 10:32:10 +1000 Subject: [PATCH 01/43] Fix missing type hints --- src/toolbox_python/dictionaries.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/toolbox_python/dictionaries.py b/src/toolbox_python/dictionaries.py index 9515cb2..37cba4d 100644 --- a/src/toolbox_python/dictionaries.py +++ b/src/toolbox_python/dictionaries.py @@ -305,7 +305,7 @@ class DotDict(dict): """ - def __init__(self, *args, **kwargs) -> None: + def __init__(self, *args: Any, **kwargs: Any) -> None: dict.__init__(self) d = dict(*args, **kwargs) for key, value in d.items(): @@ -352,15 +352,15 @@ def __delattr__(self, key) -> None: except KeyError as e: raise AttributeError(f"Key not found: '{key}'") from e - def update(self, *args, **kwargs) -> None: + def update(self, *args: Any, **kwargs: Any) -> None: """ !!! note "Summary" Override update to convert new values. Parameters: - *args: + *args (Any): Variable length argument list. - **kwargs: + **kwargs (Any): Arbitrary keyword arguments. Returns: From 2de8701e1b9dee00d34f4375f92e9eec01f6a118 Mon Sep 17 00:00:00 2001 From: Chris Mahoney <44449504+chrimaho@users.noreply.github.com> Date: Sun, 3 Aug 2025 10:32:34 +1000 Subject: [PATCH 02/43] Remove redundant lines --- docs/usage/overview.md | 2 -- src/cli/git_checks.sh | 11 ----------- 2 files changed, 13 deletions(-) delete mode 100644 src/cli/git_checks.sh diff --git a/docs/usage/overview.md b/docs/usage/overview.md index 173c6b4..612c7a5 100644 --- a/docs/usage/overview.md +++ b/docs/usage/overview.md @@ -1,3 +1 @@ -# Overview - --8<-- "README.md" diff --git a/src/cli/git_checks.sh b/src/cli/git_checks.sh deleted file mode 100644 index 4340e96..0000000 --- a/src/cli/git_checks.sh +++ /dev/null @@ -1,11 +0,0 @@ -export git_tag="v1.3.2" -git log --oneline $git_tag..HEAD > git_output/git_1.txt -git log --stat $git_tag..HEAD > git_output/git_2.txt -git log --oneline --graph --decorate $git_tag..HEAD > git_output/git_3.txt -git log --oneline --tags $git_tag..HEAD > git_output/git_4.txt -git log $git_tag..HEAD > git_output/git_5.txt -git diff --name-only $git_tag..HEAD > git_output/git_6.txt -git diff $git_tag..HEAD > git_output/git_7.txt -git rev-list --count $git_tag..HEAD > git_output/git_8.txt -git tag --list > git_output/git_9.txt -git show --format=fuller $git_tag > git_output/git_10.txt From 5f32f508de8d101f7ee77c3b311720e6390591d9 Mon Sep 17 00:00:00 2001 From: Chris Mahoney <44449504+chrimaho@users.noreply.github.com> Date: Sat, 16 Aug 2025 22:03:45 +1000 Subject: [PATCH 03/43] Enhance type checking in `checkers.py` and `output.py` by adding more comprehensive `@overload` conditions for better clarity and functionality. --- .pre-commit-config.yaml | 1 + src/tests/test_checkers.py | 69 ++++++++++++++++++++--------- src/toolbox_python/checkers.py | 79 +++++++++++++++++++++++----------- src/toolbox_python/output.py | 31 ++++++------- 4 files changed, 120 insertions(+), 60 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 034ea3c..8ba61f6 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -39,6 +39,7 @@ repos: language_version: python3.13 args: - "--config=pyproject.toml" + - "--line-length=120" - repo: https://github.com/adamchainz/blacken-docs rev: "1.19.1" hooks: diff --git a/src/tests/test_checkers.py b/src/tests/test_checkers.py index a6e3d94..6af1d44 100644 --- a/src/tests/test_checkers.py +++ b/src/tests/test_checkers.py @@ -121,6 +121,14 @@ def setUp(self) -> None: ("set_2", "a", set, False), ("set_3", {1.0, 1.0}, (tuple, set), True), ("set_4", [1, 2], (tuple, set), False), + ("list_type_1", "a", [str, int], True), + ("list_type_2", 1, [str, int], True), + ("list_type_3", 2.5, [str, int], False), + ("list_type_4", True, [str, bool], True), + ("list_type_5", 1, [str, bool], False), + ("list_type_6", (1, 2), [tuple, list], True), + ("list_type_7", [1, 2], [tuple, list], True), + ("list_type_8", {1, 2}, [tuple, list], False), ), name_func=name_func_predefined_name, ) @@ -128,7 +136,7 @@ def test_is_value_of_type( self, _nam: str, _val: Any, - _typ: Union[type, tuple[type, ...]], + _typ: Union[type, tuple[type, ...], list[type]], _res: bool, ) -> None: assert is_value_of_type(_val, _typ) == _res @@ -171,6 +179,14 @@ def test_is_value_of_type( ("set_2", ({1.0, 2.0}, {3.0, 4.0}, [5.0, 6.0]), set, False), ("set_3", ({1.0, 2.0}, {3.0, 4.0}, {5.0, 6.0}), (tuple, set), True), ("set_4", ({1.0, 2.0}, [3.0, 4.0], {5.0, 6.0}), (tuple, str), False), + ("list_type_1", ("a", "b", "c"), [str, int], True), + ("list_type_2", (1, 2, 3), [str, int], True), + ("list_type_3", (2.5, 3.5, 4.5), [str, int], False), + ("list_type_4", (True, False, True), [str, bool], True), + ("list_type_5", (1, 2, 3), [str, bool], False), + ("list_type_6", ((1, 2), (3, 4), (5, 6)), [tuple, list], True), + ("list_type_7", (["a", "b"], ["c", "d"], ["e", "f"]), [tuple, list], True), + ("list_type_8", ({1, 2}, {3, 4}, {5, 6}), [tuple, list], False), ), name_func=name_func_predefined_name, ) @@ -178,7 +194,7 @@ def test_is_all_values_of_type( self, _nam: str, _vals: Any, - _typ: Union[type, tuple[type, ...]], + _typ: Union[type, tuple[type, ...], list[type]], _res: bool, ) -> None: assert is_all_values_of_type(_vals, _typ) == _res @@ -221,6 +237,27 @@ def test_is_all_values_of_type( ("set_2", (1, "a", 2.5, True, (1, 2), [3, 4]), set, False), ("set_3", (1, "a", 2.5, True, (1, 2), {5, 6}), (set, list), True), ("set_4", (1, "a", 2.5, True, (1, 2)), (set, list), False), + ( + "list_type_1", + (1, "a", 2.5, True, (1, 2), [3, 4], {5, 6}), + [str, int], + True, + ), + ("list_type_2", (2.5, True, (1, 2), [3, 4], {5, 6}), [str, int], True), + ( + "list_type_3", + (1, "a", 2.5, True, (1, 2), [3, 4], {5, 6}), + [bool, tuple], + True, + ), + ("list_type_4", (1, "a", 2.5, [3, 4], {5, 6}), [bool, tuple], False), + ( + "list_type_5", + (1, "a", 2.5, True, (1, 2), [3, 4], {5, 6}), + [list, set], + True, + ), + ("list_type_6", (1, "a", 2.5, True, (1, 2)), [list, set], False), ), name_func=name_func_predefined_name, ) @@ -228,7 +265,7 @@ def test_is_any_values_of_type( self, _nam: str, _vals: Any, - _typ: Union[type, tuple[type, ...]], + _typ: Union[type, tuple[type, ...], list[type]], _res: bool, ) -> None: assert is_any_values_of_type(_vals, _typ) == _res @@ -294,16 +331,6 @@ def test_is_any_values_in_iterable(self, _nam: str, _vals: Any) -> None: assert_any_values_in_iterable(_val, _vals) assert_any_in(_val, _vals) - def test_raises_assert_value_of_type(self) -> None: - with raises(TypeError): - assert_value_of_type(5, str) - assert_value_of_type("5", int) - - def test_raises_all_values_of_type(self) -> None: - with raises(TypeError): - assert_all_values_of_type((1, 2, 3, 4, 5), str) - assert_all_values_of_type(("1", "2", "3", "4", "5"), int) - def test_raises_assert_value_in_iterable(self) -> None: with raises(LookupError): assert_value_in_iterable("a", (1, 2, 3)) @@ -329,10 +356,10 @@ class TestContains(TestCase): @classmethod def setUpClass(cls) -> None: values: tuple[str, ...] = ("key_SYSTEM", "key_CLUSTER", "key_GROUP", "key_NODE") - cls.list_values = list(values) - cls.tuple_values = tuple(values) - cls.set_values = set(values) - cls.values: dict[str, Union[list, tuple, set]] = { + cls.list_values: list[str] = list(values) + cls.tuple_values: tuple[str, ...] = tuple(values) + cls.set_values: set[str] = set(values) + cls.values: dict[str, Union[list[str], tuple[str, ...], set[str]]] = { "list": cls.list_values, "tuple": cls.tuple_values, "set": cls.set_values, @@ -380,9 +407,7 @@ def test_all_elements_contains(self, _typ: str, _val: str, _exp: bool) -> None: ), name_func=name_func_nested_list, ) - def test_get_elements_containing( - self, _typ: str, _val: str, _exp: str_tuple - ) -> None: + def test_get_elements_containing(self, _typ: str, _val: str, _exp: str_tuple) -> None: if _exp == "tuple_values": _exp = self.tuple_values _out: str_tuple = get_elements_containing(self.values[_typ], _val) @@ -408,6 +433,8 @@ def test_get_elements_containing( (5, "!=", 10), (3, "in", [1, 2, 3, 4]), (5, "not in", (1, 2, 3, 4)), + (True, "is", True), + (True, "is not", int), ) FALSES = ( (10, "<", 5), @@ -418,6 +445,8 @@ def test_get_elements_containing( (5, "!=", 5), (5, "in", [1, 2, 3, 4]), (3, "not in", (1, 2, 3, 4)), + (True, "is", False), + (True, "is not", True), ) diff --git a/src/toolbox_python/checkers.py b/src/toolbox_python/checkers.py index 9f5971e..61bf5ed 100644 --- a/src/toolbox_python/checkers.py +++ b/src/toolbox_python/checkers.py @@ -108,8 +108,10 @@ ">=": operator.ge, "==": operator.eq, "!=": operator.ne, - "in": lambda a, b: a in b, - "not in": lambda a, b: a not in b, + "in": lambda a, b: operator.contains(b, a), + "not in": lambda a, b: not operator.contains(b, a), + "is": operator.is_, + "is not": operator.is_not, } @@ -129,7 +131,9 @@ def is_value_of_type(value: Any, check_type: type) -> bool: ... @overload def is_value_of_type(value: Any, check_type: tuple[type, ...]) -> bool: ... -def is_value_of_type(value: Any, check_type: Union[type, tuple[type, ...]]) -> bool: +@overload +def is_value_of_type(value: Any, check_type: list[type]) -> bool: ... +def is_value_of_type(value: Any, check_type: Union[type, tuple[type, ...], list[type]]) -> bool: """ !!! note "Summary" Check if a given value is of a specified type or types. @@ -140,7 +144,7 @@ def is_value_of_type(value: Any, check_type: Union[type, tuple[type, ...]]) -> b Params: value (Any): The value to check. - check_type (Union[type, tuple[type]]): + check_type (Union[type, tuple[type], list[type]]): The type or tuple of types to check against. Returns: @@ -181,12 +185,19 @@ def is_value_of_type(value: Any, check_type: Union[type, tuple[type, ...]]) -> b - [`is_value_of_type()`][toolbox_python.checkers.is_value_of_type] - [`is_type()`][toolbox_python.checkers.is_type] """ + check_type = tuple(check_type) if isinstance(check_type, list) else check_type return isinstance(value, check_type) +@overload +def is_all_values_of_type(values: any_collection, check_type: type) -> bool: ... +@overload +def is_all_values_of_type(values: any_collection, check_type: tuple[type, ...]) -> bool: ... +@overload +def is_all_values_of_type(values: any_collection, check_type: list[type]) -> bool: ... def is_all_values_of_type( values: any_collection, - check_type: Union[type, tuple[type, ...]], + check_type: Union[type, tuple[type, ...], list[type]], ) -> bool: """ !!! note "Summary" @@ -198,7 +209,7 @@ def is_all_values_of_type( Params: values (any_collection): The iterable containing values to check. - check_type (Union[type, tuple[type]]): + check_type (Union[type, tuple[type], list[type]]): The type or tuple of types to check against. Returns: @@ -241,12 +252,19 @@ def is_all_values_of_type( - [`is_type()`][toolbox_python.checkers.is_type] - [`is_all_type()`][toolbox_python.checkers.is_all_type] """ + check_type = tuple(check_type) if isinstance(check_type, list) else check_type return all(isinstance(value, check_type) for value in values) +@overload +def is_any_values_of_type(values: any_collection, check_type: type) -> bool: ... +@overload +def is_any_values_of_type(values: any_collection, check_type: tuple[type, ...]) -> bool: ... +@overload +def is_any_values_of_type(values: any_collection, check_type: list[type]) -> bool: ... def is_any_values_of_type( values: any_collection, - check_type: Union[type, tuple[type, ...]], + check_type: Union[type, tuple[type, ...], list[type]], ) -> bool: """ !!! note "Summary" @@ -258,7 +276,7 @@ def is_any_values_of_type( Params: values (any_collection): The iterable containing values to check. - check_type (Union[type, tuple[type]]): + check_type (Union[type, tuple[type], list[type]]): The type or tuple of types to check against. Returns: @@ -301,6 +319,7 @@ def is_any_values_of_type( - [`is_type()`][toolbox_python.checkers.is_type] - [`is_any_type()`][toolbox_python.checkers.is_any_type] """ + check_type = tuple(check_type) if isinstance(check_type, list) else check_type return any(isinstance(value, check_type) for value in values) @@ -550,9 +569,7 @@ def is_valid_value(value: Any, op: str, target: Any) -> bool: """ if op not in OPERATORS: - raise ValueError( - f"Unknown operator '{op}'. Valid operators are: {list(OPERATORS.keys())}" - ) + raise ValueError(f"Unknown operator '{op}'. Valid operators are: {list(OPERATORS.keys())}") op_func: Callable[[Any, Any], bool] = OPERATORS[op] return op_func(value, target) @@ -572,9 +589,15 @@ def is_valid_value(value: Any, op: str, target: Any) -> bool: ## --------------------------------------------------------------------------- # +@overload +def assert_value_of_type(value: Any, check_type: type) -> None: ... +@overload +def assert_value_of_type(value: Any, check_type: tuple[type, ...]) -> None: ... +@overload +def assert_value_of_type(value: Any, check_type: list[type]) -> None: ... def assert_value_of_type( value: Any, - check_type: Union[type, tuple[type, ...]], + check_type: Union[type, tuple[type, ...], list[type]], ) -> None: """ !!! note "Summary" @@ -586,7 +609,7 @@ def assert_value_of_type( Params: value (Any): The value to check. - check_type (Union[type, tuple[type]]): + check_type (Union[type, tuple[type], list[type]]): The type or tuple of types to check against. Raises: @@ -660,9 +683,15 @@ def assert_value_of_type( raise TypeError(msg) +@overload +def assert_all_values_of_type(values: any_collection, check_type: type) -> None: ... +@overload +def assert_all_values_of_type(values: any_collection, check_type: tuple[type, ...]) -> None: ... +@overload +def assert_all_values_of_type(values: any_collection, check_type: list[type]) -> None: ... def assert_all_values_of_type( values: any_collection, - check_type: Union[type, tuple[type, ...]], + check_type: Union[type, tuple[type, ...], list[type]], ) -> None: """ !!! note "Summary" @@ -674,7 +703,7 @@ def assert_all_values_of_type( Params: values (any_collection): The iterable containing values to check. - check_type (Union[type, tuple[type]]): + check_type (Union[type, tuple[type], list[type]]): The type or tuple of types to check against. Raises: @@ -743,14 +772,8 @@ def assert_all_values_of_type( """ if not is_all_type(values=values, check_type=check_type): invalid_values = [value for value in values if not is_type(value, check_type)] - invalid_types = [ - f"'{type(value).__name__}'" - for value in values - if not is_type(value, check_type) - ] - msg: str = ( - f"Some elements {invalid_values} have the incorrect type {invalid_types}. " - ) + invalid_types = [f"'{type(value).__name__}'" for value in values if not is_type(value, check_type)] + msg: str = f"Some elements {invalid_values} have the incorrect type {invalid_types}. " if isinstance(check_type, type): msg += f"Must be '{check_type}'" else: @@ -759,9 +782,15 @@ def assert_all_values_of_type( raise TypeError(msg) +@overload +def assert_any_values_of_type(values: any_collection, check_type: type) -> None: ... +@overload +def assert_any_values_of_type(values: any_collection, check_type: tuple[type, ...]) -> None: ... +@overload +def assert_any_values_of_type(values: any_collection, check_type: list[type]) -> None: ... def assert_any_values_of_type( values: any_collection, - check_type: Union[type, tuple[type, ...]], + check_type: Union[type, tuple[type, ...], list[type]], ) -> None: """ !!! note "Summary" @@ -773,7 +802,7 @@ def assert_any_values_of_type( Params: values (any_collection): The iterable containing values to check. - check_type (Union[type, tuple[type]]): + check_type (Union[type, tuple[type], list[type]]): The type or tuple of types to check against. Raises: diff --git a/src/toolbox_python/output.py b/src/toolbox_python/output.py index 62237f2..5492efb 100644 --- a/src/toolbox_python/output.py +++ b/src/toolbox_python/output.py @@ -78,10 +78,21 @@ # ---------------------------------------------------------------------------- # +@overload +def print_or_log_output(message: str, print_or_log: Literal["print"]) -> None: ... +@overload +def print_or_log_output( + message: str, + print_or_log: Literal["log"], + *, + log: Logger, + log_level: log_levels = "info", +) -> None: ... @typechecked def print_or_log_output( message: str, print_or_log: Literal["print", "log"] = "print", + *, log: Optional[Logger] = None, log_level: Optional[log_levels] = None, ) -> None: @@ -221,13 +232,11 @@ def print_or_log_output( # Check in put for logging if not is_type(log, Logger): raise TypeError( - f"When `print_or_log=='log'` then `log` must be type `Logger`. " - f"Here, you have parsed: '{type(log)}'" + f"When `print_or_log=='log'` then `log` must be type `Logger`. " f"Here, you have parsed: '{type(log)}'" ) if log_level is None: raise ValueError( - f"When `print_or_log=='log'` then `log_level` must be parsed " - f"with a valid value from: {log_levels}." + f"When `print_or_log=='log'` then `log_level` must be parsed " f"with a valid value from: {log_levels}." ) # Assertions to keep `mypy` happy @@ -419,27 +428,19 @@ def list_columns( # Segment the list into chunks segmented_list: list[str_list] = [ - string_list[index : index + cols_wide] - for index in range(0, len(string_list), cols_wide) + string_list[index : index + cols_wide] for index in range(0, len(string_list), cols_wide) ] # Ensure the last segment has the correct number of columns if columnwise: if len(segmented_list[-1]) != cols_wide: - segmented_list[-1].extend( - [""] * (len(string_list) - len(segmented_list[-1])) - ) + segmented_list[-1].extend([""] * (len(string_list) - len(segmented_list[-1]))) combined_list: Union[list[str_list], Any] = zip(*segmented_list) else: combined_list = segmented_list # Create the formatted string with proper spacing - printer: str = "\n".join( - [ - "".join([element.ljust(max_len + gap) for element in group]) - for group in combined_list - ] - ) + printer: str = "\n".join(["".join([element.ljust(max_len + gap) for element in group]) for group in combined_list]) # Print the output if requested if print_output: From 713ae0e8ec4f9c448b6cdc8b8f3645b658a84c23 Mon Sep 17 00:00:00 2001 From: Chris Mahoney <44449504+chrimaho@users.noreply.github.com> Date: Sat, 16 Aug 2025 22:10:29 +1000 Subject: [PATCH 04/43] Run linting for better formatting --- .pre-commit-config.yaml | 1 - pyproject.toml | 1 + src/tests/test_dictionaries.py | 4 +-- src/tests/test_output.py | 4 +-- src/tests/test_retry.py | 58 ++++++++++---------------------- src/tests/test_strings.py | 8 ++--- src/toolbox_python/defaults.py | 9 ++--- src/toolbox_python/generators.py | 3 +- src/toolbox_python/retry.py | 15 ++------- src/utils/bump_version.py | 6 +++- src/utils/changelog.py | 27 ++++----------- src/utils/scripts.py | 57 ++++++++----------------------- 12 files changed, 54 insertions(+), 139 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8ba61f6..034ea3c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -39,7 +39,6 @@ repos: language_version: python3.13 args: - "--config=pyproject.toml" - - "--line-length=120" - repo: https://github.com/adamchainz/blacken-docs rev: "1.19.1" hooks: diff --git a/pyproject.toml b/pyproject.toml index 7882433..bd1797a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -124,6 +124,7 @@ test = [ ] [tool.black] +line-length = 120 color = true exclude = ''' /( diff --git a/src/tests/test_dictionaries.py b/src/tests/test_dictionaries.py index 322a8aa..e43c7f2 100644 --- a/src/tests/test_dictionaries.py +++ b/src/tests/test_dictionaries.py @@ -48,9 +48,7 @@ def setUpClass(cls) -> None: cls.dict_basic: dict_str_int = {"a": 1, "b": 2, "c": 3} - cls.dict_iterables: dict[ - str, Union[str_list, int_list, str_tuple, int_tuple] - ] = { + cls.dict_iterables: dict[str, Union[str_list, int_list, str_tuple, int_tuple]] = { "a": ["1", "2", "3"], "b": [4, 5, 6], "c": ("7", "8", "9"), diff --git a/src/tests/test_output.py b/src/tests/test_output.py index ddb46e5..7945ed2 100644 --- a/src/tests/test_output.py +++ b/src/tests/test_output.py @@ -263,9 +263,7 @@ def test_5_rowwise(self) -> None: assert output == expected @parameterized.expand([("list"), ("tuple"), ("set"), ("generator")]) - def test_6_types( - self, input_type: Literal["list", "tuple", "set", "generator"] - ) -> None: + def test_6_types(self, input_type: Literal["list", "tuple", "set", "generator"]) -> None: words: str_list = self.get_list_of_words(4 * 3) expected: str = "\n".join( [ diff --git a/src/tests/test_retry.py b/src/tests/test_retry.py index 188f666..fe2f786 100644 --- a/src/tests/test_retry.py +++ b/src/tests/test_retry.py @@ -54,9 +54,7 @@ def setUp(self) -> None: pass @fixture(autouse=True) - def _pass_fixtures( - self, capsys: CaptureFixture[str], caplog: LogCaptureFixture - ) -> None: + def _pass_fixtures(self, capsys: CaptureFixture[str], caplog: LogCaptureFixture) -> None: self.capsys: CaptureFixture = capsys self.caplog: LogCaptureFixture = caplog self.caplog.set_level("notset".upper()) @@ -80,12 +78,7 @@ def test_fail_always_print(self) -> None: console_print = self.capsys.readouterr() error_output = "Still could not write after 5 iterations. Please check." error_string = "Caught an expected error at iteration {i}: `{__name__}`. Retrying in 1 seconds..." - error_message = "\n".join( - [ - error_string.format(i=i, __name__=f"{__name__}.ExpectedError") - for i in range(1, 6) - ] - ) + error_message = "\n".join([error_string.format(i=i, __name__=f"{__name__}.ExpectedError") for i in range(1, 6)]) error_message += f"\n{error_output}\n" assert str(e.exception) == error_output assert console_print.out == error_message @@ -107,9 +100,7 @@ def test_fail_always_log(self) -> None: for index, record in enumerate(log_records): if index < 5: assert record.levelname == "warning".upper() - assert record.message == error_string.format( - i=index + 1, __name__=f"{__name__}.ExpectedError" - ) + assert record.message == error_string.format(i=index + 1, __name__=f"{__name__}.ExpectedError") else: assert record.levelname == "error".upper() assert record.message == error_output @@ -127,9 +118,7 @@ def test_fail_first_print(self) -> None: with self.assertRaises(RuntimeError) as e: self.fail_unknown_print() console_print = self.capsys.readouterr() - error_output = ( - f"Caught an unexpected error at iteration 1: `{__name__}.ExpectedError`." - ) + error_output = f"Caught an unexpected error at iteration 1: `{__name__}.ExpectedError`." assert console_print.out == error_output + "\n" assert str(e.exception) == error_output @@ -144,9 +133,7 @@ def test_fail_first_log(self) -> None: with self.assertRaises(RuntimeError) as e: self.fail_unknown_log() log_records = self.caplog.records - error_output = ( - f"Caught an unexpected error at iteration 1: `{__name__}.ExpectedError`." - ) + error_output = f"Caught an unexpected error at iteration 1: `{__name__}.ExpectedError`." pprint(log_records) assert log_records[0].levelname == "error".upper() assert log_records[0].message == error_output @@ -175,13 +162,12 @@ def test_fail_after_n_print(self) -> None: with self.assertRaises(RuntimeError) as e: self.fail_after_n_print(iterations=num_fail_iterations) console_print = self.capsys.readouterr() - error_output = f"Caught an unexpected error at iteration {num_fail_iterations+1}: `tests.test_retry.UnexpectedError`." + error_output = ( + f"Caught an unexpected error at iteration {num_fail_iterations+1}: `tests.test_retry.UnexpectedError`." + ) error_string = "Caught an expected error at iteration {i}: `{__name__}`. Retrying in 1 seconds..." error_message = "\n".join( - [ - error_string.format(i=i, __name__=f"{__name__}.ExpectedError") - for i in range(1, num_fail_iterations + 1) - ] + [error_string.format(i=i, __name__=f"{__name__}.ExpectedError") for i in range(1, num_fail_iterations + 1)] ) error_message += f"\n{error_output}\n" assert str(e.exception) == error_output @@ -200,14 +186,14 @@ def test_fail_after_n_log(self) -> None: with self.assertRaises(RuntimeError) as e: self.fail_after_n_log(iterations=num_fail_iterations) log_records = self.caplog.records - error_output = f"Caught an unexpected error at iteration {num_fail_iterations+1}: `tests.test_retry.UnexpectedError`." + error_output = ( + f"Caught an unexpected error at iteration {num_fail_iterations+1}: `tests.test_retry.UnexpectedError`." + ) error_string = "Caught an expected error at iteration {i}: `{__name__}`. Retrying in 1 seconds..." for index, record in enumerate(log_records): if index < num_fail_iterations: assert record.levelname == "warning".upper() - assert record.message == error_string.format( - i=index + 1, __name__=f"{__name__}.ExpectedError" - ) + assert record.message == error_string.format(i=index + 1, __name__=f"{__name__}.ExpectedError") else: assert record.levelname == "error".upper() assert record.message == error_output @@ -263,10 +249,7 @@ def test_succeed_after_n_print(self) -> None: output = f"Successfully executed at iteration {num_fail_iterations+1}." error_string = "Caught an expected error at iteration {i}: `{__name__}`. Retrying in 1 seconds..." error_message = "\n".join( - [ - error_string.format(i=i, __name__=f"{__name__}.ExpectedError") - for i in range(1, num_fail_iterations + 1) - ] + [error_string.format(i=i, __name__=f"{__name__}.ExpectedError") for i in range(1, num_fail_iterations + 1)] ) error_message += f"\n{output}\n" self.assertIsNone(result) @@ -287,9 +270,7 @@ def test_succeed_after_n_log(self) -> None: for index, record in enumerate(log_records): if index < num_fail_iterations: assert record.levelname == "warning".upper() - assert record.message == error_string.format( - i=index + 1, __name__=f"{__name__}.ExpectedError" - ) + assert record.message == error_string.format(i=index + 1, __name__=f"{__name__}.ExpectedError") else: assert record.levelname == "info".upper() assert record.message == output @@ -314,10 +295,7 @@ def fail_invalid_delay() -> None: @staticmethod def throw_unexpected_known_error(known_error: type = ValueError) -> None: - raise UnexpectedKnownError( - f"Throwing UnexpectedError. " - f"Containing known Exception: {known_error.__name__}" - ) + raise UnexpectedKnownError(f"Throwing UnexpectedError. " f"Containing known Exception: {known_error.__name__}") @retry(exceptions=ValueError, tries=5, delay=0, print_or_log="print") def succeed_unexpected_known_error(self, known_error: type = ValueError) -> None: @@ -332,9 +310,7 @@ def test_succeed_print_with_hidden_exception(self) -> None: _ = self.succeed_unexpected_known_error(known_error=ValueError) console_print = self.capsys.readouterr() num_iterations = 5 - output = ( - f"Still could not write after {num_iterations} iterations. Please check." - ) + output = f"Still could not write after {num_iterations} iterations. Please check." error_string = ( "Caught an unexpected, known error at iteration {i}: `{__name__}`.\n" "Who's message contains reference to underlying exception(s): ['ValueError'].\n" diff --git a/src/tests/test_strings.py b/src/tests/test_strings.py index 0926743..52c54d5 100644 --- a/src/tests/test_strings.py +++ b/src/tests/test_strings.py @@ -52,9 +52,7 @@ def test_str_replace_1(self) -> None: def test_str_replace_2(self) -> None: _input: str = self.complex_sentence _output: str = str_replace(_input) - _expected: str = ( - self.complex_sentence.replace(" ", "").replace(",", "").replace(".", "") - ) + _expected: str = self.complex_sentence.replace(" ", "").replace(",", "").replace(".", "") assert _output == _expected def test_str_contains_valid(self) -> None: @@ -140,8 +138,6 @@ class TestStrToList(TestCase): (123, 123), ] ) - def test_str_to_list( - self, _input: Union[str, Any], _expected: Union[str_list, Any] - ) -> None: + def test_str_to_list(self, _input: Union[str, Any], _expected: Union[str_list, Any]) -> None: _output: str_list = str_to_list(_input) assert _output == _expected diff --git a/src/toolbox_python/defaults.py b/src/toolbox_python/defaults.py index 0e528e8..b9376b9 100644 --- a/src/toolbox_python/defaults.py +++ b/src/toolbox_python/defaults.py @@ -326,11 +326,7 @@ def get( - [`Defaults._validate_value_and_default()`][toolbox_python.defaults.Defaults._validate_value_and_default] - [`Defaults._validate_type()`][toolbox_python.defaults.Defaults._validate_type] """ - ( - self._validate_value_and_default( - value=value, default=default - )._validate_type(check_type=cast) - ) + self._validate_value_and_default(value=value, default=default)._validate_type(check_type=cast) if value is None: value = default if cast is not None: @@ -418,8 +414,7 @@ def _validate_type( retype = check_type.__name__ # type: ignore if retype is not None and retype not in valid_types: raise AttributeError( - f"The value for `type` is invalid: `{retype}`.\n" - f"Must be a valid type: {valid_types}." + f"The value for `type` is invalid: `{retype}`.\n" f"Must be a valid type: {valid_types}." ) return self diff --git a/src/toolbox_python/generators.py b/src/toolbox_python/generators.py index 80e0aa1..ff0d630 100644 --- a/src/toolbox_python/generators.py +++ b/src/toolbox_python/generators.py @@ -61,7 +61,8 @@ @typechecked def generate_group_cutoffs( - total_number: int, num_groups: int + total_number: int, + num_groups: int, ) -> tuple[tuple[int, int], ...]: """ !!! note "Summary" diff --git a/src/toolbox_python/retry.py b/src/toolbox_python/retry.py index 09917f8..3a85094 100644 --- a/src/toolbox_python/retry.py +++ b/src/toolbox_python/retry.py @@ -214,9 +214,7 @@ def retry( assert_is_valid(tries, ">=", 0) assert_is_valid(delay, ">=", 0) - exceptions = ( - tuple(exceptions) if isinstance(exceptions, (list, tuple)) else (exceptions,) - ) + exceptions = tuple(exceptions) if isinstance(exceptions, (list, tuple)) else (exceptions,) log: Optional[Logger] = None @@ -256,11 +254,7 @@ def result(*args, **kwargs): >>> java.util.concurrent.ExecutionException: io.delta.exceptions. ... ConcurrentDeleteReadException: This transaction attempted to read one or more files that were deleted (for example part-00001-563449ea-73e4-4d7d-8ba8-53fee1f8a5ff.c000.snappy.parquet in the root of the table) by a concurrent update. Please try the operation again. """ - excs = ( - [exceptions] - if not isinstance(exceptions, (list, tuple)) - else exceptions - ) + excs = [exceptions] if not isinstance(exceptions, (list, tuple)) else exceptions exc_names = [exc.__name__ for exc in excs] if any(name in f"{exc}" for name in exc_names): caught_error = [name for name in exc_names if name in f"{exc}"] @@ -278,10 +272,7 @@ def result(*args, **kwargs): ) sleep(delay) else: - message = ( - f"Caught an unexpected error at iteration {i}: " - f"`{get_full_class_name(exc)}`." - ) + message = f"Caught an unexpected error at iteration {i}: " f"`{get_full_class_name(exc)}`." print_or_log_output( message=message, print_or_log=print_or_log, diff --git a/src/utils/bump_version.py b/src/utils/bump_version.py index ff6c9ea..6f93c32 100644 --- a/src/utils/bump_version.py +++ b/src/utils/bump_version.py @@ -43,7 +43,11 @@ ### Set up argument parsing ---- parser = argparse.ArgumentParser(description="Bump version in files.") parser.add_argument( - "-v", "--verbose", default=False, type=bool, help="Enable verbose output." + "-v", + "--verbose", + default=False, + type=bool, + help="Enable verbose output.", ) parser.add_argument("version", type=str, help="The new version to set in the files.") diff --git a/src/utils/changelog.py b/src/utils/changelog.py index 8911e88..6a8c373 100644 --- a/src/utils/changelog.py +++ b/src/utils/changelog.py @@ -42,13 +42,9 @@ TOKEN: str | None = os.environ.get("GITHUB_TOKEN") REPOSITORY_NAME: str | None = os.environ.get("REPOSITORY_NAME") if TOKEN is None: - raise RuntimeError( - "Environment variable `GITHUB_TOKEN` is not set. Please set it before running the script." - ) + raise RuntimeError("Environment variable `GITHUB_TOKEN` is not set. Please set it before running the script.") if REPOSITORY_NAME is None: - raise RuntimeError( - "Environment variable `REPOSITORY_NAME` is not set. Please set it before running the script." - ) + raise RuntimeError("Environment variable `REPOSITORY_NAME` is not set. Please set it before running the script.") ### Static ---- @@ -143,14 +139,9 @@ def add_release_notes(release: GitRelease) -> str: including the release body and any additional information. """ release_body: str = ( - release.body.replace(f"{BLANK_LINE}", NEW_LINE) - .replace("## ", "### ") - .replace(NEW_LINE, f"{TAB * 2}") - ) - return ( - f'{TAB}??? note "Release Notes"{BLANK_LINE}' - f"{TAB * 2}{release_body}{BLANK_LINE}" + release.body.replace(f"{BLANK_LINE}", NEW_LINE).replace("## ", "### ").replace(NEW_LINE, f"{TAB * 2}") ) + return f'{TAB}??? note "Release Notes"{BLANK_LINE}' f"{TAB * 2}{release_body}{BLANK_LINE}" def add_commit_info(commit: Commit) -> str: @@ -218,9 +209,7 @@ def main() -> None: # If there is no previous release, we fetch all commits until the current release. This is the case for the very first release in the repo. ### Determine the previous tag if it exists, otherwise set it to "0" - previous_tag: str = ( - releases[index + 1].tag_name if index + 1 < len(releases) else "0" - ) + previous_tag: str = releases[index + 1].tag_name if index + 1 < len(releases) else "0" ### Write the release information to the output file ---- f.write(add_release_info(release, REPO)) @@ -238,11 +227,7 @@ def main() -> None: ### Fetch the commits for the current release ---- commits: list[Commit] = sorted( REPO.get_commits( - since=( - releases[index + 1].created_at - if previous_tag != "0" - else NotSet - ), + since=(releases[index + 1].created_at if previous_tag != "0" else NotSet), until=release.created_at, ), key=lambda c: c.commit.committer.date, diff --git a/src/utils/scripts.py b/src/utils/scripts.py index a2567dd..e4502b2 100644 --- a/src/utils/scripts.py +++ b/src/utils/scripts.py @@ -44,10 +44,7 @@ def get_all_files(*suffixes) -> list[str]: return [ str(p) for p in Path("./").glob("**/*") - if ".venv" not in p.parts - and not p.parts[0].startswith(".") - and p.is_file() - and p.suffix in {*suffixes} + if ".venv" not in p.parts and not p.parts[0].startswith(".") and p.is_file() and p.suffix in {*suffixes} ] @@ -259,9 +256,7 @@ def docs_build_versioned(version: str) -> None: run("git config --global --list") run("git config --local --list") run("git remote --verbose") - run( - f"mike --debug deploy --update-aliases --branch=docs-site --push {version} latest" - ) + run(f"mike --debug deploy --update-aliases --branch=docs-site --push {version} latest") def docs_build_versioned_cli() -> None: @@ -374,17 +369,13 @@ class DocstringVisitor(ast.NodeVisitor): def visit_FunctionDef(self, node): if node.name.startswith("_"): # Skip private functions return - functions_and_classes.append( - FunctionAndClassDetails("function", node.name, node, node.lineno) - ) + functions_and_classes.append(FunctionAndClassDetails("function", node.name, node, node.lineno)) self.generic_visit(node) def visit_ClassDef(self, node): if node.name.startswith("_"): # Skip private classes return - functions_and_classes.append( - FunctionAndClassDetails("class", node.name, node, node.lineno) - ) + functions_and_classes.append(FunctionAndClassDetails("class", node.name, node, node.lineno)) self.generic_visit(node) visitor = DocstringVisitor() @@ -440,18 +431,14 @@ def _check_single_docstring( # Get function parameters (excluding 'self' for methods) params = [arg.arg for arg in node.args.args if arg.arg != "self"] if params and not re.search(r"Params:", docstring): - raise ValueError( - "Missing mandatory Params section for function with parameters" - ) + raise ValueError("Missing mandatory Params section for function with parameters") # Check each parameter is documented if params: for param in params: param_pattern = rf"{param}\s*\([^)]+\):" if not re.search(param_pattern, docstring): - raise ValueError( - f"Parameter '{param}' not documented or incorrectly formatted" - ) + raise ValueError(f"Parameter '{param}' not documented or incorrectly formatted") # Check Returns or Yields (but not both) has_returns = re.search(r"Returns:", docstring) @@ -466,9 +453,7 @@ def _check_single_docstring( # Check mandatory Examples section if not re.search(r'\?\?\?\+ example "Examples"', docstring, re.IGNORECASE): - raise ValueError( - 'Missing mandatory Examples section: `???+ example "Examples"`' - ) + raise ValueError('Missing mandatory Examples section: `???+ example "Examples"`') # Validate section order _check_section_order(docstring) @@ -561,9 +546,7 @@ def _validate_section_formats(docstring: str, name: str) -> None: for line in param_lines: if not line.startswith(" "): # Parameter name line if not re.match(r"\w+\s*\([^)]+\):", line): - raise ValueError( - f"Invalid parameter format: '{line}'. Expected: 'param_name (type):'" - ) + raise ValueError(f"Invalid parameter format: '{line}'. Expected: 'param_name (type):'") # Validate Raises format if re.search(r"Raises:", docstring): @@ -575,9 +558,7 @@ def _validate_section_formats(docstring: str, name: str) -> None: if raises_match: raises_content = raises_match.group(1) # Check each exception follows the format: ExceptionType: - exception_lines = [ - line for line in raises_content.split("\n") if line.strip() - ] + exception_lines = [line for line in raises_content.split("\n") if line.strip()] for line in exception_lines: if not line.startswith(" "): # Exception name line if not re.match( @@ -586,9 +567,7 @@ def _validate_section_formats(docstring: str, name: str) -> None: ): # Allow common exception patterns if not line.endswith(":"): - raise ValueError( - f"Invalid exception format: '{line}'. Expected: 'ExceptionType:'" - ) + raise ValueError(f"Invalid exception format: '{line}'. Expected: 'ExceptionType:'") # Validate Returns/Yields format returns_or_yields = re.search(r"(Returns|Yields):", docstring) @@ -602,9 +581,7 @@ def _validate_section_formats(docstring: str, name: str) -> None: if section_match: section_content = section_match.group(1) # Check format: optional_name (type): - return_lines = [ - line for line in section_content.split("\n") if line.strip() - ] + return_lines = [line for line in section_content.split("\n") if line.strip()] for line in return_lines: if not line.startswith(" "): # Return value line if not re.match(r"(\w+\s*)?\([^)]+\):", line): @@ -636,9 +613,7 @@ def check_docstrings_all() -> None: print("No Python files found to check.") return else: - print( - f"\nChecking docstrings in all Python files... Files to check: '{len(python_files)}'." - ) + print(f"\nChecking docstrings in all Python files... Files to check: '{len(python_files)}'.") errors = [] @@ -658,16 +633,12 @@ def check_docstrings_all() -> None: def check_docstrings_dir(dir: str) -> None: """Check docstrings in all Python files in the src directory.""" - python_files: list[str] = [ - file for file in get_all_files(".py") if file.startswith(dir) - ] + python_files: list[str] = [file for file in get_all_files(".py") if file.startswith(dir)] if not python_files: print("No Python files found to check.") return else: - print( - f"\nChecking docstrings in all Python files in '{dir}'... Files to check: '{len(python_files)}'." - ) + print(f"\nChecking docstrings in all Python files in '{dir}'... Files to check: '{len(python_files)}'.") errors = [] From fa4ec29d715fd5a7797cc99b4061d080f1cbd0b5 Mon Sep 17 00:00:00 2001 From: Chris Mahoney <44449504+chrimaho@users.noreply.github.com> Date: Sun, 5 Oct 2025 16:10:18 +1100 Subject: [PATCH 05/43] Add `docstring-format-checker` to `docs` dependencies and tidy up all docstrings - Details admonition: `???+ info "Details"` --> `???+ abstract "Details"` - Notes admonition: `???+ info "Notes"` --> `???+ abstract "Notes"` - Generic type list: `Type` --> `(Type)` - Typos: `Parameters` --> `Params` - Missing docstrings --- .pre-commit-config.yaml | 5 +- pyproject.toml | 17 +++++ src/toolbox_python/bools.py | 4 +- src/toolbox_python/checkers.py | 56 +++++++-------- src/toolbox_python/classes.py | 2 +- src/toolbox_python/defaults.py | 8 +-- src/toolbox_python/dictionaries.py | 106 +++++++++++++++++++++++++---- src/toolbox_python/generators.py | 6 +- src/toolbox_python/lists.py | 8 +-- src/toolbox_python/output.py | 10 +-- src/toolbox_python/retry.py | 6 +- src/toolbox_python/strings.py | 12 ++-- 12 files changed, 171 insertions(+), 69 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 034ea3c..ed0d12a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -121,6 +121,9 @@ repos: # Check - id: check-docstrings name: Check Docstrings - entry: uv run --no-sync --link-mode=copy check-docstrings + entry: uv run --link-mode=copy --no-sync dfc --check language: system types: [python] + args: + - "--output=list" + - "--config=pyproject.toml" diff --git a/pyproject.toml b/pyproject.toml index bd1797a..34b0b59 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -110,6 +110,7 @@ docs = [ "mkdocstrings==0.*", "mkdocstrings-python==1.*", "pygithub==2.*", + "docstring-format-checker", ] test = [ "mypy==1.*", @@ -229,6 +230,22 @@ files = [ { file = "pyproject.toml", pattern = "version = \"{VERSION}\"" }, ] +[tool.dfc] +allow_undefined_sections = true +require_docstrings = false +check_private = true +sections = [ + { order = 1, name = "summary", type = "free_text", required = true, admonition = "note", prefix = "!!!" }, + { order = 2, name = "details", type = "free_text", required = false, admonition = "abstract", prefix = "???+" }, + { order = 3, name = "params", type = "list_name_and_type", required = false, admonition = false }, + { order = 4, name = "raises", type = "list_type", required = false, admonition = false }, + { order = 5, name = "returns", type = "list_type", required = false, admonition = false }, + { order = 6, name = "examples", type = "free_text", required = false, admonition = "example", prefix = "???+" }, + { order = 7, name = "credit", type = "free_text", required = false, admonition = "success", prefix = "???" }, + { order = 8, name = "see also", type = "free_text", required = false, admonition = "tip", prefix = "???" }, + { order = 9, name = "references", type = "free_text", required = false, admonition = "question", prefix = "???" }, +] + [build-system] requires = ["uv_build>=0.7.19,<0.8.0"] build-backend = "uv_build" diff --git a/src/toolbox_python/bools.py b/src/toolbox_python/bools.py index e8d505d..b81d229 100644 --- a/src/toolbox_python/bools.py +++ b/src/toolbox_python/bools.py @@ -88,14 +88,14 @@ def strtobool(value: str) -> bool: Convert a `#!py str` value in to a `#!py bool` value. ???+ abstract "Details" - This process is necessary because the `d`istutils` module was completely deprecated in Python 3.12. + This process is necessary because the `distutils` module was completely deprecated in Python 3.12. Params: value (str): The string value to convert. Valid input options are defined in [`STR_TO_BOOL_MAP`][toolbox_python.bools.STR_TO_BOOL_MAP] Raises: - ValueError: + (ValueError): If the value parse'ed in to `value` is not a valid value to be able to convert to a `#!py bool` value. Returns: diff --git a/src/toolbox_python/checkers.py b/src/toolbox_python/checkers.py index 61bf5ed..f210a60 100644 --- a/src/toolbox_python/checkers.py +++ b/src/toolbox_python/checkers.py @@ -138,7 +138,7 @@ def is_value_of_type(value: Any, check_type: Union[type, tuple[type, ...], list[ !!! note "Summary" Check if a given value is of a specified type or types. - ???+ info "Details" + ???+ abstract "Details" This function is used to verify if a given value matches a specified type or any of the types in a tuple of types. Params: @@ -203,7 +203,7 @@ def is_all_values_of_type( !!! note "Summary" Check if all values in an iterable are of a specified type or types. - ???+ info "Details" + ???+ abstract "Details" This function is used to verify if all values in a given iterable match a specified type or any of the types in a tuple of types. Params: @@ -270,7 +270,7 @@ def is_any_values_of_type( !!! note "Summary" Check if any value in an iterable is of a specified type or types. - ???+ info "Details" + ???+ abstract "Details" This function is used to verify if any value in a given iterable matches a specified type or any of the types in a tuple of types. Params: @@ -332,7 +332,7 @@ def is_value_in_iterable( !!! note "Summary" Check if a given value is present in an iterable. - ???+ info "Details" + ???+ abstract "Details" This function is used to verify if a given value exists within an iterable such as a list, tuple, or set. Params: @@ -342,7 +342,7 @@ def is_value_in_iterable( The iterable to check within. Raises: - TypeCheckError: + (TypeCheckError): If any of the inputs parsed to the parameters of this function are not the correct type. Uses the [`@typeguard.typechecked`](https://typeguard.readthedocs.io/en/stable/api.html#typeguard.typechecked) decorator. Returns: @@ -395,7 +395,7 @@ def is_all_values_in_iterable( !!! note "Summary" Check if all values in an iterable are present in another iterable. - ???+ info "Details" + ???+ abstract "Details" This function is used to verify if all values in a given iterable exist within another iterable. Params: @@ -405,7 +405,7 @@ def is_all_values_in_iterable( The iterable to check within. Raises: - TypeCheckError: + (TypeCheckError): If any of the inputs parsed to the parameters of this function are not the correct type. Uses the [`@typeguard.typechecked`](https://typeguard.readthedocs.io/en/stable/api.html#typeguard.typechecked) decorator. Returns: @@ -460,7 +460,7 @@ def is_any_values_in_iterable( !!! note "Summary" Check if any value in an iterable is present in another iterable. - ???+ info "Details" + ???+ abstract "Details" This function is used to verify if any value in a given iterable exists within another iterable. Params: @@ -470,7 +470,7 @@ def is_any_values_in_iterable( The iterable to check within. Raises: - TypeCheckError: + (TypeCheckError): If any of the inputs parsed to the parameters of this function are not the correct type. Uses the [`@typeguard.typechecked`](https://typeguard.readthedocs.io/en/stable/api.html#typeguard.typechecked) decorator. Returns: @@ -521,7 +521,7 @@ def is_valid_value(value: Any, op: str, target: Any) -> bool: !!! note "Summary" Check if a value is valid based on a specified operator and target. - ???+ info "Details" + ???+ abstract "Details" This function checks if a given value meets a condition defined by an operator when compared to a target value. The operator can be one of the predefined operators in the [`OPERATORS`][toolbox_python.checkers.OPERATORS] dictionary. Params: @@ -533,7 +533,7 @@ def is_valid_value(value: Any, op: str, target: Any) -> bool: The target value to compare against. Raises: - ValueError: + (ValueError): If the operator is not recognized or is not valid. Returns: @@ -603,7 +603,7 @@ def assert_value_of_type( !!! note "Summary" Assert that a given value is of a specified type or types. - ???+ info "Details" + ???+ abstract "Details" This function is used to assert that a given value matches a specified type or any of the types in a tuple of types. If the value does not match the specified type(s), a `#!py TypeError` is raised. Params: @@ -613,7 +613,7 @@ def assert_value_of_type( The type or tuple of types to check against. Raises: - TypeError: + (TypeError): If the value is not of the specified type or one of the specified types. Returns: @@ -697,7 +697,7 @@ def assert_all_values_of_type( !!! note "Summary" Assert that all values in an iterable are of a specified type or types. - ???+ info "Details" + ???+ abstract "Details" This function is used to assert that all values in a given iterable match a specified type or any of the types in a tuple of types. If any value does not match the specified type(s), a `#!py TypeError` is raised. Params: @@ -707,7 +707,7 @@ def assert_all_values_of_type( The type or tuple of types to check against. Raises: - TypeError: + (TypeError): If any value is not of the specified type or one of the specified types. Returns: @@ -796,7 +796,7 @@ def assert_any_values_of_type( !!! note "Summary" Assert that any value in an iterable is of a specified type or types. - ???+ info "Details" + ???+ abstract "Details" This function is used to assert that at least one value in a given iterable matches a specified type or any of the types in a tuple of types. If none of the values match the specified type(s), a `#!py TypeError` is raised. Params: @@ -806,7 +806,7 @@ def assert_any_values_of_type( The type or tuple of types to check against. Raises: - TypeError: + (TypeError): If none of the values are of the specified type or one of the specified types. Returns: @@ -888,7 +888,7 @@ def assert_value_in_iterable( !!! note "Summary" Assert that a given value is present in an iterable. - ???+ info "Details" + ???+ abstract "Details" This function is used to assert that a given value exists within an iterable such as a `#!py list`, `#!py tuple`, or `#!py set`. If the value is not found in the iterable, a `#!py LookupError` is raised. Params: @@ -898,7 +898,7 @@ def assert_value_in_iterable( The iterable to check within. Raises: - LookupError: + (LookupError): If the value is not found in the iterable. Returns: @@ -951,7 +951,7 @@ def assert_any_values_in_iterable( !!! note "Summary" Assert that any value in an iterable is present in another iterable. - ???+ info "Details" + ???+ abstract "Details" This function is used to assert that at least one value in a given iterable exists within another iterable. If none of the values are found in the iterable, a `#!py LookupError` is raised. Params: @@ -961,7 +961,7 @@ def assert_any_values_in_iterable( The iterable to check within. Raises: - LookupError: + (LookupError): If none of the values are found in the iterable. Returns: @@ -1016,7 +1016,7 @@ def assert_all_values_in_iterable( !!! note "Summary" Assert that all values in an iterable are present in another iterable. - ???+ info "Details" + ???+ abstract "Details" This function is used to assert that all values in a given iterable exist within another iterable. If any value is not found in the iterable, a `#!py LookupError` is raised. Params: @@ -1026,7 +1026,7 @@ def assert_all_values_in_iterable( The iterable to check within. Raises: - LookupError: + (LookupError): If any value is not found in the iterable. Returns: @@ -1079,7 +1079,7 @@ def assert_is_valid_value(value: Any, op: str, target: Any) -> None: !!! note "Summary" Assert that a value is valid based on a specified operator and target. - ???+ info "Details" + ???+ abstract "Details" This function checks if a given value meets a condition defined by an operator when compared to a target value. The operator can be one of the predefined operators in the [`OPERATORS`][toolbox_python.checkers.OPERATORS] dictionary. If the condition is not met, a `#!py ValueError` is raised. Params: @@ -1091,7 +1091,7 @@ def assert_is_valid_value(value: Any, op: str, target: Any) -> None: The target value to compare against. Raises: - ValueError: + (ValueError): If the operator is not recognized or if the value does not meet the condition defined by the operator and target. Returns: @@ -1168,7 +1168,7 @@ def any_element_contains( The string value to check exists in any of the elements in `iterable`. Raises: - TypeCheckError: + (TypeCheckError): If any of the inputs parsed to the parameters of this function are not the correct type. Uses the [`@typeguard.typechecked`](https://typeguard.readthedocs.io/en/stable/api.html#typeguard.typechecked) decorator. Returns: @@ -1225,7 +1225,7 @@ def all_elements_contains(iterable: str_collection, check: str) -> bool: The string value to check exists in any of the elements in `iterable`. Raises: - TypeCheckError: + (TypeCheckError): If any of the inputs parsed to the parameters of this function are not the correct type. Uses the [`@typeguard.typechecked`](https://typeguard.readthedocs.io/en/stable/api.html#typeguard.typechecked) decorator. Returns: @@ -1279,7 +1279,7 @@ def get_elements_containing(iterable: str_collection, check: str) -> tuple[str, The string value to check exists in any of the elements in `iterable`. Raises: - TypeCheckError: + (TypeCheckError): If any of the inputs parsed to the parameters of this function are not the correct type. Uses the [`@typeguard.typechecked`](https://typeguard.readthedocs.io/en/stable/api.html#typeguard.typechecked) decorator. Returns: diff --git a/src/toolbox_python/classes.py b/src/toolbox_python/classes.py index b8e1b6b..ffa8f19 100644 --- a/src/toolbox_python/classes.py +++ b/src/toolbox_python/classes.py @@ -187,7 +187,7 @@ class class_property(property): !!! failure "Conclusion: Failed to set a class property." - ???+ info "Notes" + ???+ abstract "Notes" - `@class_property` only works for *read-only* properties. It does not currently allow writeable/deletable properties, due to subtleties of how Python descriptors work. In order to implement such properties on a class, a metaclass for that class must be implemented. - `@class_property` is not a drop-in replacement for `property`. It is designed to be used as a class-level property, not an instance-level property. If you need to use it as an instance-level property, you will need to use the `@property` decorator instead. - `@class_property` is defined at class scope, not instance scope. This means that it is not bound to the instance of the class, but rather to the class itself; hence the name `class_property`. This means that it is designed to be used as a class-level property and is accessed through the class itself, not an instance-level property which is accessed through the instance of a class. If it is necessary to access the instance-level property, you will need to use the instance itself (eg. `instantiated_class_name._internal_attribute`) or create an instance-level property using the `@property` decorator. diff --git a/src/toolbox_python/defaults.py b/src/toolbox_python/defaults.py index b9376b9..6f86de2 100644 --- a/src/toolbox_python/defaults.py +++ b/src/toolbox_python/defaults.py @@ -204,7 +204,7 @@ def get( !!! note "Summary" From the value that is parsed in to the `value` parameter, convert it to `default` if `value` is `#!py None`, and convert it to `cast` if `cast` is not `#!py None`. - ???+ info "Details" + ???+ abstract "Details" The detailed steps will be: 1. Validate the input (using the internal [`._validate_value_and_default()`][toolbox_python.defaults.Defaults._validate_value_and_default] & [`._validate_type()`][toolbox_python.defaults.Defaults._validate_type] methods), @@ -226,7 +226,7 @@ def get( Defaults to `#!py None`. Raises: - TypeCheckError: + (TypeCheckError): If any of the inputs parsed to the parameters of this function are not the correct type. Uses the [`@typeguard.typechecked`](https://typeguard.readthedocs.io/en/stable/api.html#typeguard.typechecked) decorator. Returns: @@ -356,7 +356,7 @@ def _validate_value_and_default( Defaults to `#!py None`. Raises: - AttributeError: If both `value` and `default` are `#!py None`. + (AttributeError): If both `value` and `default` are `#!py None`. Returns: self (Defaults): @@ -387,7 +387,7 @@ def _validate_type( Defaults to `#!py None`. Raises: - AttributeError: If `check_type` is _both_ not `#!py None` _and_ if it is not one of the valid Python types. + (AttributeError): If `check_type` is _both_ not `#!py None` _and_ if it is not one of the valid Python types. Returns: self (Defaults): diff --git a/src/toolbox_python/dictionaries.py b/src/toolbox_python/dictionaries.py index 37cba4d..0687a52 100644 --- a/src/toolbox_python/dictionaries.py +++ b/src/toolbox_python/dictionaries.py @@ -68,7 +68,7 @@ def dict_reverse_keys_and_values(dictionary: dict_any) -> dict_str_any: !!! note "Summary" Take the `key` and `values` of a dictionary, and reverse them. - ???+ info "Details" + ???+ abstract "Details" This process is simple enough if the `values` are atomic types, like `#!py str`, `#!py int`, or `#!py float` types. But it is a little more tricky when the `values` are more complex types, like `#!py list` or `#!py dict`; here we need to use some recursion. Params: @@ -76,9 +76,9 @@ def dict_reverse_keys_and_values(dictionary: dict_any) -> dict_str_any: The input `#!py dict` that you'd like to have the `keys` and `values` switched. Raises: - TypeCheckError: + (TypeCheckError): If any of the inputs parsed to the parameters of this function are not the correct type. Uses the [`@typeguard.typechecked`](https://typeguard.readthedocs.io/en/stable/api.html#typeguard.typechecked) decorator. - KeyError: + (KeyError): When there are duplicate `values` being coerced to `keys` in the new dictionary. Raised because a Python `#!py dict` cannot have duplicate keys of the same value. Returns: @@ -312,7 +312,18 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: self[key] = self._convert_value(value) def _convert_value(self, value): - """Convert dictionary values recursively.""" + """ + !!! note "Summary" + Convert dictionary values recursively. + + Params: + value (Any): + The value to convert. + + Returns: + (Any): + The converted value. + """ if isinstance(value, dict): return DotDict(value) elif isinstance(value, list): @@ -320,33 +331,104 @@ def _convert_value(self, value): elif isinstance(value, tuple): return tuple(self._convert_value(item) for item in value) elif isinstance(value, set): - return set(self._convert_value(item) for item in value) + return {self._convert_value(item) for item in value} return value def __getattr__(self, key) -> Any: - """Allow dictionary keys to be accessed as attributes.""" + """ + !!! note "Summary" + Allow dictionary keys to be accessed as attributes. + + Params: + key (str): + The key to access. + + Raises: + (AttributeError): + If the key does not exist in the dictionary. + + Returns: + (Any): + The value associated with the key. + """ try: return self[key] except KeyError as e: raise AttributeError(f"Key not found: '{key}'") from e def __setattr__(self, key, value) -> None: - """Allow setting dictionary keys via attributes.""" + """ + !!! note "Summary" + Allow setting dictionary keys via attributes. + + Params: + key (str): + The key to set. + value (Any): + The value to set. + + Returns: + (None): + This function does not return a value. It sets the key-value pair in the dictionary. + """ self[key] = value def __setitem__(self, key, value) -> None: - """Intercept item setting to convert dictionaries.""" + """ + !!! note "Summary" + Intercept item setting to convert dictionaries. + + Params: + key (str): + The key to set. + value (Any): + The value to set. + + Returns: + (None): + This function does not return a value. It sets the key-value pair in the dictionary. + """ dict.__setitem__(self, key, self._convert_value(value)) def __delitem__(self, key) -> None: - """Intercept item deletion to remove keys.""" + """ + !!! note "Summary" + Intercept item deletion to remove keys. + + Params: + key (str): + The key to delete. + + Raises: + (KeyError): + If the key does not exist in the dictionary. + + Returns: + (None): + This function does not return a value. It deletes the key-value pair from the dictionary. + """ try: dict.__delitem__(self, key) except KeyError as e: raise KeyError(f"Key not found: '{key}'.") from e def __delattr__(self, key) -> None: - """Allow deleting dictionary keys via attributes.""" + """ + !!! note "Summary" + Allow deleting dictionary keys via attributes. + + Params: + key (str): + The key to delete. + + Raises: + (AttributeError): + If the key does not exist in the dictionary. + + Returns: + (None): + This function does not return a value. It deletes the key-value pair from the dictionary. + """ try: del self[key] except KeyError as e: @@ -357,7 +439,7 @@ def update(self, *args: Any, **kwargs: Any) -> None: !!! note "Summary" Override update to convert new values. - Parameters: + Params: *args (Any): Variable length argument list. **kwargs (Any): @@ -414,7 +496,7 @@ def _convert_back(obj) -> Any: elif isinstance(obj, tuple): return tuple(_convert_back(item) for item in obj) elif isinstance(obj, set): - return set(_convert_back(item) for item in obj) + return {_convert_back(item) for item in obj} return obj return _convert_back(self) diff --git a/src/toolbox_python/generators.py b/src/toolbox_python/generators.py index ff0d630..e76aaee 100644 --- a/src/toolbox_python/generators.py +++ b/src/toolbox_python/generators.py @@ -68,7 +68,7 @@ def generate_group_cutoffs( !!! note "Summary" Generate group cutoffs for a given total number and number of groups. - !!! details "Details" + !!! abstract "Details" This function divides a total number of items into a specified number of groups, returning a `#!py tuple` of `#!py tuple`'s where each inner `#!py tuple` contains the start and end indices for each group. The last group may contain fewer items if the total number is not evenly divisible by the number of groups. Params: @@ -78,9 +78,9 @@ def generate_group_cutoffs( The number of groups to create. Raises: - TypeCheckError: + (TypeCheckError): If any of the inputs parsed to the parameters of this function are not the correct type. Uses the [`@typeguard.typechecked`](https://typeguard.readthedocs.io/en/stable/api.html#typeguard.typechecked) decorator. - ValueError: + (ValueError): If `total_number` is less than 1, or if `num_groups` is less than 1, or if `total_number` is less than `num_groups`. Uses the [`assert_is_valid`][toolbox_python.checkers.assert_is_valid] function to validate the inputs. Returns: diff --git a/src/toolbox_python/lists.py b/src/toolbox_python/lists.py index e194a5b..7f9a79f 100644 --- a/src/toolbox_python/lists.py +++ b/src/toolbox_python/lists.py @@ -80,7 +80,7 @@ def flatten( !!! note "Summary" For a given `#!py list` of `#!py list`'s, flatten it out to be a single `#!py list`. - ???+ info "Details" + ???+ abstract "Details" Under the hood, this function will call the [`#!py more_itertools.collapse()`][more_itertools.collapse] function. The difference between this function and the [`#!py more_itertools.collapse()`][more_itertools.collapse] function is that the one from [`#!py more_itertools`][more_itertools] will return a `chain` object, not a `list` object. So, all we do here is call the [`#!py more_itertools.collapse()`][more_itertools.collapse] function, then parse the result in to a `#!py list()` function to ensure that the result is always a `#!py list` object. [more_itertools]: https://more-itertools.readthedocs.io/en/stable/api.html @@ -97,7 +97,7 @@ def flatten( Defaults to `#!py None`. Raises: - TypeCheckError: + (TypeCheckError): If any of the inputs parsed to the parameters of this function are not the correct type. Uses the [`@typeguard.typechecked`](https://typeguard.readthedocs.io/en/stable/api.html#typeguard.typechecked) decorator. Returns: @@ -214,7 +214,7 @@ def flat_list(*inputs: Any) -> any_list: Any input. Raises: - TypeCheckError: + (TypeCheckError): If any of the inputs parsed to the parameters of this function are not the correct type. Uses the [`@typeguard.typechecked`](https://typeguard.readthedocs.io/en/stable/api.html#typeguard.typechecked) decorator. Returns: @@ -318,7 +318,7 @@ def product(*iterables) -> list[any_tuple]: !!! note "Summary" For a given number of `#!py iterables`, perform a cartesian product on them, and return the result as a list. - ???+ info "Details" + ???+ abstract "Details" Under the hood, this function will call the [`#!py itertools.product()`][itertools.product] function. The difference between this function and the [`#!py itertools.product()`][itertools.product] function is that the one from [`#!py itertools`][itertools] will return a `product` object, not a `list` object. So, all we do here is call the [`#!py itertools.product()`][itertools.product] function, then parse the result in to a `#!py list()` function to ensure that the result is always a `#!py list` object. [itertools]: https://docs.python.org/3/library/itertools.html diff --git a/src/toolbox_python/output.py b/src/toolbox_python/output.py index 5492efb..f078f56 100644 --- a/src/toolbox_python/output.py +++ b/src/toolbox_python/output.py @@ -116,9 +116,9 @@ def print_or_log_output( Defaults to `#!py None`. Raises: - TypeCheckError: + (TypeCheckError): If any of the inputs parsed to the parameters of this function are not the correct type. Uses the [`@typeguard.typechecked`](https://typeguard.readthedocs.io/en/stable/api.html#typeguard.typechecked) decorator. - AssertError: + (AssertError): If `#!py print_or_log=="log"` and `#!py log` is not an instance of `#!py Logger`. Returns: @@ -311,11 +311,11 @@ def list_columns( Defaults to: `#!py True`. Raises: - TypeCheckError: + (TypeCheckError): If any of the inputs parsed to the parameters of this function are not the correct type. Uses the [`@typeguard.typechecked`](https://typeguard.readthedocs.io/en/stable/api.html#typeguard.typechecked) decorator. - TypeError: + (TypeError): If `#!py obj` is not a valid type. Must be one of: `#!py list`, `#!py set`, `#!py tuple`, or `#!py Generator`. - ValueError: + (ValueError): If `#!py cols_wide` is not greater than `0`, or if `#!py gap` is not greater than `0`. Returns: diff --git a/src/toolbox_python/retry.py b/src/toolbox_python/retry.py index 3a85094..2c4c789 100644 --- a/src/toolbox_python/retry.py +++ b/src/toolbox_python/retry.py @@ -157,11 +157,11 @@ def retry( Defaults to `#!py "print"`. Raises: - TypeCheckError: + (TypeCheckError): If any of the inputs parsed to the parameters of this function are not the correct type. Uses the [`@typeguard.typechecked`](https://typeguard.readthedocs.io/en/stable/api.html#typeguard.typechecked) decorator. - ValueError: + (ValueError): If either `tries` or `delay` are less than `#!py 0` - RuntimeError: + (RuntimeError): If _either_ an unexpected `#!py Exception` was thrown, which was not declared in the `exceptions` collection, _or_ if the `func` was still not able to be executed after `tries` number of iterations. Returns: diff --git a/src/toolbox_python/strings.py b/src/toolbox_python/strings.py index 6f2757d..e9d3b67 100644 --- a/src/toolbox_python/strings.py +++ b/src/toolbox_python/strings.py @@ -94,7 +94,7 @@ def str_replace( Defaults to `""`. Raises: - TypeCheckError: + (TypeCheckError): If any of the inputs parsed to the parameters of this function are not the correct type. Uses the [`@typeguard.typechecked`](https://typeguard.readthedocs.io/en/stable/api.html#typeguard.typechecked) decorator. Returns: @@ -171,7 +171,7 @@ def str_contains(check_string: str, sub_string: str) -> bool: The substring to check. Raises: - TypeCheckError: + (TypeCheckError): If any of the inputs parsed to the parameters of this function are not the correct type. Uses the [`@typeguard.typechecked`](https://typeguard.readthedocs.io/en/stable/api.html#typeguard.typechecked) decorator. Returns: @@ -239,7 +239,7 @@ def str_contains_any( The collection of substrings to check. Raises: - TypeCheckError: + (TypeCheckError): If any of the inputs parsed to the parameters of this function are not the correct type. Uses the [`@typeguard.typechecked`](https://typeguard.readthedocs.io/en/stable/api.html#typeguard.typechecked) decorator. Returns: @@ -313,7 +313,7 @@ def str_contains_all( The collection of substrings to check. Raises: - TypeCheckError: + (TypeCheckError): If any of the inputs parsed to the parameters of this function are not the correct type. Uses the [`@typeguard.typechecked`](https://typeguard.readthedocs.io/en/stable/api.html#typeguard.typechecked) decorator. Returns: @@ -396,7 +396,7 @@ def str_separate_number_chars(text: str) -> str_list: The string to split. Raises: - TypeCheckError: + (TypeCheckError): If any of the inputs parsed to the parameters of this function are not the correct type. Uses the [`@typeguard.typechecked`](https://typeguard.readthedocs.io/en/stable/api.html#typeguard.typechecked) decorator. Returns: @@ -492,7 +492,7 @@ def str_to_list(obj: Any) -> Union[str_list, Any]: The object to convert to a list if it is a string. Raises: - TypeCheckError: + (TypeCheckError): If `obj` is not a string or a list. Returns: From 5107d2d12236ecb801510a394763a12ccb99ec37 Mon Sep 17 00:00:00 2001 From: Chris Mahoney <44449504+chrimaho@users.noreply.github.com> Date: Sun, 5 Oct 2025 16:12:20 +1100 Subject: [PATCH 06/43] Refactor docstring validation and fix CLI argument handling - Replace custom docstring validation logic with external `dfc` tool for improved consistency and maintainability - Remove unused imports including `ast`, `re`, `math.e`, and various typing components - Fix CLI argument indexing from `sys.argv[1]` to `sys.argv[2]` across multiple functions to account for function name parameter - Remove complex `FunctionAndClassDetails` class and associated validation methods - Simplify `check_docstrings()` function to use single `dfc --output=table ./src/toolbox_python` command - Add centralised CLI execution logic with function name resolution and error handling - Eliminate over 290 lines of custom docstring parsing and validation code --- src/utils/scripts.py | 355 +++---------------------------------------- 1 file changed, 24 insertions(+), 331 deletions(-) diff --git a/src/utils/scripts.py b/src/utils/scripts.py index e4502b2..635b5f0 100644 --- a/src/utils/scripts.py +++ b/src/utils/scripts.py @@ -4,13 +4,10 @@ # ## Python StdLib Imports ---- -import ast -import re import subprocess import sys -from math import e from pathlib import Path -from typing import Literal, NamedTuple, Union +from typing import Union ## --------------------------------------------------------------------------- # @@ -141,7 +138,7 @@ def check() -> None: check_codespell() check_pylint() check_pycln() - check_docstrings_dir("src/toolbox_python") + check_docstrings() check_mkdocs() check_build() check_pytest() @@ -209,10 +206,10 @@ def git_update_version(version: str) -> None: def git_update_version_cli() -> None: - if len(sys.argv) < 2: + if len(sys.argv) < 3: print("Requires argument: ") sys.exit(1) - git_update_version(sys.argv[1]) + git_update_version(sys.argv[2]) def git_fix_tag_reference(version: str) -> None: @@ -229,10 +226,10 @@ def git_fix_tag_reference(version: str) -> None: def git_fix_tag_reference_cli() -> None: - if len(sys.argv) < 2: + if len(sys.argv) < 3: print("Requires argument: ") sys.exit(1) - git_fix_tag_reference(sys.argv[1]) + git_fix_tag_reference(sys.argv[2]) ## --------------------------------------------------------------------------- # @@ -260,10 +257,10 @@ def docs_build_versioned(version: str) -> None: def docs_build_versioned_cli() -> None: - if len(sys.argv) < 2: + if len(sys.argv) < 3: print("Requires argument: ") sys.exit(1) - docs_build_versioned(sys.argv[1]) + docs_build_versioned(sys.argv[2]) def update_git_docs(version: str) -> None: @@ -278,10 +275,10 @@ def update_git_docs(version: str) -> None: def update_git_docs_cli() -> None: - if len(sys.argv) < 2: + if len(sys.argv) < 3: print("Requires argument: ") sys.exit(1) - update_git_docs(sys.argv[1]) + update_git_docs(sys.argv[2]) def docs_check_versions() -> None: @@ -309,10 +306,10 @@ def build_static_docs(version: str) -> None: def build_static_docs_cli() -> None: - if len(sys.argv) < 2: + if len(sys.argv) < 3: print("Requires argument: ") sys.exit(1) - build_static_docs(sys.argv[1]) + build_static_docs(sys.argv[2]) def build_versioned_docs(version: str) -> None: @@ -321,10 +318,10 @@ def build_versioned_docs(version: str) -> None: def build_versioned_docs_cli() -> None: - if len(sys.argv) < 2: + if len(sys.argv) < 3: print("Requires argument: ") sys.exit(1) - build_versioned_docs(sys.argv[1]) + build_versioned_docs(sys.argv[2]) ## --------------------------------------------------------------------------- # @@ -332,324 +329,20 @@ def build_versioned_docs_cli() -> None: ## --------------------------------------------------------------------------- # -class FunctionAndClassDetails(NamedTuple): - item_type: Literal["function", "class"] - name: str - node: Union[ast.FunctionDef, ast.ClassDef] - lineno: int - - -def check_docstrings_file(file: str) -> None: - """ - Check docstrings in a Python file for completeness and correct formatting. - - This function performs extensive validation of docstrings according to the - project's documentation standards. - """ - file_path = Path(file) - if not file_path.exists(): - raise FileNotFoundError(f"File not found: {file}") - - if not file_path.suffix == ".py": - raise ValueError(f"File must be a Python file (.py): {file}") - - # Read and parse the file - with open(file_path, encoding="utf-8") as f: - content = f.read() - - try: - tree = ast.parse(content) - except SyntaxError as e: - raise SyntaxError(f"Invalid Python syntax in {file}: {e}") - - # Extract all functions and classes with docstrings - functions_and_classes: list[FunctionAndClassDetails] = [] - - class DocstringVisitor(ast.NodeVisitor): - def visit_FunctionDef(self, node): - if node.name.startswith("_"): # Skip private functions - return - functions_and_classes.append(FunctionAndClassDetails("function", node.name, node, node.lineno)) - self.generic_visit(node) - - def visit_ClassDef(self, node): - if node.name.startswith("_"): # Skip private classes - return - functions_and_classes.append(FunctionAndClassDetails("class", node.name, node, node.lineno)) - self.generic_visit(node) - - visitor = DocstringVisitor() - visitor.visit(tree) - - errors = [] - - for item_type, name, node, lineno in functions_and_classes: - try: - _check_single_docstring(item_type, name, node, lineno, file) - except Exception as e: - errors.append(f"Line {lineno}, {item_type} '{name}': {str(e)}") - - if errors: - error_msg = f"Docstring validation errors in {file}:\n" + "\n".join(errors) - raise RuntimeError(error_msg) - - print(f"✓ All docstrings are valid in: '{file}'") - - -def _check_single_docstring( - item_type: Literal["function", "class"], - name: str, - node: Union[ast.FunctionDef, ast.ClassDef], - lineno: int, - file: str, -) -> None: - """Check a single function or class docstring.""" - docstring = ast.get_docstring(node) - - if not docstring: - # raise ValueError(f"Missing docstring") - return # Skip if no docstring is present - - # Required sections in order - required_sections = ["summary", "params", "returns_or_yields", "examples"] - optional_sections = [ - "details", - "raises", - "credit", - "equation", - "notes", - "references", - "see_also", - ] +def check_docstrings() -> None: + run("dfc --output=table ./src/toolbox_python") - # Check for mandatory sections - if not re.search(r'!!! note "Summary"', docstring, re.IGNORECASE): - raise ValueError('Missing mandatory Summary section: `!!! note "Summary"`') - - # Check Params section - if item_type == "function" and isinstance(node, ast.FunctionDef): - # Get function parameters (excluding 'self' for methods) - params = [arg.arg for arg in node.args.args if arg.arg != "self"] - if params and not re.search(r"Params:", docstring): - raise ValueError("Missing mandatory Params section for function with parameters") - - # Check each parameter is documented - if params: - for param in params: - param_pattern = rf"{param}\s*\([^)]+\):" - if not re.search(param_pattern, docstring): - raise ValueError(f"Parameter '{param}' not documented or incorrectly formatted") - - # Check Returns or Yields (but not both) - has_returns = re.search(r"Returns:", docstring) - has_yields = re.search(r"Yields:", docstring) - - if has_returns and has_yields: - raise ValueError("Docstring cannot have both Returns and Yields sections") - - if not has_returns and not has_yields: - if item_type == "function": - raise ValueError("Missing mandatory Returns or Yields section") - - # Check mandatory Examples section - if not re.search(r'\?\?\?\+ example "Examples"', docstring, re.IGNORECASE): - raise ValueError('Missing mandatory Examples section: `???+ example "Examples"`') - - # Validate section order - _check_section_order(docstring) - - # Validate specific section formats - _validate_section_formats(docstring, name) - - -def _check_section_order(docstring: str) -> None: - """Check that sections appear in the correct order.""" - section_patterns = [ - (r'!!! note "Summary"', "Summary"), - (r'!!! details "Details"', "Details"), - (r"Params:", "Params"), - (r"Raises:", "Raises"), - (r"Returns:", "Returns"), - (r"Yields:", "Yields"), - (r'\?\?\?+ example "Examples"', "Examples"), - (r'\?\?\?\+ success "Credit"', "Credit"), - (r'\?\?\?\+ calculation "Equation"', "Equation"), - (r'\?\?\?\+ info "Notes"', "Notes"), - (r'\?\?\? question "References"', "References"), - (r'\?\?\? tip "See Also"', "See Also"), - ] - found_sections = [] - for pattern, section_name in section_patterns: - match = re.search(pattern, docstring, re.IGNORECASE) - if match: - found_sections.append((match.start(), section_name)) - - # Sort by position in docstring - found_sections.sort(key=lambda x: x[0]) - - # Check order matches expected order - expected_order = [ - "Summary", - "Details", - "Params", - "Raises", - "Returns", - "Yields", - "Examples", - "Credit", - "Equation", - "Notes", - "References", - "See Also", - ] +## --------------------------------------------------------------------------- # +## Execute #### +## --------------------------------------------------------------------------- # - last_expected_index = -1 - for _, section_name in found_sections: - try: - current_index = expected_order.index(section_name) - if current_index < last_expected_index: - raise ValueError(f"Section '{section_name}' appears out of order") - last_expected_index = current_index - except ValueError: - # Section not in expected order list - this shouldn't happen - pass - - -def _validate_section_formats(docstring: str, name: str) -> None: - """Validate the format of specific sections.""" - - # Check Summary is single paragraph - summary_match = re.search( - r'!!! note "Summary"\s*\n\s*(.+?)(?=\n\s*\n|\n\s*[!?])', - docstring, - re.DOTALL | re.IGNORECASE, - ) - if summary_match: - summary_text = summary_match.group(1).strip() - # Check if summary has multiple paragraphs (contains double newlines) - if "\n\n" in summary_text or re.search(r"\n\s*\n", summary_text): - raise ValueError("Summary section should be a single paragraph") - - # Validate Params format - if re.search(r"Params:", docstring): - # Find Params section - params_match = re.search( - r"Params:\s*\n(.*?)(?=\n\s*(?:Raises|Returns|Yields|Examples|!!!|\?\?\?))", - docstring, - re.DOTALL, - ) - if params_match: - params_content = params_match.group(1) - # Check each parameter follows the format: param_name (type): - param_lines = [line for line in params_content.split("\n") if line.strip()] - for line in param_lines: - if not line.startswith(" "): # Parameter name line - if not re.match(r"\w+\s*\([^)]+\):", line): - raise ValueError(f"Invalid parameter format: '{line}'. Expected: 'param_name (type):'") - - # Validate Raises format - if re.search(r"Raises:", docstring): - raises_match = re.search( - r"Raises:\s*\n(.*?)(?=\n\s*(?:Returns|Yields|Examples|!!!|\?\?\?))", - docstring, - re.DOTALL, - ) - if raises_match: - raises_content = raises_match.group(1) - # Check each exception follows the format: ExceptionType: - exception_lines = [line for line in raises_content.split("\n") if line.strip()] - for line in exception_lines: - if not line.startswith(" "): # Exception name line - if not re.match( - r"\w+Error:|TypeError:|ValueError:|RuntimeError:|Exception:", - line, - ): - # Allow common exception patterns - if not line.endswith(":"): - raise ValueError(f"Invalid exception format: '{line}'. Expected: 'ExceptionType:'") - - # Validate Returns/Yields format - returns_or_yields = re.search(r"(Returns|Yields):", docstring) - if returns_or_yields: - section_name = returns_or_yields.group(1) - section_match = re.search( - rf"{section_name}:\s*\n(.*?)(?=\n\s*(?:Examples|!!!|\?\?\?))", - docstring, - re.DOTALL, - ) - if section_match: - section_content = section_match.group(1) - # Check format: optional_name (type): - return_lines = [line for line in section_content.split("\n") if line.strip()] - for line in return_lines: - if not line.startswith(" "): # Return value line - if not re.match(r"(\w+\s*)?\([^)]+\):", line): - raise ValueError( - f"Invalid {section_name} format: '{line}'. Expected: 'name (type):' or '(type):'" - ) - - -def check_docstrings_cli() -> None: - """Command line interface for docstring checking.""" - if len(sys.argv) < 2: - print("Usage: python scripts.py ") - sys.exit(1) - print(f"Checking docstrings in {sys.argv[1]}...") +if __name__ == "__main__": - try: - check_docstrings_file(sys.argv[1]) - except Exception as e: - print(f"Error: {e}") + function_name: str = sys.argv[1].lower().replace("-", "_") + if function_name not in globals(): + print(f"Function not found: '{function_name}'.") sys.exit(1) - -def check_docstrings_all() -> None: - """Check docstrings in all Python files in the src directory.""" - - python_files: list[str] = get_all_files(".py") - if not python_files: - print("No Python files found to check.") - return - else: - print(f"\nChecking docstrings in all Python files... Files to check: '{len(python_files)}'.") - - errors = [] - - for file in python_files: - try: - check_docstrings_file(file) - except Exception as e: - errors.append(str(e)) - - if errors: - error_msg = "Docstring validation errors:\n" + "\n".join(errors) - raise RuntimeError(error_msg) - - print("✓ All docstrings are valid across all files.") - - -def check_docstrings_dir(dir: str) -> None: - """Check docstrings in all Python files in the src directory.""" - - python_files: list[str] = [file for file in get_all_files(".py") if file.startswith(dir)] - if not python_files: - print("No Python files found to check.") - return - else: - print(f"\nChecking docstrings in all Python files in '{dir}'... Files to check: '{len(python_files)}'.") - - errors = [] - - for file in python_files: - try: - check_docstrings_file(file) - except Exception as e: - errors.append(str(e)) - - if errors: - error_msg = "Docstring validation errors:\n" + "\n".join(errors) - raise RuntimeError(error_msg) - - print("✓ All docstrings are valid across all files.") + globals()[function_name]() From e4e76760de06b85eb998c9e8db24539693218ce1 Mon Sep 17 00:00:00 2001 From: Chris Mahoney <44449504+chrimaho@users.noreply.github.com> Date: Sun, 5 Oct 2025 16:12:53 +1100 Subject: [PATCH 07/43] Comment out project scripts in `pyproject.toml` - Temporarily disable all console script entry points to prevent command-line tool conflicts - Preserve script definitions for future re-enablement by commenting rather than removing - Affects syncing, linting, checking, git operations, and documentation scripts - Maintains project configuration structure whilst removing executable commands --- pyproject.toml | 90 +++++++++++++++++++++++++------------------------- 1 file changed, 45 insertions(+), 45 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 34b0b59..2a2fbda 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,51 +38,51 @@ Repository = "https://github.com/data-science-extensions/toolbox-python" Changelog = "https://github.com/data-science-extensions/toolbox-python/releases" Issues = "https://github.com/data-science-extensions/toolbox-python/issues" -[project.scripts] -# Syncing -sync = "utils.scripts:uv_sync" -# Linting -run-black = "utils.scripts:run_black" -run-blacken_docs = "utils.scripts:run_blacken_docs" -run-isort = "utils.scripts:run_isort" -run-pycln = "utils.scripts:run_pycln" -run-pyupgrade = "utils.scripts:run_pyupgrade" -lint = "utils.scripts:lint" -# Checking -check-black = "utils.scripts:check_black" -check-blacken_docs = "utils.scripts:check_blacken_docs" -check-mypy = "utils.scripts:check_mypy" -check-isort = "utils.scripts:check_isort" -check-codespell = "utils.scripts:check_codespell" -check-pylint = "utils.scripts:check_pylint" -check-pycln = "utils.scripts:check_pycln" -check-build = "utils.scripts:check_build" -check-mkdocs = "utils.scripts:check_mkdocs" -check-pytest = "utils.scripts:check_pytest" -check-docstrings = "utils.scripts:check_docstrings_cli" -check = "utils.scripts:check" -lint-check = "utils.scripts:lint_check" -# Git -add-git-credentials = "utils.scripts:add_git_credentials" -git-switch-to-main-branch = "utils.scripts:git_switch_to_main_branch" -git-switch-to-docs-branch = "utils.scripts:git_switch_to_docs_branch" -git-refresh-current-branch = "utils.scripts:git_refresh_current_branch" -git-add-coverage-report = "utils.scripts:git_add_coverage_report" -bump-version = "utils.bump_version:main" -git-update-version = "utils.scripts:git_update_version_cli" -git-fix-tag-reference = "utils.scripts:git_fix_tag_reference_cli" -# Docs -docs-serve-static = "utils.scripts:docs_serve_static" -docs-serve-versioned = "utils.scripts:docs_serve_versioned" -docs-build-static = "utils.scripts:docs_build_static" -docs-build-versioned = "utils.scripts:docs_build_versioned_cli" -update-git-docs = "utils.scripts:update_git_docs_cli" -docs-check-versions = "utils.scripts:docs_check_versions" -docs-delete-version = "utils.scripts:docs_delete_version_cli" -docs-set-default = "utils.scripts:docs_set_default" -build-static-docs = "utils.scripts:build_static_docs_cli" -build-versioned-docs = "utils.scripts:build_versioned_docs_cli" -generate-changelog = "utils.changelog:main" +# [project.scripts] +# # Syncing +# sync = "utils.scripts:uv_sync" +# # Linting +# run-black = "utils.scripts:run_black" +# run-blacken_docs = "utils.scripts:run_blacken_docs" +# run-isort = "utils.scripts:run_isort" +# run-pycln = "utils.scripts:run_pycln" +# run-pyupgrade = "utils.scripts:run_pyupgrade" +# lint = "utils.scripts:lint" +# # Checking +# check-black = "utils.scripts:check_black" +# check-blacken_docs = "utils.scripts:check_blacken_docs" +# check-mypy = "utils.scripts:check_mypy" +# check-isort = "utils.scripts:check_isort" +# check-codespell = "utils.scripts:check_codespell" +# check-pylint = "utils.scripts:check_pylint" +# check-pycln = "utils.scripts:check_pycln" +# check-build = "utils.scripts:check_build" +# check-mkdocs = "utils.scripts:check_mkdocs" +# check-pytest = "utils.scripts:check_pytest" +# check-docstrings = "utils.scripts:check_docstrings_cli" +# check = "utils.scripts:check" +# lint-check = "utils.scripts:lint_check" +# # Git +# add-git-credentials = "utils.scripts:add_git_credentials" +# git-switch-to-main-branch = "utils.scripts:git_switch_to_main_branch" +# git-switch-to-docs-branch = "utils.scripts:git_switch_to_docs_branch" +# git-refresh-current-branch = "utils.scripts:git_refresh_current_branch" +# git-add-coverage-report = "utils.scripts:git_add_coverage_report" +# bump-version = "utils.bump_version:main" +# git-update-version = "utils.scripts:git_update_version_cli" +# git-fix-tag-reference = "utils.scripts:git_fix_tag_reference_cli" +# # Docs +# docs-serve-static = "utils.scripts:docs_serve_static" +# docs-serve-versioned = "utils.scripts:docs_serve_versioned" +# docs-build-static = "utils.scripts:docs_build_static" +# docs-build-versioned = "utils.scripts:docs_build_versioned_cli" +# update-git-docs = "utils.scripts:update_git_docs_cli" +# docs-check-versions = "utils.scripts:docs_check_versions" +# docs-delete-version = "utils.scripts:docs_delete_version_cli" +# docs-set-default = "utils.scripts:docs_set_default" +# build-static-docs = "utils.scripts:build_static_docs_cli" +# build-versioned-docs = "utils.scripts:build_versioned_docs_cli" +# generate-changelog = "utils.changelog:main" [dependency-groups] dev = [ From ff3dba7e7ab7eb4dde9351494568ef2670db1e7c Mon Sep 17 00:00:00 2001 From: Chris Mahoney <44449504+chrimaho@users.noreply.github.com> Date: Sun, 5 Oct 2025 16:14:16 +1100 Subject: [PATCH 08/43] Standardise GitHub Actions workflow configuration - Consolidate permissions at workflow level instead of per-job for better maintainability - Update all action versions to latest stable releases for improved security and features - Replace hardcoded values with environment variables for better configurability - Enable PyPI package publishing by uncommenting the publish step - Improve package installation verification with explicit version constraints - Reorganise job execution order by moving tag reference fix after package upload - Add python-version-file configuration for consistent Python version management - Correct CI job matrix to run on specified operating systems instead of ubuntu-latest only - Enhance environment variable coverage for tokens, repository details, and build settings --- .github/workflows/cd.yml | 116 +++++++++++++++++++++------------------ .github/workflows/ci.yml | 21 ++----- 2 files changed, 68 insertions(+), 69 deletions(-) diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index eb5afa3..6e709fb 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -11,14 +11,27 @@ on: # Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued. # However, do NOT cancel in-progress runs as we want to allow these production deployments to complete. concurrency: - group: "pages" + group: "deployment" cancel-in-progress: true +permissions: + contents: write # <-- to allow assets to be uploaded to the release + id-token: write # <-- to allow access to the tokens + pages: write # <-- to allow publishing to GitHub Pages + env: VERSION: ${{ github.event.release.tag_name }} + PACKAGE_NAME: toolbox-python UV_LINK_MODE: copy - UV_NATIVE_TLS: true UV_NO_SYNC: true + UV_INDEX_STRATEGY: unsafe-best-match + GITHUB_ACTOR: ${{ github.actor }} + PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + REPOSITORY_NAME: data-science-extensions/toolbox-python + GIT_BRANCH: ${{ github.event.release.target_commitish }} + PYTHON_VERSION: '3.13' jobs: @@ -109,25 +122,22 @@ jobs: if: ${{ always() }} runs-on: ubuntu-latest - permissions: - contents: write #<-- to allow push changes to the repository - steps: - name: Checkout repository id: checkout-repository - uses: actions/checkout@v4 + uses: actions/checkout@v5 with: - ref: main + ref: ${{ env.GIT_BRANCH }} - - name: Set up uv - uses: astral-sh/setup-uv@v5 + - name: Set up UV + uses: astral-sh/setup-uv@v6 - name: Set up Python id: setup-python uses: actions/setup-python@v5 with: - python-version: '3.13' + python-version: ${{ env.PYTHON_VERSION }} - name: Install dependencies id: install-dependencies @@ -152,9 +162,9 @@ jobs: - name: Upload coverage id: upload-coverage - uses: codecov/codecov-action@v4 + uses: codecov/codecov-action@v5 with: - token: ${{ secrets.CODECOV_TOKEN }} + token: ${{ env.CODECOV_TOKEN }} files: ./cov-report/xml/cov-report.xml verbose: true @@ -165,25 +175,22 @@ jobs: if: ${{ always() && needs.test.result == 'success' }} runs-on: ubuntu-latest - permissions: - contents: write #<-- to allow assets to be uploaded to the release - steps: - name: Checkout repository id: checkout-repository - uses: actions/checkout@v4 + uses: actions/checkout@v5 with: - ref: main + ref: ${{ env.GIT_BRANCH }} - - name: Set up uv - uses: astral-sh/setup-uv@v5 + - name: Set up UV + uses: astral-sh/setup-uv@v6 - name: Setup Python id: setup-python uses: actions/setup-python@v5 with: - python-version: '3.13' + python-version: ${{ env.PYTHON_VERSION }} - name: Check VERSION id: check-version @@ -211,10 +218,6 @@ jobs: id: update-git-version run: uv run git-update-version ${VERSION} - - name: Fix tag reference - id: fix-tag-reference - run: uv run git-fix-tag-reference ${VERSION} - - name: Build package id: build-package run: uv build --out-dir=dist @@ -234,28 +237,34 @@ jobs: retention-days: 1 overwrite: true + - name: Fix tag reference + id: fix-tag-reference + run: uv run git-fix-tag-reference ${VERSION} + deploy-package: name: Deploy to PyPI needs: build-package + if: ${{ always() && needs.build-package.result == 'success' }} runs-on: ubuntu-latest steps: - name: Checkout repository id: checkout-repository - uses: actions/checkout@v4 + uses: actions/checkout@v5 with: - ref: main + ref: ${{ env.GIT_BRANCH }} - - name: Set up uv - uses: astral-sh/setup-uv@v5 + - name: Set up UV + uses: astral-sh/setup-uv@v6 - name: Setup Python id: setup-python uses: actions/setup-python@v5 with: - python-version: '3.13' + python-version: ${{ env.PYTHON_VERSION }} + python-version-file: pyproject.toml - name: Download artifacts id: download-artifacts @@ -264,41 +273,38 @@ jobs: name: dist path: dist - # - name: Publish package - # id: publish-package - # env: - # PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }} - # run: uv publish --token ${PYPI_TOKEN} + - name: Publish package + id: publish-package + env: + PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }} + run: uv publish --token ${PYPI_TOKEN} - name: Check id: check run: | - echo 'Package deployed to PyPI 👉 https://pypi.org/project/toolbox-python/' - uvx pip install --dry-run --no-deps --no-cache toolbox-python + echo 'Package deployed to PyPI 👉 https://pypi.org/project/${{ env.PACKAGE_NAME }}' + uvx pip install --dry-run --no-deps --no-cache ${{ env.PACKAGE_NAME }} install-package: - name: Install Package on '${{ matrix.os }}' with '${{ matrix.python-version }}' needs: deploy-package - if: ${{ always() && needs.deploy-package.result == 'success' }} - strategy: matrix: os: ['ubuntu-latest', 'macos-latest', 'windows-latest'] python-version: ['3.9', '3.10', '3.11', '3.12', '3.13'] fail-fast: false max-parallel: 15 - + name: Install Package on '${{ matrix.os }}' with '${{ matrix.python-version }}' runs-on: ${{ matrix.os }} steps: - name: Checkout repository id: checkout-repository - uses: actions/checkout@v4 + uses: actions/checkout@v5 with: - ref: main + ref: ${{ env.GIT_BRANCH }} - name: Setup Python id: setup-python @@ -308,36 +314,34 @@ jobs: - name: Install package id: install-package - run: pip install --no-cache toolbox-python + run: pip install --no-cache --verbose --no-python-version-warning "${{ env.PACKAGE_NAME }}==${{ env.VERSION }}" build-docs: - name: Build Docs needs: - test - deploy-package if: ${{ always() && needs.test.result == 'success' && needs.deploy-package.result == 'success' }} + name: Build Docs runs-on: ubuntu-latest - permissions: - contents: write #<-- to allow mike to push to the repository - steps: - name: Checkout repository id: checkout-repository - uses: actions/checkout@v4 + uses: actions/checkout@v5 with: - ref: main + ref: ${{ env.GIT_BRANCH }} - - name: Set up uv - uses: astral-sh/setup-uv@v5 + - name: Set up UV + uses: astral-sh/setup-uv@v6 - name: Setup Python id: setup-python uses: actions/setup-python@v5 with: - python-version: '3.13' + python-version: ${{ env.PYTHON_VERSION }} + python-version-file: pyproject.toml - name: Install dependencies id: install-dependencies @@ -355,8 +359,8 @@ jobs: - name: Generate ChangeLog id: generate-changelog env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - REPOSITORY_NAME: data-science-extensions/toolbox-python + GITHUB_TOKEN: ${{ env.GITHUB_TOKEN }} + REPOSITORY_NAME: ${{ env.REPOSITORY_NAME }} run: uv run generate-changelog - name: Commit ChangeLog @@ -370,3 +374,7 @@ jobs: - name: Build docs id: build-docs run: uv run build-versioned-docs ${VERSION} + + - name: Fix tag reference + id: fix-tag-reference + run: uv run ./src/utils/scripts.py git_fix_tag_reference_cli ${VERSION} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1f9f9c1..55f45fe 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -89,50 +89,41 @@ jobs: git branch check: - if: github.ref_type == 'branch' && github.event_name == 'push' && github.ref_name != 'main' name: Run checks runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@v4 - + uses: actions/checkout@v5 - name: Set up uv - uses: astral-sh/setup-uv@v5 - + uses: astral-sh/setup-uv@v6 - name: Set up Python uses: actions/setup-python@v5 with: python-version-file: "pyproject.toml" - - name: Install dependencies run: uv sync --no-cache --all-groups - - name: Run checks run: uv run check ci: - if: github.event_name == 'pull_request' && github.base_ref == 'main' - name: Run Checks on '${{ matrix.os }}' with '${{ matrix.python-version }}' - runs-on: ubuntu-latest - strategy: matrix: os: ['ubuntu-latest', 'macos-latest', 'windows-latest'] python-version: ['3.9', '3.10', '3.11', '3.12', '3.13'] fail-fast: false max-parallel: 15 + name: Run Checks on '${{ matrix.os }}' with '${{ matrix.python-version }}' + runs-on: ${{ matrix.os }} steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@v5 - name: Set up uv - uses: astral-sh/setup-uv@v5 + uses: astral-sh/setup-uv@v6 - name: Set up Python uses: actions/setup-python@v5 From b408c3950194652a6ac1ce0da14cd5150684f4f4 Mon Sep 17 00:00:00 2001 From: Chris Mahoney <44449504+chrimaho@users.noreply.github.com> Date: Mon, 6 Oct 2025 16:42:43 +1100 Subject: [PATCH 09/43] Migrate docstring format checker to pre-commit hook - Replace local docstring checking implementation with external pre-commit repository - Use `docstring-format-checker` from `data-science-extensions` organisation at version 1.3.0 - Maintain same configuration and output format whilst leveraging standardised tooling - Comment out previous local implementation to preserve configuration for reference --- .pre-commit-config.yaml | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ed0d12a..75020ae 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -103,6 +103,17 @@ repos: # args: # - "--all-groups" + # Check docstrings + - repo: https://github.com/data-science-extensions/docstring-format-checker + rev: "v1.3.0" + hooks: + - id: docstring-format-checker + name: Docstring Format Checker + args: + - "--config=pyproject.toml" + - "--output=list" + - "--check" + # Everything run locally - repo: local hooks: @@ -119,11 +130,11 @@ repos: - "-sn" # Don't display the score # Check - - id: check-docstrings - name: Check Docstrings - entry: uv run --link-mode=copy --no-sync dfc --check - language: system - types: [python] - args: - - "--output=list" - - "--config=pyproject.toml" + # - id: check-docstrings + # name: Check Docstrings + # entry: uv run --link-mode=copy --no-sync dfc --check + # language: system + # types: [python] + # args: + # - "--output=list" + # - "--config=pyproject.toml" From 1fb978e07c21cb2ce727fb3daf1ddaa58201e29c Mon Sep 17 00:00:00 2001 From: Chris Mahoney <44449504+chrimaho@users.noreply.github.com> Date: Mon, 6 Oct 2025 16:46:33 +1100 Subject: [PATCH 10/43] Refactor type annotations and improve commit formatting - Replace union type syntax with `Optional` and `Literal` imports for better compatibility - Enhance commit message processing to filter out co-authored-by lines and empty lines - Update commit output format to include short SHA with link and improved author attribution - Fix release title reference to use `name` property instead of deprecated `title` - Add type ignore comment for repository retrieval to suppress type checker warnings - Include main execution guard for proper script entry point handling --- src/utils/changelog.py | 55 ++++++++++++++++++++++++++++++------------ 1 file changed, 40 insertions(+), 15 deletions(-) diff --git a/src/utils/changelog.py b/src/utils/changelog.py index 6a8c373..73508e0 100644 --- a/src/utils/changelog.py +++ b/src/utils/changelog.py @@ -23,6 +23,7 @@ import os import re from pathlib import Path +from typing import Literal, Optional # ## Python Third Party Imports ---- from github import Auth, Github @@ -39,8 +40,8 @@ ### Environment Variables ---- -TOKEN: str | None = os.environ.get("GITHUB_TOKEN") -REPOSITORY_NAME: str | None = os.environ.get("REPOSITORY_NAME") +TOKEN: Optional[str] = os.environ.get("GITHUB_TOKEN") +REPOSITORY_NAME: Optional[str] = os.environ.get("REPOSITORY_NAME") if TOKEN is None: raise RuntimeError("Environment variable `GITHUB_TOKEN` is not set. Please set it before running the script.") if REPOSITORY_NAME is None: @@ -48,13 +49,13 @@ ### Static ---- -OUTPUT_FILENAME: str = "CHANGELOG.md" +OUTPUT_FILENAME: Literal["CHANGELOG.md"] = "CHANGELOG.md" OUTPUT_FILEPATH: Path = Path(OUTPUT_FILENAME) AUTH: Token = Auth.Token(TOKEN) -NEW_LINE: str = "\n" -BLANK_LINE: str = "\n\n" -LINE_BREAK: str = "
" -TAB: str = " " +NEW_LINE: Literal["\n"] = "\n" +BLANK_LINE: Literal["\n\n"] = "\n\n" +LINE_BREAK: Literal["
"] = "
" +TAB: Literal[" "] = " " # ---------------------------------------------------------------------------- # @@ -125,7 +126,7 @@ def add_release_info(release: GitRelease, repo: Repository) -> str: return ( f'!!! info "{release.tag_name}"{NEW_LINE}' f"{NEW_LINE}" - f"{TAB}## **{release.title}**{BLANK_LINE}" + f"{TAB}## **{release.name}**{BLANK_LINE}" f"{TAB}{LINE_BREAK}{NEW_LINE}" f"{TAB}{LINE_BREAK}{NEW_LINE}" f"{TAB}{BLANK_LINE}" @@ -153,13 +154,26 @@ def add_commit_info(commit: Commit) -> str: # NOTE: We write the commit message to the output file. # We format the commit message to replace newlines with `{LINE_BREAK}` tags for better readability in Markdown. # We also include the author's login and a link to their GitHub profile, as well as a link to the commit itself. - commit_message: str = commit.commit.message.replace(BLANK_LINE, NEW_LINE).replace( - NEW_LINE, f"{LINE_BREAK}{NEW_LINE}{TAB * 3}" - ) + + commit_message_list: list[str] = [] + for idx, line in enumerate(commit.commit.message.split(NEW_LINE)): + if idx == 0: + commit_message_list.append(line.strip()) + elif line.strip() == "": + continue + elif line.lower().startswith("co-authored-by:"): + continue + else: + commit_message_list.append(line.strip()) + + commit_message_str: str = "\n".join(commit_message_list) + commit_message_str: str = commit_message_str.replace(NEW_LINE, f"{LINE_BREAK}{NEW_LINE}{TAB * 3}") + return ( - f"{TAB * 2}* {commit_message}" - f" (by [{commit.author.login if commit.author else ''}]({commit.author.html_url if commit.author else ''}))" - f" [View]({commit.html_url}){BLANK_LINE}" + f"{TAB * 2}* [`{commit.sha[:7]}`]({commit.html_url}): {commit_message_str}" + f"{NEW_LINE}" + f"{TAB * 3}(by [{commit.author.login if commit.author else ''}]({commit.author.html_url if commit.author else ''}))" + f"{NEW_LINE}" ) @@ -184,7 +198,7 @@ def main() -> None: with Github(auth=AUTH) as g, open(OUTPUT_FILENAME, "w") as f: ### Get the repository ---- - REPO: Repository = g.get_repo(REPOSITORY_NAME) + REPO: Repository = g.get_repo(REPOSITORY_NAME) # type: ignore ### Write the header to the output file ---- f.write(add_header(REPO)) @@ -253,3 +267,14 @@ def main() -> None: ### Add a newline after each release section ---- f.write(f"{BLANK_LINE}") + + +# ---------------------------------------------------------------------------- # +# # +# Execute #### +# # +# ---------------------------------------------------------------------------- # + + +if __name__ == "__main__": + main() From baf5032f0f752874119ea019e6f553043288ecfd Mon Sep 17 00:00:00 2001 From: Chris Mahoney <44449504+chrimaho@users.noreply.github.com> Date: Sun, 28 Dec 2025 17:50:19 +1100 Subject: [PATCH 11/43] Replace `mypy` with `ty` for type checking - Replace `mypy` with `ty` in the `test` dependency group to standardise type checking - Remove the `[tool.mypy]` configuration block - Rename the `check_mypy()` function to the `check_ty()` function - Update the `check_ty()` function to execute the `ty check` command using a dynamic directory path - Update the `check()` function to call the `check_ty()` function --- .pre-commit-config.yaml | 19 ++++++++----------- pyproject.toml | 11 +---------- src/utils/scripts.py | 13 ++++++------- 3 files changed, 15 insertions(+), 28 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 75020ae..f05cbdd 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -46,17 +46,6 @@ repos: additional_dependencies: - "black>=23.3" - # Run MyPy type checks - - repo: https://github.com/pre-commit/mirrors-mypy - rev: "v1.16.1" - hooks: - - id: mypy - files: src/toolbox_python - args: - - "--install-types" - - "--config-file=pyproject.toml" - - "--allow-redefinition" - # Reorder Python imports - repo: https://github.com/pycqa/isort rev: "6.0.1" @@ -129,6 +118,14 @@ repos: - "-rn" # Only display messages - "-sn" # Don't display the score + - id: ty + name: ty-check + entry: uv run ty check ./src/toolbox_python + language: python + types: [python] + pass_filenames: true + exclude: ^src/tests/.*$ + # Check # - id: check-docstrings # name: Check Docstrings diff --git a/pyproject.toml b/pyproject.toml index 2a2fbda..8e3035e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -113,7 +113,7 @@ docs = [ "docstring-format-checker", ] test = [ - "mypy==1.*", + "ty==0.*", "parameterized==0.*", "pytest==8.*", "pytest-clarity==1.*", @@ -160,15 +160,6 @@ testpaths = [ "src/tests", ] -[tool.mypy] -ignore_missing_imports = true -pretty = true -disable_error_code = [ - "valid-type", - "attr-defined", - "no-redef", -] - [tool.isort] import_heading_future = "## Future Python Library Imports ----" import_heading_stdlib = "## Python StdLib Imports ----" diff --git a/src/utils/scripts.py b/src/utils/scripts.py index 635b5f0..ac31efc 100644 --- a/src/utils/scripts.py +++ b/src/utils/scripts.py @@ -90,13 +90,12 @@ def check_blacken_docs() -> None: run("blacken-docs --check", *get_all_files(".md", ".py", ".ipynb")) -def check_mypy() -> None: +def check_ty() -> None: run( - "mypy", - "--install-types", - "--non-interactive", - "--config-file=pyproject.toml", - "./src/toolbox_python", + "ty", + "check", + # "--config-file=pyproject.toml", + f"./src/{DIRECTORY_NAME}", ) @@ -133,7 +132,7 @@ def check_pytest() -> None: def check() -> None: check_black() check_blacken_docs() - check_mypy() + check_ty() check_isort() check_codespell() check_pylint() From 31dc7764c495d08a6bf5b6d49043d915c695056b Mon Sep 17 00:00:00 2001 From: Chris Mahoney <44449504+chrimaho@users.noreply.github.com> Date: Sun, 28 Dec 2025 17:51:17 +1100 Subject: [PATCH 12/43] Bump package versions in the `pre-commit` config file --- .pre-commit-config.yaml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f05cbdd..6e2c1dd 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -5,7 +5,7 @@ repos: # Fixes - repo: https://github.com/pre-commit/pre-commit-hooks - rev: "v5.0.0" + rev: "v6.0.0" hooks: # File name fixes - id: check-case-conflict @@ -33,14 +33,14 @@ repos: # Linter - repo: https://github.com/psf/black - rev: "25.1.0" + rev: "25.12.0" hooks: - id: black language_version: python3.13 args: - "--config=pyproject.toml" - repo: https://github.com/adamchainz/blacken-docs - rev: "1.19.1" + rev: "1.20.0" hooks: - id: blacken-docs additional_dependencies: @@ -48,7 +48,7 @@ repos: # Reorder Python imports - repo: https://github.com/pycqa/isort - rev: "6.0.1" + rev: "7.0.0" hooks: - id: isort name: isort (python) @@ -57,7 +57,7 @@ repos: # Find any outdated syntax and replace with modern equivalents - repo: https://github.com/asottile/pyupgrade - rev: "v3.20.0" + rev: "v3.21.2" hooks: - id: pyupgrade name: Upgrade Python features @@ -77,7 +77,7 @@ repos: # Remove unused import statements - repo: https://github.com/hadialqattan/pycln - rev: "v2.5.0" + rev: "v2.6.0" hooks: - id: pycln args: @@ -85,7 +85,7 @@ repos: # Check uv configs - repo: https://github.com/astral-sh/uv-pre-commit - rev: "0.7.20" + rev: "0.9.18" hooks: - id: uv-lock - id: uv-sync @@ -94,7 +94,7 @@ repos: # Check docstrings - repo: https://github.com/data-science-extensions/docstring-format-checker - rev: "v1.3.0" + rev: "v1.9.0" hooks: - id: docstring-format-checker name: Docstring Format Checker From 0609a0173544ae743138314c828d327e1452baf1 Mon Sep 17 00:00:00 2001 From: Chris Mahoney <44449504+chrimaho@users.noreply.github.com> Date: Sun, 28 Dec 2025 17:51:41 +1100 Subject: [PATCH 13/43] Clean up outdated checks in the `pre-commit` config file --- .pre-commit-config.yaml | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6e2c1dd..009c1d6 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -125,13 +125,3 @@ repos: types: [python] pass_filenames: true exclude: ^src/tests/.*$ - - # Check - # - id: check-docstrings - # name: Check Docstrings - # entry: uv run --link-mode=copy --no-sync dfc --check - # language: system - # types: [python] - # args: - # - "--output=list" - # - "--config=pyproject.toml" From 167398f359e51d02e743c973e0521be96b989694 Mon Sep 17 00:00:00 2001 From: Chris Mahoney <44449504+chrimaho@users.noreply.github.com> Date: Sun, 28 Dec 2025 17:53:49 +1100 Subject: [PATCH 14/43] Eliminate redundant scripts from the `pyproject.toml` file, prefer the `src/utils/scripts.py` module --- pyproject.toml | 46 ---------------------------------------------- 1 file changed, 46 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 8e3035e..bc8d4dc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,52 +38,6 @@ Repository = "https://github.com/data-science-extensions/toolbox-python" Changelog = "https://github.com/data-science-extensions/toolbox-python/releases" Issues = "https://github.com/data-science-extensions/toolbox-python/issues" -# [project.scripts] -# # Syncing -# sync = "utils.scripts:uv_sync" -# # Linting -# run-black = "utils.scripts:run_black" -# run-blacken_docs = "utils.scripts:run_blacken_docs" -# run-isort = "utils.scripts:run_isort" -# run-pycln = "utils.scripts:run_pycln" -# run-pyupgrade = "utils.scripts:run_pyupgrade" -# lint = "utils.scripts:lint" -# # Checking -# check-black = "utils.scripts:check_black" -# check-blacken_docs = "utils.scripts:check_blacken_docs" -# check-mypy = "utils.scripts:check_mypy" -# check-isort = "utils.scripts:check_isort" -# check-codespell = "utils.scripts:check_codespell" -# check-pylint = "utils.scripts:check_pylint" -# check-pycln = "utils.scripts:check_pycln" -# check-build = "utils.scripts:check_build" -# check-mkdocs = "utils.scripts:check_mkdocs" -# check-pytest = "utils.scripts:check_pytest" -# check-docstrings = "utils.scripts:check_docstrings_cli" -# check = "utils.scripts:check" -# lint-check = "utils.scripts:lint_check" -# # Git -# add-git-credentials = "utils.scripts:add_git_credentials" -# git-switch-to-main-branch = "utils.scripts:git_switch_to_main_branch" -# git-switch-to-docs-branch = "utils.scripts:git_switch_to_docs_branch" -# git-refresh-current-branch = "utils.scripts:git_refresh_current_branch" -# git-add-coverage-report = "utils.scripts:git_add_coverage_report" -# bump-version = "utils.bump_version:main" -# git-update-version = "utils.scripts:git_update_version_cli" -# git-fix-tag-reference = "utils.scripts:git_fix_tag_reference_cli" -# # Docs -# docs-serve-static = "utils.scripts:docs_serve_static" -# docs-serve-versioned = "utils.scripts:docs_serve_versioned" -# docs-build-static = "utils.scripts:docs_build_static" -# docs-build-versioned = "utils.scripts:docs_build_versioned_cli" -# update-git-docs = "utils.scripts:update_git_docs_cli" -# docs-check-versions = "utils.scripts:docs_check_versions" -# docs-delete-version = "utils.scripts:docs_delete_version_cli" -# docs-set-default = "utils.scripts:docs_set_default" -# build-static-docs = "utils.scripts:build_static_docs_cli" -# build-versioned-docs = "utils.scripts:build_versioned_docs_cli" -# generate-changelog = "utils.changelog:main" - [dependency-groups] dev = [ "black==25.*", From b0da3ddce5d747c3a304943de94fdd2e316535fd Mon Sep 17 00:00:00 2001 From: Chris Mahoney <44449504+chrimaho@users.noreply.github.com> Date: Sun, 28 Dec 2025 17:57:43 +1100 Subject: [PATCH 15/43] Add `complexipy` for code complexity analysis - Define the `check_complexity()` function to automate quality checks and analyse code complexity levels to guide developers - Configure the `[tool.complexipy]` section to establish analysis parameters that enforce coding standards - Set the `max-complexity-allowed` parameter to 15 to ensure code remains maintainable --- pyproject.toml | 8 ++++++++ src/utils/scripts.py | 12 ++++++++++++ 2 files changed, 20 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index bc8d4dc..6d18ab3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -168,6 +168,14 @@ disable = [ "E1137", # unsupported-assignment-operation ] +[tool.complexipy] +paths = "src/toolbox_python" +max-complexity-allowed = 15 +quiet = false +ignore-complexity = true +details = "normal" +sort = "asc" + [tool.bump_version.replacements] files = [ { file = "src/toolbox_python/__init__.py", pattern = "__version__ = \"{VERSION}\"" }, diff --git a/src/utils/scripts.py b/src/utils/scripts.py index ac31efc..6836ab2 100644 --- a/src/utils/scripts.py +++ b/src/utils/scripts.py @@ -128,6 +128,18 @@ def check_mkdocs() -> None: def check_pytest() -> None: run("pytest --config-file=pyproject.toml") +def check_complexity() -> None: + notes: str = dedent( + """ + Notes from: https://rohaquinlop.github.io/complexipy/#running-the-analysis + - Complexity <= 5: Simple, easy to understand + - Complexity 6-15: Moderate, acceptable for most cases + - Complexity >= 15: Complex, consider refactoring into simpler functions + """ + ) + print(notes) + run(f"complexipy ./src/{DIRECTORY_NAME}") + def check() -> None: check_black() From 008cdf5fc7f59b8cf98690f3d42e8afff89f94c0 Mon Sep 17 00:00:00 2001 From: Chris Mahoney <44449504+chrimaho@users.noreply.github.com> Date: Sun, 28 Dec 2025 18:03:32 +1100 Subject: [PATCH 16/43] Update pre-commit hook exclusions - Expand the exclusion pattern in `.pre-commit-config.yaml` to include the `src/utils/` and `src/tests` directories. - Utilise a multi-line regex format to manage multiple excluded paths more effectively. --- .pre-commit-config.yaml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 009c1d6..c111feb 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -102,6 +102,11 @@ repos: - "--config=pyproject.toml" - "--output=list" - "--check" + exclude: | + (?x)^( + src/tests/.*$| + src/utils/.*$ + ) # Everything run locally - repo: local @@ -124,4 +129,8 @@ repos: language: python types: [python] pass_filenames: true - exclude: ^src/tests/.*$ + exclude: | + (?x)^( + src/tests/.*$| + src/utils/.*$ + ) From 6dcf1cd12520d8462893769e8306546245de859a Mon Sep 17 00:00:00 2001 From: Chris Mahoney <44449504+chrimaho@users.noreply.github.com> Date: Sun, 28 Dec 2025 18:07:18 +1100 Subject: [PATCH 17/43] Update utility scripts to use dynamic package name constants. - Define `PACKAGE_NAME` and `DIRECTORY_NAME` constants to centralise configuration - Update `check_pylint()` function to use the dynamic `DIRECTORY_NAME` constant - Refine `check_pycln()` function to target the specific package directory --- src/utils/scripts.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/src/utils/scripts.py b/src/utils/scripts.py index 6836ab2..220e0ee 100644 --- a/src/utils/scripts.py +++ b/src/utils/scripts.py @@ -10,6 +10,17 @@ from typing import Union +## --------------------------------------------------------------------------- # +## Constants #### +## --------------------------------------------------------------------------- # + + +PACKAGE_NAME: str = "toolbox-python" +'''PACKAGE_NAME="toolbox-python"''' +DIRECTORY_NAME: str = PACKAGE_NAME.replace("-", "_") +'''DIRECTORY_NAME="toolbox_python"''' + + ## --------------------------------------------------------------------------- # ## Generic #### ## --------------------------------------------------------------------------- # @@ -108,11 +119,11 @@ def check_codespell() -> None: def check_pylint() -> None: - run("pylint --rcfile=pyproject.toml src/toolbox_python") + run(f"pylint --rcfile=pyproject.toml src/{DIRECTORY_NAME}") def check_pycln() -> None: - run("pycln --check --config=pyproject.toml src/") + run(f"pycln --check --config=pyproject.toml src/{DIRECTORY_NAME}") def check_build() -> None: From 1781bf05fe00ceda8fe7fb2c88f26ee6f05f0063 Mon Sep 17 00:00:00 2001 From: Chris Mahoney <44449504+chrimaho@users.noreply.github.com> Date: Sun, 28 Dec 2025 18:09:00 +1100 Subject: [PATCH 18/43] Refactor and generalise `check_docstrings()` - Relocate the `check_docstrings()` function to improve logical file structure - Update the `check_docstrings()` function to reference the `DIRECTORY_NAME` constant - Standardise the source path within the `check_docstrings()` function to support dynamic directory names --- src/utils/scripts.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/src/utils/scripts.py b/src/utils/scripts.py index 220e0ee..b4cec28 100644 --- a/src/utils/scripts.py +++ b/src/utils/scripts.py @@ -139,6 +139,11 @@ def check_mkdocs() -> None: def check_pytest() -> None: run("pytest --config-file=pyproject.toml") + +def check_docstrings() -> None: + run(f"dfc --output=table ./src/{DIRECTORY_NAME}") + + def check_complexity() -> None: notes: str = dedent( """ @@ -346,15 +351,6 @@ def build_versioned_docs_cli() -> None: build_versioned_docs(sys.argv[2]) -## --------------------------------------------------------------------------- # -## Docstrings #### -## --------------------------------------------------------------------------- # - - -def check_docstrings() -> None: - run("dfc --output=table ./src/toolbox_python") - - ## --------------------------------------------------------------------------- # ## Execute #### ## --------------------------------------------------------------------------- # From af5d00cc24fd09357f6e5cee2b084e14fd89df85 Mon Sep 17 00:00:00 2001 From: Chris Mahoney <44449504+chrimaho@users.noreply.github.com> Date: Sun, 28 Dec 2025 18:10:56 +1100 Subject: [PATCH 19/43] Reorder checks and add complexity validation - Move `check_pylint()` function after `check_pycln()` function to improve linting workflow - Add `check_complexity()` function to the `check()` function sequence to monitor code quality - Prioritise `check_pytest()` function execution by moving it before documentation and build checks --- src/utils/scripts.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/utils/scripts.py b/src/utils/scripts.py index b4cec28..79dd0e9 100644 --- a/src/utils/scripts.py +++ b/src/utils/scripts.py @@ -7,6 +7,7 @@ import subprocess import sys from pathlib import Path +from textwrap import dedent from typing import Union @@ -163,12 +164,13 @@ def check() -> None: check_ty() check_isort() check_codespell() - check_pylint() check_pycln() + check_pylint() + check_complexity() check_docstrings() + check_pytest() check_mkdocs() check_build() - check_pytest() ## --------------------------------------------------------------------------- # From b51e693e0e7df939e1205b92061a564d0ee88c09 Mon Sep 17 00:00:00 2001 From: Chris Mahoney <44449504+chrimaho@users.noreply.github.com> Date: Sun, 28 Dec 2025 18:11:51 +1100 Subject: [PATCH 20/43] Fix missing `complexipy` dependency --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 6d18ab3..0f1e502 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -76,6 +76,7 @@ test = [ "pytest-sugar==1.*", "pytest-xdist==3.*", "requests==2.*", + "complexipy==4.*", ] [tool.black] From 43c039df0c88f425b292259daff88bc2caf81c3a Mon Sep 17 00:00:00 2001 From: Chris Mahoney <44449504+chrimaho@users.noreply.github.com> Date: Sun, 28 Dec 2025 18:16:30 +1100 Subject: [PATCH 21/43] Update `check_type` variadic `tuple` hints - Correct the `tuple` type hint to include the ellipsis `...` to properly represent variadic tuples in the docstrings. - Ensure the `check_type` parameter documentation accurately reflects that multiple types can be provided. - Standardise the docstrings for the `is_value_of_type()`, `is_all_values_of_type()`, `is_any_values_of_type()`, `assert_value_of_type()`, `assert_all_values_of_type()`, and `assert_any_values_of_type()` functions. --- src/toolbox_python/checkers.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/toolbox_python/checkers.py b/src/toolbox_python/checkers.py index f210a60..70451c1 100644 --- a/src/toolbox_python/checkers.py +++ b/src/toolbox_python/checkers.py @@ -144,7 +144,7 @@ def is_value_of_type(value: Any, check_type: Union[type, tuple[type, ...], list[ Params: value (Any): The value to check. - check_type (Union[type, tuple[type], list[type]]): + check_type (Union[type, tuple[type, ...], list[type]]): The type or tuple of types to check against. Returns: @@ -209,7 +209,7 @@ def is_all_values_of_type( Params: values (any_collection): The iterable containing values to check. - check_type (Union[type, tuple[type], list[type]]): + check_type (Union[type, tuple[type, ...], list[type]]): The type or tuple of types to check against. Returns: @@ -276,7 +276,7 @@ def is_any_values_of_type( Params: values (any_collection): The iterable containing values to check. - check_type (Union[type, tuple[type], list[type]]): + check_type (Union[type, tuple[type, ...], list[type]]): The type or tuple of types to check against. Returns: @@ -609,7 +609,7 @@ def assert_value_of_type( Params: value (Any): The value to check. - check_type (Union[type, tuple[type], list[type]]): + check_type (Union[type, tuple[type, ...], list[type]]): The type or tuple of types to check against. Raises: @@ -703,7 +703,7 @@ def assert_all_values_of_type( Params: values (any_collection): The iterable containing values to check. - check_type (Union[type, tuple[type], list[type]]): + check_type (Union[type, tuple[type, ...], list[type]]): The type or tuple of types to check against. Raises: @@ -802,7 +802,7 @@ def assert_any_values_of_type( Params: values (any_collection): The iterable containing values to check. - check_type (Union[type, tuple[type], list[type]]): + check_type (Union[type, tuple[type, ...], list[type]]): The type or tuple of types to check against. Raises: From 4415ef485290be6ef790febb8da0fae8607d7675 Mon Sep 17 00:00:00 2001 From: Chris Mahoney <44449504+chrimaho@users.noreply.github.com> Date: Sun, 28 Dec 2025 18:19:58 +1100 Subject: [PATCH 22/43] Standardise type hints and improve type checks - Update type annotations to use the `Optional()` and `Union()` classes for better compatibility within the `Defaults()` class. - Replace the `is_type()` function call with the standard `isinstance()` function within the `.get()` method to verify string values. - Standardise method signatures for the `._validate_value_and_default()` and `._validate_type()` methods to use explicit typing constructs. --- src/toolbox_python/defaults.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/toolbox_python/defaults.py b/src/toolbox_python/defaults.py index 6f86de2..8172401 100644 --- a/src/toolbox_python/defaults.py +++ b/src/toolbox_python/defaults.py @@ -40,7 +40,7 @@ from __future__ import annotations # ## Python StdLib Imports ---- -from typing import Any +from typing import Any, Optional, Union # ## Python Third Party Imports ---- from typeguard import typechecked @@ -197,8 +197,8 @@ def __call__(self, *args, **kwargs) -> Any: def get( self, value: Any, - default: Any | None = None, - cast: str | type | None = None, + default: Optional[Any] = None, + cast: Optional[Union[str, type]] = None, ) -> Any: """ !!! note "Summary" @@ -330,7 +330,7 @@ def get( if value is None: value = default if cast is not None: - if (cast is bool or cast == "bool") and is_type(value, str): + if (cast is bool or cast == "bool") and isinstance(value, str): value = bool(strtobool(value)) elif isinstance(cast, str): value = eval(cast)(value) @@ -340,8 +340,8 @@ def get( def _validate_value_and_default( self, - value: Any | None = None, - default: Any | None = None, + value: Optional[Any] = None, + default: Optional[Any] = None, ) -> Defaults: """ !!! note "Summary" @@ -374,7 +374,7 @@ def _validate_value_and_default( def _validate_type( self, - check_type: str | type | None = None, + check_type: Optional[Union[str, type]] = None, ) -> Defaults: """ !!! note "Summary" From f2d915217a91cf838d4950ca9a16cf1b69f86841 Mon Sep 17 00:00:00 2001 From: Chris Mahoney <44449504+chrimaho@users.noreply.github.com> Date: Sun, 28 Dec 2025 18:21:58 +1100 Subject: [PATCH 23/43] Add new `validators` utility - Introduce `Validators()` class to centralise logic for checking if numeric values fall within specified bounds - Implement `._value_is_between()` method to perform individual range checks and validate that boundary arguments are logical - Provide `._assert_value_is_between()` method to raise an `AssertionError` when a value violates the defined range - Include `._all_values_are_between()` method and `._assert_all_values_are_between()` method to facilitate bulk validation of sequences - Add unit tests to verify correct behaviour and error handling for the new `Validators()` class --- src/tests/test_validators.py | 95 ++++++++++++++++++++ src/toolbox_python/validators.py | 149 +++++++++++++++++++++++++++++++ 2 files changed, 244 insertions(+) create mode 100644 src/tests/test_validators.py create mode 100644 src/toolbox_python/validators.py diff --git a/src/tests/test_validators.py b/src/tests/test_validators.py new file mode 100644 index 0000000..5e11afa --- /dev/null +++ b/src/tests/test_validators.py @@ -0,0 +1,95 @@ +# ---------------------------------------------------------------------------- # +# # +# Setup #### +# # +# ---------------------------------------------------------------------------- # + + +## --------------------------------------------------------------------------- # +## Imports #### +## --------------------------------------------------------------------------- # + + +# ## Python StdLib Imports ---- +from unittest import TestCase + +# ## Python Third Party Imports ---- +from parameterized import parameterized +from pytest import raises + +# ## Local First Party Imports ---- +from tests.setup import name_func_predefined_name +from toolbox_python.validators import Validators + + +# ---------------------------------------------------------------------------- # +# # +# Test Suite #### +# # +# ---------------------------------------------------------------------------- # + + +class TestValidators(TestCase): + + def setUp(self) -> None: + pass + + ## ----------------------------------------------------------------------- # + ## _value_is_between #### + ## ----------------------------------------------------------------------- # + + @parameterized.expand( + input=( + ("within_range", 5, 0, 10, True), + ("at_min", 0, 0, 10, True), + ("at_max", 10, 0, 10, True), + ("below_range", -1, 0, 10, False), + ("above_range", 11, 0, 10, False), + ("float_within", 5.5, 5.0, 6.0, True), + ), + name_func=name_func_predefined_name, + ) + def test_value_is_between(self, _name, value, min_val, max_val, expected) -> None: + assert Validators._value_is_between(value, min_val, max_val) == expected + + def test_value_is_between_raises(self) -> None: + with raises(ValueError, match="Invalid range"): + Validators._value_is_between(5, 10, 0) + + ## ----------------------------------------------------------------------- # + ## _assert_value_is_between #### + ## ----------------------------------------------------------------------- # + + def test_assert_value_is_between_valid(self) -> None: + Validators._assert_value_is_between(5, 0, 10) + + def test_assert_value_is_between_invalid(self) -> None: + with raises(AssertionError, match="Invalid Value"): + Validators._assert_value_is_between(11, 0, 10) + + ## ----------------------------------------------------------------------- # + ## _all_values_are_between #### + ## ----------------------------------------------------------------------- # + + @parameterized.expand( + input=( + ("all_within", [1, 2, 3], 0, 5, True), + ("one_below", [-1, 2, 3], 0, 5, False), + ("one_above", [1, 2, 6], 0, 5, False), + ("empty_list", [], 0, 5, True), + ), + name_func=name_func_predefined_name, + ) + def test_all_values_are_between(self, _name, values, min_val, max_val, expected) -> None: + assert Validators._all_values_are_between(values, min_val, max_val) == expected + + ## ----------------------------------------------------------------------- # + ## _assert_all_values_are_between #### + ## ----------------------------------------------------------------------- # + + def test_assert_all_values_are_between_valid(self) -> None: + Validators._assert_all_values_are_between([1, 2, 3], 0, 5) + + def test_assert_all_values_are_between_invalid(self) -> None: + with raises(AssertionError, match="Values not between"): + Validators._assert_all_values_are_between([1, 6, -1], 0, 5) diff --git a/src/toolbox_python/validators.py b/src/toolbox_python/validators.py new file mode 100644 index 0000000..e024309 --- /dev/null +++ b/src/toolbox_python/validators.py @@ -0,0 +1,149 @@ +# ============================================================================ # +# # +# Title: Validators Utility Module # +# Purpose: Provides validation functions and classes for numeric ranges # +# # +# ============================================================================ # + + +# ---------------------------------------------------------------------------- # +# # +# Setup #### +# # +# ---------------------------------------------------------------------------- # + + +## --------------------------------------------------------------------------- # +## Imports #### +## --------------------------------------------------------------------------- # + + +# ## Future Python Library Imports ---- +from __future__ import annotations + +# ## Python StdLib Imports ---- +from collections.abc import Sequence +from numbers import Real + +# ## Local First Party Imports ---- +from toolbox_python.checkers import is_valid + + +## --------------------------------------------------------------------------- # +## Exports #### +## --------------------------------------------------------------------------- # + + +__all__: list[str] = ["Validators"] + + +# ---------------------------------------------------------------------------- # +# # +# Validators #### +# # +# ---------------------------------------------------------------------------- # + + +class Validators: + + @staticmethod + def _value_is_between(value: Real, min_value: Real, max_value: Real) -> bool: + """ + !!! note "Summary" + Check if a value is between two other values. + + Params: + value (Real): + The value to check. + min_value (Real): + The minimum value. + max_value (Real): + The maximum value. + + Returns: + (bool): + True if the value is between the minimum and maximum values, False otherwise. + """ + if not is_valid(min_value, "<=", max_value): + raise ValueError( + f"Invalid range: min_value `{min_value}` must be less than or equal to max_value `{max_value}`" + ) + result: bool = is_valid(value, ">=", min_value) and is_valid(value, "<=", max_value) + return result + + @staticmethod + def _assert_value_is_between( + value: Real, + min_value: Real, + max_value: Real, + ) -> None: + """ + !!! note "Summary" + Assert that a value is between two other values. + + Params: + value (Real): + The value to check. + min_value (Real): + The minimum value. + max_value (Real): + The maximum value. + + Raises: + (AssertionError): + If the value is not between the minimum and maximum values. + """ + if not Validators._value_is_between(value, min_value, max_value): + raise AssertionError(f"Invalid Value: `{value}`. Must be between `{min_value}` and `{max_value}`") + + @staticmethod + def _all_values_are_between( + values: Sequence[Real], + min_value: Real, + max_value: Real, + ) -> bool: + """ + !!! note "Summary" + Check if all values in an array are between two other values. + + Params: + values (Sequence[Real]): + The array of values to check. + min_value (Real): + The minimum value. + max_value (Real): + The maximum value. + + Returns: + (bool): + True if all values are between the minimum and maximum values, False otherwise. + """ + return all(Validators._value_is_between(value, min_value, max_value) for value in values) + + @staticmethod + def _assert_all_values_are_between( + values: Sequence[Real], + min_value: Real, + max_value: Real, + ) -> None: + """ + !!! note "Summary" + Assert that all values in an array are between two other values. + + Params: + values (Sequence[Real]): + The array of values to check. + min_value (Real): + The minimum value. + max_value (Real): + The maximum value. + + Raises: + (AssertionError): + If any value is not between the minimum and maximum values. + """ + values_not_between: list[Real] = [ + value for value in values if not Validators._value_is_between(value, min_value, max_value) + ] + if not len(values_not_between) == 0: + raise AssertionError(f"Values not between `{min_value}` and `{max_value}`: {values_not_between}") From 9bb4c8bc401e474a975a083b592adfec26c19208 Mon Sep 17 00:00:00 2001 From: Chris Mahoney <44449504+chrimaho@users.noreply.github.com> Date: Sun, 28 Dec 2025 18:23:15 +1100 Subject: [PATCH 24/43] Add missing typehints to the functions and methods in the `dictionaries` module --- src/toolbox_python/dictionaries.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/toolbox_python/dictionaries.py b/src/toolbox_python/dictionaries.py index 0687a52..e0a500a 100644 --- a/src/toolbox_python/dictionaries.py +++ b/src/toolbox_python/dictionaries.py @@ -311,7 +311,7 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: for key, value in d.items(): self[key] = self._convert_value(value) - def _convert_value(self, value): + def _convert_value(self, value: Any): """ !!! note "Summary" Convert dictionary values recursively. @@ -334,7 +334,7 @@ def _convert_value(self, value): return {self._convert_value(item) for item in value} return value - def __getattr__(self, key) -> Any: + def __getattr__(self, key: str) -> Any: """ !!! note "Summary" Allow dictionary keys to be accessed as attributes. @@ -356,7 +356,7 @@ def __getattr__(self, key) -> Any: except KeyError as e: raise AttributeError(f"Key not found: '{key}'") from e - def __setattr__(self, key, value) -> None: + def __setattr__(self, key: str, value: Any) -> None: """ !!! note "Summary" Allow setting dictionary keys via attributes. @@ -373,7 +373,7 @@ def __setattr__(self, key, value) -> None: """ self[key] = value - def __setitem__(self, key, value) -> None: + def __setitem__(self, key: str, value: Any) -> None: """ !!! note "Summary" Intercept item setting to convert dictionaries. @@ -390,7 +390,7 @@ def __setitem__(self, key, value) -> None: """ dict.__setitem__(self, key, self._convert_value(value)) - def __delitem__(self, key) -> None: + def __delitem__(self, key: str) -> None: """ !!! note "Summary" Intercept item deletion to remove keys. @@ -412,7 +412,7 @@ def __delitem__(self, key) -> None: except KeyError as e: raise KeyError(f"Key not found: '{key}'.") from e - def __delattr__(self, key) -> None: + def __delattr__(self, key: str) -> None: """ !!! note "Summary" Allow deleting dictionary keys via attributes. From 02acdeb5ccf745484423fdecd3fbfc20dde766b2 Mon Sep 17 00:00:00 2001 From: Chris Mahoney <44449504+chrimaho@users.noreply.github.com> Date: Sun, 28 Dec 2025 18:23:55 +1100 Subject: [PATCH 25/43] Fix docsting error in the `dictionaries` module - Related to the use of `*` in the Params section --- src/toolbox_python/dictionaries.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/toolbox_python/dictionaries.py b/src/toolbox_python/dictionaries.py index e0a500a..bdd4a43 100644 --- a/src/toolbox_python/dictionaries.py +++ b/src/toolbox_python/dictionaries.py @@ -440,9 +440,9 @@ def update(self, *args: Any, **kwargs: Any) -> None: Override update to convert new values. Params: - *args (Any): + args (Any): Variable length argument list. - **kwargs (Any): + kwargs (Any): Arbitrary keyword arguments. Returns: From 1eeacc8e0840b32beeb6816fab8287469c8dcddd Mon Sep 17 00:00:00 2001 From: Chris Mahoney <44449504+chrimaho@users.noreply.github.com> Date: Sun, 28 Dec 2025 18:26:13 +1100 Subject: [PATCH 26/43] In the `lists` module, fix types in docstrings to match the function signatures --- src/toolbox_python/lists.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/toolbox_python/lists.py b/src/toolbox_python/lists.py index 7f9a79f..efcb906 100644 --- a/src/toolbox_python/lists.py +++ b/src/toolbox_python/lists.py @@ -49,7 +49,6 @@ # ## Local First Party Imports ---- from toolbox_python.collection_types import ( any_list, - any_tuple, collection, scalar, str_list, @@ -87,7 +86,7 @@ def flatten( [more_itertools.collapse]: https://more-itertools.readthedocs.io/en/stable/api.html#more_itertools.collapse Params: - list_of_lists (list[any_list]): + list_of_lists (Union[scalar, collection]): The input `#!py list` of `#!py list`'s that you'd like to flatten to a single-level `#!py list`. base_type (Optional[type], optional): Binary and text strings are not considered iterable and will not be collapsed. To avoid collapsing other types, specify `base_type`.
@@ -313,7 +312,7 @@ def flat_list(*inputs: Any) -> any_list: return flatten(list(inputs)) -def product(*iterables) -> list[any_tuple]: +def product(*iterables: Any) -> list[tuple[Any, ...]]: """ !!! note "Summary" For a given number of `#!py iterables`, perform a cartesian product on them, and return the result as a list. From 3ea392b3540ed26c3b182d97ce93e640798bfff5 Mon Sep 17 00:00:00 2001 From: Chris Mahoney <44449504+chrimaho@users.noreply.github.com> Date: Sun, 28 Dec 2025 18:27:24 +1100 Subject: [PATCH 27/43] Fix tuple type hint for nested parameters - Update the `tuple()` class type hint within the `name_func_nested_list()` function to include an ellipsis for variable-length support. --- src/tests/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tests/setup.py b/src/tests/setup.py index 8ddf091..1a3537d 100644 --- a/src/tests/setup.py +++ b/src/tests/setup.py @@ -45,7 +45,7 @@ def name_func_flat_list( def name_func_nested_list( func: Callable, idx: int, - params: Union[list[any_list_tuple,], tuple[any_list_tuple,]], + params: Union[list[any_list_tuple,], tuple[any_list_tuple, ...]], ) -> str: return f"{func.__name__}_{int(idx)+1:02}_{params[0][0]}_{params[0][1]}" From 48da0dd0482a86ec8d5cafeb7aeb38ef508502a8 Mon Sep 17 00:00:00 2001 From: Chris Mahoney <44449504+chrimaho@users.noreply.github.com> Date: Sun, 28 Dec 2025 18:27:43 +1100 Subject: [PATCH 28/43] Fix formatting --- src/tests/test_classes.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/tests/test_classes.py b/src/tests/test_classes.py index e415e6b..3103943 100644 --- a/src/tests/test_classes.py +++ b/src/tests/test_classes.py @@ -146,9 +146,7 @@ def test_class_property_with_parens(self) -> None: def test_class_property_with_doc(self) -> None: assert MyClass.class_value_with_doc == "original with doc" - MyClass.class_value_with_doc.__doc__ == ( - "This is a class property with a docstring." - ) + MyClass.class_value_with_doc.__doc__ == ("This is a class property with a docstring.") def test_class_property_errors(self) -> None: with raises(AttributeError): From 9ececa76977ff4f7e53a998704a2259cbf1e5839 Mon Sep 17 00:00:00 2001 From: Chris Mahoney <44449504+chrimaho@users.noreply.github.com> Date: Sun, 28 Dec 2025 18:29:44 +1100 Subject: [PATCH 29/43] Remove Python version upper bound - Update `requires-python` to remove the `<4.0` restriction - Standardise the configuration to improve forward compatibility --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 0f1e502..1567310 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,7 +25,7 @@ classifiers = [ "Programming Language :: Python :: 3.13", "Intended Audience :: Developers", ] -requires-python = ">=3.9,<4.0" +requires-python = ">=3.9" dependencies = [ "typeguard==4.*", "more-itertools==10.*", From e1d12401598cdda1e0a46e58d8a0a6d0f4589324 Mon Sep 17 00:00:00 2001 From: Chris Mahoney <44449504+chrimaho@users.noreply.github.com> Date: Sun, 28 Dec 2025 18:45:45 +1100 Subject: [PATCH 30/43] Standardise package metadata and versioning - Replace hardcoded version strings with dynamic retrieval via the `metadata()` function - Remove the custom `bump_version.py` utility and its configuration in the `pyproject.toml` file - Update the `CD` workflow to use the native `uv version` command - Delete the `test_version.py` file as hardcoded version checks are no longer necessary - Expand the `__init__.py` file to dynamically expose package metadata such as version and author --- .github/workflows/cd.yml | 2 +- pyproject.toml | 7 -- src/tests/test_version.py | 40 --------- src/toolbox_python/__init__.py | 21 ++++- src/utils/bump_version.py | 143 --------------------------------- 5 files changed, 20 insertions(+), 193 deletions(-) delete mode 100644 src/tests/test_version.py delete mode 100644 src/utils/bump_version.py diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index 6e709fb..71b6cc1 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -212,7 +212,7 @@ jobs: - name: Bump version id: bump-version - run: uv run bump-version --verbose=true ${VERSION} + run: uv version ${VERSION} - name: Update Git Version id: update-git-version diff --git a/pyproject.toml b/pyproject.toml index 1567310..d1631da 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -177,13 +177,6 @@ ignore-complexity = true details = "normal" sort = "asc" -[tool.bump_version.replacements] -files = [ - { file = "src/toolbox_python/__init__.py", pattern = "__version__ = \"{VERSION}\"" }, - { file = "src/tests/test_version.py", pattern = "__version__ = \"{VERSION}\"" }, - { file = "pyproject.toml", pattern = "version = \"{VERSION}\"" }, -] - [tool.dfc] allow_undefined_sections = true require_docstrings = false diff --git a/src/tests/test_version.py b/src/tests/test_version.py deleted file mode 100644 index 2e926a6..0000000 --- a/src/tests/test_version.py +++ /dev/null @@ -1,40 +0,0 @@ -# ---------------------------------------------------------------------------- # -# # -# Setup #### -# # -# ---------------------------------------------------------------------------- # - - -# ---------------------------------------------------------------------------- # -# Imports #### -# ---------------------------------------------------------------------------- # - - -# ## Python StdLib Imports ---- -from unittest import TestCase - -# ## Local First Party Imports ---- -from toolbox_python import __version__ as version - - -# ---------------------------------------------------------------------------- # -# Constants #### -# ---------------------------------------------------------------------------- # - - -__version__ = "v1.4.1" - - -# ---------------------------------------------------------------------------- # -# # -# Test Suite #### -# # -# ---------------------------------------------------------------------------- # - - -class TestStrings(TestCase): - def setUp(self) -> None: - pass - - def test_version(self) -> None: - assert version == __version__ diff --git a/src/toolbox_python/__init__.py b/src/toolbox_python/__init__.py index e2b1bd4..b97399a 100644 --- a/src/toolbox_python/__init__.py +++ b/src/toolbox_python/__init__.py @@ -1,2 +1,19 @@ -__version__ = "v1.4.1" -__author__ = "Chris Mahoney" +""" +Python Toolbox + +A collection of utility functions and classes for Python development. +""" + +# ## Python StdLib Imports ---- +from importlib.metadata import metadata + + +### Define package metadata ---- +_metadata = metadata("toolbox-python") +__name__: str = _metadata["Name"] +__version__: str = _metadata["Version"] +__author__: str = _metadata["Author"] +__author_email__: str = _metadata["Author-email"] +__license__: str = _metadata["License"] +__url__: str = _metadata["Home-page"] +__description__: str = _metadata["Summary"] diff --git a/src/utils/bump_version.py b/src/utils/bump_version.py deleted file mode 100644 index 6f93c32..0000000 --- a/src/utils/bump_version.py +++ /dev/null @@ -1,143 +0,0 @@ -# ============================================================================ # -# # -# Title: Bump Version # -# Purpose: This script reads a pyproject.toml file and extracts the # -# "files" section under "tool.bump_version.replacements". It also # -# accepts a version number as an argument to this module. It will # -# then update the version in the files with the version number # -# provided. # -# Args: # -# - version: The new version to set in the files. # -# # -# ============================================================================ # - - -# ---------------------------------------------------------------------------- # -# # -# Setup #### -# # -# ---------------------------------------------------------------------------- # - - -## --------------------------------------------------------------------------- # -## Imports #### -## --------------------------------------------------------------------------- # - - -# ## Python StdLib Imports ---- -import argparse -import re -import tomllib -from pathlib import Path -from typing import Any - -# ## Local First Party Imports ---- -from toolbox_python.dictionaries import DotDict - - -## --------------------------------------------------------------------------- # -## Args #### -## --------------------------------------------------------------------------- # - - -### Set up argument parsing ---- -parser = argparse.ArgumentParser(description="Bump version in files.") -parser.add_argument( - "-v", - "--verbose", - default=False, - type=bool, - help="Enable verbose output.", -) -parser.add_argument("version", type=str, help="The new version to set in the files.") - -### Parse the arguments ---- -args: argparse.Namespace = parser.parse_args() - -### Check ---- -if args.verbose: - print("Arguments:") - for arg in vars(args): - print(f"{arg}: {getattr(args, arg)}") - - -## --------------------------------------------------------------------------- # -## Config #### -## --------------------------------------------------------------------------- # - - -def get_config() -> list[DotDict]: - - ### Read the pyproject.toml file ---- - with open("pyproject.toml", "rb") as f: - data: dict[str, Any] = tomllib.load(f) - - ### Convert the dictionary to a DotDict for easier access ---- - data = DotDict(data) - - ### Extract the relevant sections ---- - files: list[DotDict] = data.tool.bump_version.replacements.files - - ### Return ---- - return files - - -# ---------------------------------------------------------------------------- # -# # -# Main Section #### -# # -# ---------------------------------------------------------------------------- # - - -def update_files(files: list[DotDict]) -> None: - - ### Check the files ---- - if args.verbose: - print("Updating files:") - - ### Loop through the files ---- - for file in files: - - ### Extract variables ---- - filepath = Path(file.file) - pattern: str = file.pattern - search_pattern: str = "^" + pattern.replace("{VERSION}", ".*?") - - ### Check ---- - if args.verbose: - print(f"- {file.file}") - - ### Check if the file exists ---- - if not filepath.exists(): - print(f"-- File does not exist: {file.file}") - continue - - ### Read the file ---- - content: str = filepath.read_text() - - ### Check if the pattern exists in the file ---- - if not re.search(pattern=search_pattern, string=content, flags=re.MULTILINE): - print(f"-- !! Pattern not found in file: {file.pattern}") - continue - - new_content: list[str] = [] - for line in content.splitlines(): - if re.search(pattern=search_pattern, string=line, flags=re.MULTILINE): - new_line: str = re.sub( - pattern=search_pattern, - repl=pattern.replace("{VERSION}", args.version), - string=line, - ) - new_content.append(new_line) - if args.verbose: - print(f"-- old--> {line}") - print(f"-- new--> {new_line}") - else: - new_content.append(line) - - ### Write the new content to the file ---- - filepath.write_text("\n".join(new_content) + "\n") - - -def main() -> None: - update_files(get_config()) From 2e057dd3e90c633035a695c08ce9db13375a973e Mon Sep 17 00:00:00 2001 From: Chris Mahoney <44449504+chrimaho@users.noreply.github.com> Date: Sun, 28 Dec 2025 18:45:58 +1100 Subject: [PATCH 31/43] Remove redundant `Makefile` --- Makefile | 212 ------------------------------------------------------- 1 file changed, 212 deletions(-) delete mode 100644 Makefile diff --git a/Makefile b/Makefile deleted file mode 100644 index 650fac8..0000000 --- a/Makefile +++ /dev/null @@ -1,212 +0,0 @@ -#* Variables -PYTHON := python3 -PACKAGE_NAME := toolbox_python -PYTHONPATH := `pwd` -VERSION ?= v0.0.0 -VERSION_CLEAN := $(shell echo $(VERSION) | awk '{gsub(/v/,"")}1') -VERSION_NO_PATCH := "$(shell echo $(VERSION) | cut --delimiter '.' --fields 1-2).*" -UV_LINK_MODE := copy - - -#* Environment -.PHONY: check-environment -update-build-essentials: - sudo apt-get install build-essential -update-environment: - sudo apt-get update --yes - sudo apt-get upgrade --yes -install-git: - sudo apt-get install git --yes - - -#* Python -.PHONY: prepare-python -install-python: - sudo apt-get install python3-venv --yes -install-pip: - sudo apt-get install python3-pip --yes -upgrade-pip: - $(PYTHON) -m pip install --upgrade pip -install-python-and-pip: install-python install-pip upgrade-pip - - -#* Poetry -.PHONY: poetry-installs -poetry-install-poetry: - $(PYTHON) -m pip install poetry - poetry --version -poetry-install: - poetry lock - poetry install --no-interaction --only main -poetry-install-dev: - poetry lock - poetry install --no-interaction --with dev -poetry-install-docs: - poetry lock - poetry install --no-interaction --with docs -poetry-install-test: - poetry lock - poetry install --no-interaction --with test -poetry-install-dev-test: - poetry lock - poetry install --no-interaction --with dev,test -poetry-install-all: - poetry lock - poetry install --no-interaction --with dev,docs,test - - -#* UV -.PHONY: uv -uv-shell: - bash -c "source .venv/bin/activate && exec bash" -install-uv: - curl -LsSf https://astral.sh/uv/install.sh | sh - uv --version -uv-self-update: - uv self update - uv --version -uv-install-main: - uv sync --link-mode=copy --no-cache --no-group=dev --no-group=docs --no-group=test -uv-install-dev: - uv sync --link-mode=copy --no-cache --group=dev -uv-install-docs: - uv sync --link-mode=copy --no-cache --group=docs -uv-install-test: - uv sync --link-mode=copy --no-cache --group=test -uv-install-dev-test: - uv sync --link-mode=copy --no-cache --group=dev --group=test -uv-install-all: - uv sync --link-mode=copy --no-cache --all-groups -uv-lock: - uv lock --link-mode=copy -uv-sync-main: uv-install-main -uv-sync-dev: uv-install-dev -uv-sync-docs: uv-install-docs -uv-sync-test: uv-install-test -uv-sync-dev-test: uv-install-dev-test -uv-sync-all: uv-install-all -uv-sync: uv-install-all -uv-update: uv-install-all -uv-lock-sync: uv-lock uv-sync -install: uv-install-main -install-main: uv-install-main -install-dev: uv-install-dev -install-docs: uv-install-docs -install-test: uv-install-test -install-dev-test: uv-install-dev-test -install-all: uv-install-all - - -#* Linting -.PHONY: linting -run-black: - uv run --link-mode=copy black --config pyproject.toml ./ -run-isort: - uv run --link-mode=copy isort --settings-file pyproject.toml ./ -lint: run-black run-isort - - -#* Checking -.PHONY: checking -check-black: - uv run --link-mode=copy black --diff --check --config pyproject.toml ./ -check-mypy: - uv run --link-mode=copy mypy --install-types --config-file pyproject.toml src/$(PACKAGE_NAME) -check-isort: - uv run --link-mode=copy isort --settings-file pyproject.toml ./ -check-codespell: - uv run --link-mode=copy codespell --toml pyproject.toml src/ *.py -check-pylint: - uv run --link-mode=copy pylint --rcfile=pyproject.toml src/$(PACKAGE_NAME) -check-pytest: - uv run --link-mode=copy pytest --config-file pyproject.toml -check-pycln: - uv run --link-mode=copy pycln --config="pyproject.toml" src/$(PACKAGE_NAME) -check-build: - uv build --out-dir=dist - if [ -d "dist" ]; then rm --recursive dist; fi -check-mkdocs: - uv run --link-mode=copy mkdocs build --site-dir="temp" - if [ -d "temp" ]; then rm --recursive temp; fi -check: check-black check-mypy check-pycln check-isort check-codespell check-pylint check-mkdocs check-build check-pytest - - -#* Testing -.PHONY: pytest -pytest: - uv run --link-mode=copy pytest --config-file pyproject.toml -copy-coverage-report: - cp --recursive --update "./cov-report/html/." "./docs/code/coverage/" -commit-coverage-report: - git add . - git commit --no-verify --message "Update coverage report [skip ci]" - git push - - -#* Git -.PHONY: git-processes -git-add-credentials-old: - git config --global user.name ${GITHUB_ACTOR} - git config --global user.email "${GITHUB_ACTOR}@users.noreply.github.com" -git-add-credentials: - git config --global user.name "github-actions[bot]" - git config --global user.email "github-actions[bot]@users.noreply.github.com" -configure-git: git-add-credentials -git-refresh-current-branch: - git remote update - git fetch --verbose - git fetch --verbose --tags - git pull --verbose - git status --verbose - git branch --list --verbose - git tag --list --sort=-creatordate -git-switch-to-main-branch: - git checkout -B main --track origin/main -git-switch-to-docs-branch: - git checkout -B docs-site --track origin/docs-site - - -#* Deploy Package -# See: https://github.com/monim67/poetry-bumpversion -.PHONY: deployment -bump-version: - uv run --link-mode=copy python -m src.utils.bump_version --verbose=true $(VERSION_CLEAN) -update-git: - git add . - git commit --message="Bump to version \`$(VERSION)\` [skip ci]" --allow-empty - git push --force --no-verify - git status -uv-build: - uv build --out-dir=dist -uv-publish: - uv publish --token ${PYPI_TOKEN} -build-package: uv-build -publish-package: uv-publish -deploy-package: uv-publish - - -#* Docs -.PHONY: docs -docs-serve-static: - uv run --link-mode=copy mkdocs serve -docs-serve-versioned: - uv run --link-mode=copy mike serve --branch=docs-site -docs-build-static: - uv run --link-mode=copy mkdocs build --clean -docs-build-versioned: - git config --global --list - git config --local --list - git remote -v - uv run --link-mode=copy mike --debug deploy --update-aliases --branch=docs-site --push $(VERSION) latest -update-git-docs: - git add . - git commit -m "Build docs [skip ci]" - git push --force --no-verify --push-option ci.skip -docs-check-versions: - uv run --link-mode=copy mike --debug list --branch=docs-site -docs-delete-version: - uv run --link-mode=copy mike --debug delete --branch=docs-site $(VERSION) -docs-set-default: - uv run --link-mode=copy mike --debug set-default --branch=docs-site --push latest -build-static-docs: docs-build-static update-git-docs -build-versioned-docs: docs-build-versioned docs-set-default From 14781b7869295e87a36f0127a36813b8fadef808 Mon Sep 17 00:00:00 2001 From: Chris Mahoney <44449504+chrimaho@users.noreply.github.com> Date: Sun, 28 Dec 2025 18:48:42 +1100 Subject: [PATCH 32/43] Update `print_or_log_output()` function typing - Add a new `@overload` for the `print_or_log_output()` function to support `Optional` parameters. - Update the `print_or_log_output()` function signature to accept an `Optional` value for the `print_or_log` argument. - Standardise the `log_level` type hint within the docstring of the `print_or_log_output()` function. - Reformat existing `@overload` declarations for the `print_or_log_output()` function to improve readability. - Add an `assert` to ensure that the `print_or_log` parameter is never `None` --- src/toolbox_python/output.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/src/toolbox_python/output.py b/src/toolbox_python/output.py index f078f56..9486731 100644 --- a/src/toolbox_python/output.py +++ b/src/toolbox_python/output.py @@ -79,7 +79,10 @@ @overload -def print_or_log_output(message: str, print_or_log: Literal["print"]) -> None: ... +def print_or_log_output( + message: str, + print_or_log: Literal["print"], +) -> None: ... @overload def print_or_log_output( message: str, @@ -88,10 +91,18 @@ def print_or_log_output( log: Logger, log_level: log_levels = "info", ) -> None: ... +@overload +def print_or_log_output( + message: str, + print_or_log: Optional[Literal["print", "log"]] = None, + *, + log: Optional[Logger] = None, + log_level: Optional[log_levels] = None, +) -> None: ... @typechecked def print_or_log_output( message: str, - print_or_log: Literal["print", "log"] = "print", + print_or_log: Optional[Literal["print", "log"]] = "print", *, log: Optional[Logger] = None, log_level: Optional[log_levels] = None, @@ -110,7 +121,7 @@ def print_or_log_output( If `#!py print_or_log=="log"`, then this parameter must contain the `#!py Logger` object to be processed, otherwise it will raise an `#!py AssertError`.
Defaults to `#!py None`. - log_level (Optional[_log_levels], optional): + log_level (Optional[log_levels], optional): If `#!py print_or_log=="log"`, then this parameter must contain the required log level for the `message`. Must be one of the log-levels available in the `#!py logging` module.
Defaults to `#!py None`. @@ -240,6 +251,7 @@ def print_or_log_output( ) # Assertions to keep `mypy` happy + assert print_or_log is not None assert log is not None assert log_level is not None From 857900fb879a00347801a3497f69030085a12966 Mon Sep 17 00:00:00 2001 From: Chris Mahoney <44449504+chrimaho@users.noreply.github.com> Date: Sun, 28 Dec 2025 18:49:55 +1100 Subject: [PATCH 33/43] Fix misaligned parameter docs and function signatures --- src/toolbox_python/output.py | 2 +- src/toolbox_python/retry.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/toolbox_python/output.py b/src/toolbox_python/output.py index 9486731..c4df8ca 100644 --- a/src/toolbox_python/output.py +++ b/src/toolbox_python/output.py @@ -293,7 +293,7 @@ def list_columns( Print the given list in evenly-spaced columns. Params: - obj (list): + obj (Union[any_list, any_set, any_tuple, Generator]): The list to be formatted. cols_wide (int, optional): diff --git a/src/toolbox_python/retry.py b/src/toolbox_python/retry.py index 2c4c789..1e4ae35 100644 --- a/src/toolbox_python/retry.py +++ b/src/toolbox_python/retry.py @@ -152,7 +152,7 @@ def retry( delay (int, optional): The number of seconds to delay between each retry.
Defaults to `#!py 0`. - print_or_log (Optional[Literal["print", "log"]], optional): + print_or_log (Literal["print", "log"], optional): Whether or not the messages should be written to the terminal in a `#!py print()` statement, or to a log file in a `#!py log()` statement.
Defaults to `#!py "print"`. @@ -211,6 +211,7 @@ def retry( - https://pypi.org/project/retry/ - https://stackoverflow.com/questions/21786382/pythonic-way-of-retry-running-a-function#answer-21788594 """ + assert_is_valid(tries, ">=", 0) assert_is_valid(delay, ">=", 0) From 2abcd3b19e94441065ed92e7d0a43d916c3da7b6 Mon Sep 17 00:00:00 2001 From: Chris Mahoney <44449504+chrimaho@users.noreply.github.com> Date: Sun, 28 Dec 2025 19:20:33 +1100 Subject: [PATCH 34/43] Refactor retry logic into a helper class - Extract the core logic from the `retry()` decorator into a new internal `_Retry()` class to improve maintainability. - Modularise error handling by introducing specific methods like `._handle_expected_error()` and `._handle_unexpected_error()` within the `_Retry()` class. - Update the `retry()` decorator to instantiate the `_Retry()` class and invoke its `.run()` method. - Strengthen type hinting by incorporating `Any` and defining explicit return types for the internal wrapper. - Reduce module complexity from >17 to ~4 --- src/toolbox_python/retry.py | 200 +++++++++++++++++++++++------------- 1 file changed, 127 insertions(+), 73 deletions(-) diff --git a/src/toolbox_python/retry.py b/src/toolbox_python/retry.py index 1e4ae35..dfb503e 100644 --- a/src/toolbox_python/retry.py +++ b/src/toolbox_python/retry.py @@ -45,7 +45,7 @@ from logging import Logger from time import sleep from types import ModuleType -from typing import Callable, Literal, Optional, TypeVar, Union, overload +from typing import Any, Callable, Literal, Optional, TypeVar, Union, overload # ## Python Third Party Imports ---- from typeguard import typechecked @@ -91,8 +91,124 @@ # ---------------------------------------------------------------------------- # -class Retry: - pass +class _Retry: + """ + !!! note "Summary" + A helper class to handle the retry logic for the `retry` decorator. + + ???+ abstract "Details" + This class is not intended to be used directly. Instead, it is used internally by the `retry` decorator to manage the retry logic. + + Methods: + run(): Run the retry loop for the given function. + """ + + def __init__( + self, + exceptions: _exceptions, + tries: int, + delay: int, + print_or_log: Literal["print", "log"], + log: Optional[Logger], + ) -> None: + """ + !!! note "Summary" + Initialize the `_Retry` class with the given parameters. + + Params: + exceptions (_exceptions): + A given single or collection of expected exceptions for which to catch and retry for. + tries (int): + The number of retries to attempt. + delay (int): + The number of seconds to delay between each retry. + print_or_log (Literal["print", "log"]): + Whether or not the messages should be written to the terminal in a `#!py print()` statement, or to a log file in a `#!py log()` statement. + log (Optional[Logger]): + An optional logger instance to use when `print_or_log` is set to `"log"`. + """ + self.exceptions: _exceptions = exceptions + self.tries: int = tries + self.delay: int = delay + self.print_or_log: Literal["print", "log"] = print_or_log + self.log: Optional[Logger] = log + + def run(self, func: Callable[..., R], *args: Any, **kwargs: Any) -> R: + """ + !!! note "Summary" + Run the retry loop for the given function. + """ + for i in range(1, self.tries + 1): + try: + results = func(*args, **kwargs) + self._handle_success(i) + return results + except self.exceptions as e: + self._handle_expected_error(i, e) + except Exception as exc: + self._handle_unexpected_error(i, exc) + self._handle_final_failure() + + def _handle_success(self, i: int) -> None: + message: str = f"Successfully executed at iteration {i}." + print_or_log_output( + message=message, + print_or_log=self.print_or_log, + log=self.log, + log_level="info", + ) + + def _handle_expected_error(self, i: int, e: Exception) -> None: + message = ( + f"Caught an expected error at iteration {i}: " + f"`{get_full_class_name(e)}`. " + f"Retrying in {self.delay} seconds..." + ) + print_or_log_output( + message=message, + print_or_log=self.print_or_log, + log=self.log, + log_level="warning", + ) + sleep(self.delay) + + def _handle_unexpected_error(self, i: int, exc: Exception) -> None: + excs = self.exceptions if isinstance(self.exceptions, (list, tuple)) else (self.exceptions,) + exc_names: list[str] = [e.__name__ for e in excs] + if any(name in f"{exc}" for name in exc_names): + caught_errors: list[str] = [name for name in exc_names if name in f"{exc}"] + message: str = ( + f"Caught an unexpected, known error at iteration {i}: " + f"`{get_full_class_name(exc)}`.\n" + f"Who's message contains reference to underlying exception(s): {caught_errors}.\n" + f"Retrying in {self.delay} seconds..." + ) + print_or_log_output( + message=message, + print_or_log=self.print_or_log, + log=self.log, + log_level="warning", + ) + sleep(self.delay) + else: + message = f"Caught an unexpected error at iteration {i}: `{get_full_class_name(exc)}`." + print_or_log_output( + message=message, + print_or_log=self.print_or_log, + log=self.log, + log_level="error", + ) + raise RuntimeError(message) from exc + + def _handle_final_failure(self) -> None: + message: str = f"Still could not write after {self.tries} iterations. Please check." + print_or_log_output( + message=message, + print_or_log=self.print_or_log, + log=self.log, + log_level="error", + ) + raise RuntimeError(message) # ---------------------------------------------------------------------------- # @@ -225,80 +341,18 @@ def retry( if mod is not None: log: Optional[Logger] = logging.getLogger(mod.__name__) - def decorator(func: Callable): + def decorator(func: Callable[..., R]) -> Callable[..., R]: @wraps(func) - def result(*args, **kwargs): - for i in range(1, tries + 1): - try: - results = func(*args, **kwargs) - except exceptions as e: - # Catch raw exceptions as defined in the `exceptions` parameter. - message = ( - f"Caught an expected error at iteration {i}: " - f"`{get_full_class_name(e)}`. " - f"Retrying in {delay} seconds..." - ) - print_or_log_output( - message=message, - print_or_log=print_or_log, - log=log, - log_level="warning", - ) - sleep(delay) - except Exception as exc: - """ - Catch unknown exception, however still need to check whether the name of any of the exceptions defined in `exceptions` are somehow listed in the text output of the caught exception. - The cause here is shown in the below chunk. You see here that it throws a 'Py4JJavaError', which was not listed in the `exceptions` parameter, yet within the text output, it showed the 'ConcurrentDeleteReadException' which _was_ listed in the `exceptions` parameter. Therefore, in this instance, we still want to sleep and retry - - >>> Caught an unexpected error at iteration 1: `py4j.protocol.Py4JJavaError`. - >>> Time for fct_Receipt: 27secs - >>> java.util.concurrent.ExecutionException: io.delta.exceptions. - ... ConcurrentDeleteReadException: This transaction attempted to read one or more files that were deleted (for example part-00001-563449ea-73e4-4d7d-8ba8-53fee1f8a5ff.c000.snappy.parquet in the root of the table) by a concurrent update. Please try the operation again. - """ - excs = [exceptions] if not isinstance(exceptions, (list, tuple)) else exceptions - exc_names = [exc.__name__ for exc in excs] - if any(name in f"{exc}" for name in exc_names): - caught_error = [name for name in exc_names if name in f"{exc}"] - message = ( - f"Caught an unexpected, known error at iteration {i}: " - f"`{get_full_class_name(exc)}`.\n" - f"Who's message contains reference to underlying exception(s): {caught_error}.\n" - f"Retrying in {delay} seconds..." - ) - print_or_log_output( - message=message, - print_or_log=print_or_log, - log=log, - log_level="warning", - ) - sleep(delay) - else: - message = f"Caught an unexpected error at iteration {i}: " f"`{get_full_class_name(exc)}`." - print_or_log_output( - message=message, - print_or_log=print_or_log, - log=log, - log_level="error", - ) - raise RuntimeError(message) from exc - else: - message = f"Successfully executed at iteration {i}." - print_or_log_output( - message=message, - print_or_log=print_or_log, - log=log, - log_level="info", - ) - return results - message = f"Still could not write after {tries} iterations. Please check." - print_or_log_output( - message=message, + def wrapper(*args: Any, **kwargs: Any) -> R: + retry_handler = _Retry( + exceptions=exceptions, + tries=tries, + delay=delay, print_or_log=print_or_log, log=log, - log_level="error", ) - raise RuntimeError(message) + return retry_handler.run(func, *args, **kwargs) - return result + return wrapper return decorator From bb46a37697d2810d8457d26a8eb0280fc771e10c Mon Sep 17 00:00:00 2001 From: Chris Mahoney <44449504+chrimaho@users.noreply.github.com> Date: Sun, 28 Dec 2025 19:25:45 +1100 Subject: [PATCH 35/43] Standardise exception storage and update type hints - Standardise the `exceptions` attribute in the `_Retry()` class by ensuring it always stores a tuple of exception types. - Update the `._handle_final_failure()` method to use the `NoReturn` type hint as it consistently raises a `RuntimeError()` class. --- src/toolbox_python/retry.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/toolbox_python/retry.py b/src/toolbox_python/retry.py index dfb503e..709decd 100644 --- a/src/toolbox_python/retry.py +++ b/src/toolbox_python/retry.py @@ -45,7 +45,7 @@ from logging import Logger from time import sleep from types import ModuleType -from typing import Any, Callable, Literal, Optional, TypeVar, Union, overload +from typing import Any, Callable, Literal, NoReturn, Optional, TypeVar, Union, overload # ## Python Third Party Imports ---- from typeguard import typechecked @@ -127,7 +127,9 @@ def __init__( log (Optional[Logger]): An optional logger instance to use when `print_or_log` is set to `"log"`. """ - self.exceptions: _exceptions = exceptions + self.exceptions: tuple[type[Exception], ...] = ( + tuple(exceptions) if isinstance(exceptions, (list, tuple)) else (exceptions,) + ) self.tries: int = tries self.delay: int = delay self.print_or_log: Literal["print", "log"] = print_or_log @@ -200,7 +202,7 @@ def _handle_unexpected_error(self, i: int, exc: Exception) -> None: ) raise RuntimeError(message) from exc - def _handle_final_failure(self) -> None: + def _handle_final_failure(self) -> NoReturn: message: str = f"Still could not write after {self.tries} iterations. Please check." print_or_log_output( message=message, From 5999a7940fb6c2c04a03e69c382977ace5b0a3db Mon Sep 17 00:00:00 2001 From: Chris Mahoney <44449504+chrimaho@users.noreply.github.com> Date: Sun, 28 Dec 2025 19:30:07 +1100 Subject: [PATCH 36/43] Remove redundant metadata attributes - Remove `__license__`, `__url__`, and `__description__` variables - Organise the package root by removing secondary metadata attributes --- src/toolbox_python/__init__.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/toolbox_python/__init__.py b/src/toolbox_python/__init__.py index b97399a..7c7ad0f 100644 --- a/src/toolbox_python/__init__.py +++ b/src/toolbox_python/__init__.py @@ -14,6 +14,3 @@ __version__: str = _metadata["Version"] __author__: str = _metadata["Author"] __author_email__: str = _metadata["Author-email"] -__license__: str = _metadata["License"] -__url__: str = _metadata["Home-page"] -__description__: str = _metadata["Summary"] From 50efdd980b09437b3a7d89942ca84731f8e98cff Mon Sep 17 00:00:00 2001 From: Chris Mahoney <44449504+chrimaho@users.noreply.github.com> Date: Sun, 28 Dec 2025 20:18:51 +1100 Subject: [PATCH 37/43] Refactor git utilities and subprocess handling - Add `encoding="utf-8"` to the `run()` function call within the `run_command()` function to standardise text handling - Extract branch checkout logic into a new `git_checkout_branch()` function to deduplicate `git_switch_to_main_branch()` and `git_switch_to_docs_branch()` functions - Introduce `git_switch_to_branch()` function to support branch switching via command-line arguments - Optimise `git_add_coverage_report()` function by ensuring the destination directory exists before copying files - Simplify `add_git_credentials()` and `git_fix_tag_reference()` functions by removing redundant comments and streamlining command execution --- src/utils/scripts.py | 58 +++++++++++++++++--------------------------- 1 file changed, 22 insertions(+), 36 deletions(-) diff --git a/src/utils/scripts.py b/src/utils/scripts.py index 79dd0e9..ac02a7e 100644 --- a/src/utils/scripts.py +++ b/src/utils/scripts.py @@ -34,7 +34,7 @@ def expand_space(lst: Union[list[str], tuple[str, ...]]) -> list[str]: def run_command(*command, expand: bool = True) -> None: _command: list[str] = expand_space(command) if expand else list(command) print("\n", " ".join(_command), sep="", flush=True) - subprocess.run(_command, check=True) + subprocess.run(_command, check=True, encoding="utf-8") run = run_command @@ -179,57 +179,51 @@ def check() -> None: def add_git_credentials() -> None: - run("git", "config", "--global", "user.name", "github-actions[bot]") - run( - "git", - "config", - "--global", - "user.email", - "github-actions[bot]@users.noreply.github.com", - ) + run("git config --global user.name github-actions[bot]") + run("git config --global user.email github-actions[bot]@users.noreply.github.com") def git_refresh_current_branch() -> None: run("git remote update") run("git fetch --verbose") run("git fetch --verbose --tags") - run("git pull --verbose") + run("git pull --verbose") run("git status --verbose") run("git branch --list --verbose") run("git tag --list --sort=-creatordate") +def git_checkout_branch(branch_name: str) -> None: + run(f"git checkout -B {branch_name} --track origin/{branch_name}") + + +def git_switch_to_branch() -> None: + if len(sys.argv) < 3: + print("Requires argument: ") + sys.exit(1) + git_checkout_branch(sys.argv[2]) + + def git_switch_to_main_branch() -> None: - run("git checkout -B main --track origin/main") + git_checkout_branch("main") def git_switch_to_docs_branch() -> None: - run("git checkout -B docs-site --track origin/docs-site") + git_checkout_branch("docs-site") def git_add_coverage_report() -> None: - run("cp --recursive --update ./cov-report/html/ ./docs/code/coverage/") - run("git add ./docs/code/coverage/*") - run( - "git", - "commit", - "--no-verify", - '--message="Update coverage report [skip ci]"', - expand=False, - ) + run("mkdir -p ./docs/code/coverage/") + run("cp -r ./cov-report/html/. ./docs/code/coverage/") + run("git add ./docs/code/coverage/") + run("git", "commit", "--no-verify", '--message="Update coverage report [skip ci]"', expand=False) run("git push") def git_update_version(version: str) -> None: run(f'echo VERSION="{version}"') run("git add .") - run( - "git", - "commit", - "--allow-empty", - f'--message="Bump to version `{version}` [skip ci]"', - expand=False, - ) + run("git", "commit", "--allow-empty", f'--message="Bump to version `{version}` [skip ci]"', expand=False) run("git push --force --no-verify") run("git status") @@ -242,15 +236,7 @@ def git_update_version_cli() -> None: def git_fix_tag_reference(version: str) -> None: - """ - Force update the tag to point to the latest commit with correct version number. - This also ensures the tag shows the correct version in the `pyproject.toml` file for that tag. - """ - - ### Force update the tag to point to the current commit ---- run(f"git tag --force {version}") - - ### Force push the updated tag ---- run(f"git push --force origin {version}") From d7ef4f676965fd2113477e3f1a55c061762572ec Mon Sep 17 00:00:00 2001 From: Chris Mahoney <44449504+chrimaho@users.noreply.github.com> Date: Sun, 28 Dec 2025 20:21:08 +1100 Subject: [PATCH 38/43] Refactor CI/CD and add support Python for 3.14 - Include Python `3.14` in test matrices to ensure forward compatibility. - Remove redundant `debug` jobs to reduce workflow clutter and execution time. - Standardise script execution by invoking utility scripts directly via `uv run`. - Force package reinstallation and upgrades using the `--reinstall-package` and `--upgrade` flags in the `uv sync` command. - Set `PYTHONIOENCODING` to `utf-8` to ensure consistent log output. - Increase `max-parallel` to 30 to speed up multi-platform installation checks. - Extend artifact retention to 5 days for better post-build access. - Refine the `uv publish` command to use explicit token flags and disable caching. --- .github/workflows/cd.yml | 262 +++++++++++++-------------------------- .github/workflows/ci.yml | 95 +------------- 2 files changed, 95 insertions(+), 262 deletions(-) diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index 71b6cc1..4cfdd46 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -31,91 +31,11 @@ env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} REPOSITORY_NAME: data-science-extensions/toolbox-python GIT_BRANCH: ${{ github.event.release.target_commitish }} - PYTHON_VERSION: '3.13' + PYTHON_VERSION: '3.14' + PYTHONIOENCODING: utf-8 jobs: - debug: - - name: Run Debugging - runs-on: ubuntu-latest - - steps: - - - name: Checkout repository - uses: actions/checkout@v4 - with: - ref: main - - - name: Check variables - run: | - echo "github.action: ${{ github.action }}" - echo "github.action_path: ${{ github.action_path }}" - echo "github.action_ref: ${{ github.action_ref }}" - echo "github.action_repository: ${{ github.action_repository }}" - echo "github.action_status: ${{ github.action_status }}" - echo "github.actor: ${{ github.actor }}" - echo "github.actor_id: ${{ github.actor_id }}" - echo "github.api_url: ${{ github.api_url }}" - echo "github.base_ref: ${{ github.base_ref }}" - echo "github.env: ${{ github.env }}" - echo "github.event_name: ${{ github.event_name }}" - echo "github.event_path: ${{ github.event_path }}" - echo "github.graphql_url: ${{ github.graphql_url }}" - echo "github.head_ref: ${{ github.head_ref }}" - echo "github.job: ${{ github.job }}" - echo "github.job_workflow_sha: ${{ github.job_workflow_sha }}" - echo "github.path: ${{ github.path }}" - echo "github.ref: ${{ github.ref }}" - echo "github.ref_name: ${{ github.ref_name }}" - echo "github.ref_protected: ${{ github.ref_protected }}" - echo "github.ref_type: ${{ github.ref_type }}" - echo "github.repository: ${{ github.repository }}" - echo "github.repository_id: ${{ github.repository_id }}" - echo "github.repository_owner: ${{ github.repository_owner }}" - echo "github.repository_owner_id: ${{ github.repository_owner_id }}" - echo "github.repositoryUrl: ${{ github.repositoryUrl }}" - echo "github.retention_days: ${{ github.retention_days }}" - echo "github.run_attempt: ${{ github.run_attempt }}" - echo "github.run_id: ${{ github.run_id }}" - echo "github.run_number: ${{ github.run_number }}" - echo "github.secret_source: ${{ github.secret_source }}" - echo "github.server_url: ${{ github.server_url }}" - echo "github.sha: ${{ github.sha }}" - echo "github.token: ${{ github.token }}" - echo "github.triggering_actor: ${{ github.triggering_actor }}" - echo "github.workflow: ${{ github.workflow }}" - echo "github.workflow_ref: ${{ github.workflow_ref }}" - echo "github.workflow_sha: ${{ github.workflow_sha }}" - echo "github.workspace: ${{ github.workspace }}" - echo "github.event.action: ${{ github.event.action }}" - echo "github.event.enterprise: ${{ github.event.enterprise }}" - echo "github.event.organization: ${{ github.event.organization }}" - echo "github.event.repository: ${{ github.event.repository }}" - echo "github.event.sender: ${{ github.event.sender }}" - echo "github.event.release.assets_url: ${{ github.event.release.assets_url }}" - echo "github.event.release.author: ${{ github.event.release.author }}" - echo "github.event.release.created_at: ${{ github.event.release.created_at }}" - echo "github.event.release.draft: ${{ github.event.release.draft }}" - echo "github.event.release.html_url: ${{ github.event.release.html_url }}" - echo "github.event.release.id: ${{ github.event.release.id }}" - echo "github.event.release.node_id: ${{ github.event.release.node_id }}" - echo "github.event.release.prerelease: ${{ github.event.release.prerelease }}" - echo "github.event.release.published_at: ${{ github.event.release.published_at }}" - echo "github.event.release.tag_name: ${{ github.event.release.tag_name }}" - echo "github.event.release.tarball_url: ${{ github.event.release.tarball_url }}" - echo "github.event.release.target_commitish: ${{ github.event.release.target_commitish }}" - echo "github.event.release.upload_url: ${{ github.event.release.upload_url }}" - echo "github.event.release.url: ${{ github.event.release.url }}" - echo "github.event.release.zipball_url: ${{ github.event.release.zipball_url }}" - echo -E "github.event.release.name: ${{ github.event.release.name }}" - echo -E "github.event.release.body: ${{ github.event.release.body }}" - - - name: Check Git - run: | - git status - git branch - test: name: Run Tests @@ -141,24 +61,24 @@ jobs: - name: Install dependencies id: install-dependencies - run: uv sync --no-cache --all-groups + run: uv sync --no-cache --all-groups --upgrade --reinstall-package=${{ env.PACKAGE_NAME }} - name: Set up Git id: setup-git env: GITHUB_ACTOR: ${{ github.actor }} run: | - uv run add-git-credentials - uv run git-switch-to-main-branch - uv run git-refresh-current-branch + uv run ./src/utils/scripts.py add_git_credentials + uv run ./src/utils/scripts.py git_switch_to_branch ${{ env.GIT_BRANCH }} + uv run ./src/utils/scripts.py git_refresh_current_branch - name: Run checks id: run-checks - run: uv run check + run: uv run ./src/utils/scripts.py check - name: Add coverage report id: add-coverage-report - run: uv run git-add-coverage-report + run: uv run ./src/utils/scripts.py git_add_coverage_report - name: Upload coverage id: upload-coverage @@ -170,76 +90,76 @@ jobs: build-package: - name: Build Package - needs: test - if: ${{ always() && needs.test.result == 'success' }} - runs-on: ubuntu-latest - - steps: - - - name: Checkout repository - id: checkout-repository - uses: actions/checkout@v5 - with: - ref: ${{ env.GIT_BRANCH }} - - - name: Set up UV - uses: astral-sh/setup-uv@v6 - - - name: Setup Python - id: setup-python - uses: actions/setup-python@v5 - with: - python-version: ${{ env.PYTHON_VERSION }} - - - name: Check VERSION - id: check-version - run: | - if [ -z "${VERSION}" ]; then - echo "/$VERSION is missing. Please try again." - exit 1 - fi - - - name: Install dependencies - run: uv sync --no-cache - - - name: Setup Git - id: setup-git - run: | - uv run add-git-credentials - uv run git-switch-to-main-branch - uv run git-refresh-current-branch - - - name: Bump version - id: bump-version - run: uv version ${VERSION} - - - name: Update Git Version - id: update-git-version - run: uv run git-update-version ${VERSION} - - - name: Build package - id: build-package - run: uv build --out-dir=dist - - - name: Upload assets - id: upload-assets - uses: softprops/action-gh-release@v2 - with: - files: dist/* - - - name: Upload artifacts - id: upload-artifacts - uses: actions/upload-artifact@v4 - with: - name: dist - path: dist/* - retention-days: 1 - overwrite: true - - - name: Fix tag reference - id: fix-tag-reference - run: uv run git-fix-tag-reference ${VERSION} + name: Build Package + needs: test + if: ${{ always() && needs.test.result == 'success' }} + runs-on: ubuntu-latest + + steps: + + - name: Checkout repository + id: checkout-repository + uses: actions/checkout@v5 + with: + ref: ${{ env.GIT_BRANCH }} + + - name: Set up UV + uses: astral-sh/setup-uv@v6 + + - name: Setup Python + id: setup-python + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: Check VERSION + id: check-version + run: | + if [ -z "${{ env.VERSION }}" ]; then + echo "/$VERSION is missing. Please try again." + exit 1 + fi + + - name: Install dependencies + run: uv sync --no-cache --upgrade --reinstall-package=${{ env.PACKAGE_NAME }} + + - name: Setup Git + id: setup-git + run: | + uv run ./src/utils/scripts.py add_git_credentials + uv run ./src/utils/scripts.py git_switch_to_branch ${{ env.GIT_BRANCH }} + uv run ./src/utils/scripts.py git_refresh_current_branch + + - name: Bump version + id: bump-version + run: uv version ${VERSION} + + - name: Update Git Version + id: update-git-version + run: uv run ./src/utils/scripts.py git_update_version_cli ${VERSION} + + - name: Build package + id: build-package + run: uv build --out-dir=dist + + - name: Upload assets + id: upload-assets + uses: softprops/action-gh-release@v2 + with: + files: dist/* + + - name: Upload artifacts + id: upload-artifacts + uses: actions/upload-artifact@v4 + with: + name: dist + path: dist/* + retention-days: 5 + overwrite: true + + - name: Fix tag reference + id: fix-tag-reference + run: uv run ./src/utils/scripts.py git_fix_tag_reference_cli ${{ env.VERSION }} deploy-package: @@ -264,7 +184,6 @@ jobs: uses: actions/setup-python@v5 with: python-version: ${{ env.PYTHON_VERSION }} - python-version-file: pyproject.toml - name: Download artifacts id: download-artifacts @@ -275,9 +194,7 @@ jobs: - name: Publish package id: publish-package - env: - PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }} - run: uv publish --token ${PYPI_TOKEN} + run: uv publish --token=${{ env.PYPI_TOKEN }} --no-cache dist/* - name: Check id: check @@ -292,9 +209,9 @@ jobs: strategy: matrix: os: ['ubuntu-latest', 'macos-latest', 'windows-latest'] - python-version: ['3.9', '3.10', '3.11', '3.12', '3.13'] + python-version: ['3.9', '3.10', '3.11', '3.12', '3.13', '3.14'] fail-fast: false - max-parallel: 15 + max-parallel: 30 name: Install Package on '${{ matrix.os }}' with '${{ matrix.python-version }}' runs-on: ${{ matrix.os }} @@ -341,40 +258,39 @@ jobs: uses: actions/setup-python@v5 with: python-version: ${{ env.PYTHON_VERSION }} - python-version-file: pyproject.toml - name: Install dependencies id: install-dependencies - run: uv sync --group=docs + run: uv sync --no-cache --upgrade --group=docs --reinstall-package=${{ env.PACKAGE_NAME}} - name: Setup Git id: setup-git env: GITHUB_ACTOR: ${{ github.actor }} run: | - uv run add-git-credentials - uv run git-switch-to-main-branch - uv run git-refresh-current-branch + uv run ./src/utils/scripts.py add_git_credentials + uv run ./src/utils/scripts.py git_switch_to_branch ${{ env.GIT_BRANCH }} + uv run ./src/utils/scripts.py git_refresh_current_branch - name: Generate ChangeLog id: generate-changelog env: GITHUB_TOKEN: ${{ env.GITHUB_TOKEN }} REPOSITORY_NAME: ${{ env.REPOSITORY_NAME }} - run: uv run generate-changelog + run: uv run ./src/utils/changelog.py - name: Commit ChangeLog id: commit-changelog run: | git add . - git commit --message "Update changelog to \`${VERSION}\` [skip ci]" || echo "No changes to commit" + git commit --message "Update changelog to \`${{ env.VERSION }}\` [skip ci]" || echo "No changes to commit" git push --force --no-verify git status - name: Build docs id: build-docs - run: uv run build-versioned-docs ${VERSION} + run: uv run ./src/utils/scripts.py build_versioned_docs_cli ${{ env.VERSION }} - name: Fix tag reference id: fix-tag-reference - run: uv run ./src/utils/scripts.py git_fix_tag_reference_cli ${VERSION} + run: uv run ./src/utils/scripts.py git_fix_tag_reference_cli ${{ env.VERSION }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 55f45fe..d143dbc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -3,91 +3,14 @@ name: CI on: [push, pull_request] env: + PACKAGE_NAME: toolbox-python UV_LINK_MODE: copy UV_NATIVE_TLS: true UV_NO_SYNC: true + PYTHONINIOENCODING: utf-8 jobs: - debug: - - name: Run Debugging - runs-on: ubuntu-latest - - steps: - - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Check variables - run: | - echo "github.action: ${{ github.action }}" - echo "github.action_path: ${{ github.action_path }}" - echo "github.action_ref: ${{ github.action_ref }}" - echo "github.action_repository: ${{ github.action_repository }}" - echo "github.action_status: ${{ github.action_status }}" - echo "github.actor: ${{ github.actor }}" - echo "github.actor_id: ${{ github.actor_id }}" - echo "github.api_url: ${{ github.api_url }}" - echo "github.base_ref: ${{ github.base_ref }}" - echo "github.env: ${{ github.env }}" - echo "github.event_name: ${{ github.event_name }}" - echo "github.event_path: ${{ github.event_path }}" - echo "github.graphql_url: ${{ github.graphql_url }}" - echo "github.head_ref: ${{ github.head_ref }}" - echo "github.job: ${{ github.job }}" - echo "github.job_workflow_sha: ${{ github.job_workflow_sha }}" - echo "github.path: ${{ github.path }}" - echo "github.ref: ${{ github.ref }}" - echo "github.ref_name: ${{ github.ref_name }}" - echo "github.ref_protected: ${{ github.ref_protected }}" - echo "github.ref_type: ${{ github.ref_type }}" - echo "github.repository: ${{ github.repository }}" - echo "github.repository_id: ${{ github.repository_id }}" - echo "github.repository_owner: ${{ github.repository_owner }}" - echo "github.repository_owner_id: ${{ github.repository_owner_id }}" - echo "github.repositoryUrl: ${{ github.repositoryUrl }}" - echo "github.retention_days: ${{ github.retention_days }}" - echo "github.run_attempt: ${{ github.run_attempt }}" - echo "github.run_id: ${{ github.run_id }}" - echo "github.run_number: ${{ github.run_number }}" - echo "github.secret_source: ${{ github.secret_source }}" - echo "github.server_url: ${{ github.server_url }}" - echo "github.sha: ${{ github.sha }}" - echo "github.token: ${{ github.token }}" - echo "github.triggering_actor: ${{ github.triggering_actor }}" - echo "github.workflow: ${{ github.workflow }}" - echo "github.workflow_ref: ${{ github.workflow_ref }}" - echo "github.workflow_sha: ${{ github.workflow_sha }}" - echo "github.workspace: ${{ github.workspace }}" - echo "github.event.action: ${{ github.event.action }}" - echo "github.event.enterprise: ${{ github.event.enterprise }}" - echo "github.event.organization: ${{ github.event.organization }}" - echo "github.event.repository: ${{ github.event.repository }}" - echo "github.event.sender: ${{ github.event.sender }}" - echo "github.event.release.assets_url: ${{ github.event.release.assets_url }}" - echo "github.event.release.author: ${{ github.event.release.author }}" - echo "github.event.release.body: ${{ github.event.release.body }}" - echo "github.event.release.created_at: ${{ github.event.release.created_at }}" - echo "github.event.release.draft: ${{ github.event.release.draft }}" - echo "github.event.release.html_url: ${{ github.event.release.html_url }}" - echo "github.event.release.id: ${{ github.event.release.id }}" - echo "github.event.release.name: ${{ github.event.release.name }}" - echo "github.event.release.node_id: ${{ github.event.release.node_id }}" - echo "github.event.release.prerelease: ${{ github.event.release.prerelease }}" - echo "github.event.release.published_at: ${{ github.event.release.published_at }}" - echo "github.event.release.tag_name: ${{ github.event.release.tag_name }}" - echo "github.event.release.tarball_url: ${{ github.event.release.tarball_url }}" - echo "github.event.release.target_commitish: ${{ github.event.release.target_commitish }}" - echo "github.event.release.upload_url: ${{ github.event.release.upload_url }}" - echo "github.event.release.url: ${{ github.event.release.url }}" - echo "github.event.release.zipball_url: ${{ github.event.release.zipball_url }}" - - - name: Check Git - run: | - git status - git branch - check: if: github.ref_type == 'branch' && github.event_name == 'push' && github.ref_name != 'main' name: Run checks @@ -100,9 +23,9 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version-file: "pyproject.toml" + python-version-file: pyproject.toml - name: Install dependencies - run: uv sync --no-cache --all-groups + run: uv sync --no-cache --all-groups --upgrade --reinstall-package=${{ env.PACKAGE_NAME }} - name: Run checks run: uv run check @@ -111,27 +34,21 @@ jobs: strategy: matrix: os: ['ubuntu-latest', 'macos-latest', 'windows-latest'] - python-version: ['3.9', '3.10', '3.11', '3.12', '3.13'] + python-version: ['3.9', '3.10', '3.11', '3.12', '3.13', '3.14'] fail-fast: false max-parallel: 15 name: Run Checks on '${{ matrix.os }}' with '${{ matrix.python-version }}' runs-on: ${{ matrix.os }} - steps: - - name: Checkout repository uses: actions/checkout@v5 - - name: Set up uv uses: astral-sh/setup-uv@v6 - - name: Set up Python uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - - name: Install dependencies - run: uv sync --no-cache --all-groups - + run: uv sync --no-cache --all-groups --upgrade --reinstall-package=${{ env.PACKAGE_NAME }} - name: Run checks run: uv run check From 152e120c13d35ee74bb3eb284e7f734206addbe7 Mon Sep 17 00:00:00 2001 From: Chris Mahoney <44449504+chrimaho@users.noreply.github.com> Date: Sun, 28 Dec 2025 20:36:26 +1100 Subject: [PATCH 39/43] Update README to include package changes - Add `uv` as a supported installation method to improve package management efficiency - Replace legacy `pipenv` and `poetry` setup guides with `uv sync` workflows to simplify environment builds - Standardise development scripts using the `uv run src/utils/scripts.py` script for unified task execution - Expand quality assurance requirements to include complexity and docstring checks for better code maintainability - Update reference links to include `uv`, `ty`, `complexipy`, and `dfc` and fix the documentation URL --- README.md | 105 +++++++++++++++++++++++++----------------------------- 1 file changed, 48 insertions(+), 57 deletions(-) diff --git a/README.md b/README.md index e889a64..9f5f783 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,7 @@ CD

+ ### Introduction The purpose of this package is to provide some helper files/functions/classes for generic Python processes. @@ -43,16 +44,16 @@ The purpose of this package is to provide some helper files/functions/classes fo For reference, these URL's are used: -| Type | Source | URL | -|---|---|---| -| Git Repo | GitHub | https://github.com/data-science-extensions/toolbox-python | -| Python Package | PyPI | https://pypi.org/project/toolbox-python | -| Package Docs | Pages | https://data-science-extensions.com/python-toolbox/ | +| Type | Source | URL | +| -------------- | ------ | --------------------------------------------------------- | +| Git Repo | GitHub | https://github.com/data-science-extensions/toolbox-python | +| Python Package | PyPI | https://pypi.org/project/toolbox-python | +| Package Docs | Pages | https://data-science-extensions.com/python-toolbox/ | ### Installation -You can install and use this package multiple ways by using [`pip`][pip], [`pipenv`][pipenv], or [`poetry`][poetry]. +You can install and use this package multiple ways by using [`pip`][pip], [`uv`][uv], [`pipenv`][pipenv], or [`poetry`][poetry]. #### Using [`pip`][pip]: @@ -78,6 +79,15 @@ You can install and use this package multiple ways by using [`pip`][pip], [`pipe ``` +#### Using [`uv`][uv]: + +1. In your terminal, run: + + ```sh + uv add toolbox-python + ``` + + #### Using [`pipenv`][pipenv]: 1. Install using environment variables: @@ -154,52 +164,18 @@ Contribution is always welcome. 3. Build your environment: - 1. With [`pipenv`][pipenv] on Windows: + 1. With [`uv`][uv] on Windows: ```pwsh - if (-not (Test-Path .venv)) {mkdir .venv} - python -m pipenv install --requirements requirements.txt --requirements requirements-dev.txt --skip-lock - python -m poetry run pre-commit install - python -m poetry shell + uv sync --all-groups + uv run pre-commit install ``` - 2. With [`pipenv`][pipenv] on Linux: + 2. With [`uv`][uv] on Linux: ```sh - mkdir .venv - python3 -m pipenv install --requirements requirements.txt --requirements requirements-dev.txt --skip-lock - python3 -m poetry run pre-commit install - python3 -m poetry shell - ``` - - 3. With [`poetry`][poetry] on Windows: - - ```pwsh - python -m pip install --upgrade pip - python -m pip install poetry - python -m poetry init - python -m poetry add $(cat requirements/root.txt) - python -m poetry add --group=dev $(cat requirements/dev.txt) - python -m poetry add --group=test $(cat requirements/test.txt) - python -m poetry add --group=docs $(cat requirements/docs.txt) - python -m poetry install - python -m poetry run pre-commit install - python -m poetry shell - ``` - - 4. With [`poetry`][poetry] on Linux: - - ```sh - python3 -m pip install --upgrade pip - python3 -m pip install poetry - python3 -m poetry init - python3 -m poetry add $(cat requirements/root.txt) - python3 -m poetry add --group=dev $(cat requirements/dev.txt) - python3 -m poetry add --group=test $(cat requirements/test.txt) - python3 -m poetry add --group=docs $(cat requirements/docs.txt) - python3 -m poetry install - python3 -m poetry run pre-commit install - python3 -m poetry shell + uv sync --all-groups + uv run pre-commit install ``` 4. Start contributing. @@ -215,32 +191,44 @@ To ensure that the package is working as expected, please ensure that: 2. You write a [UnitTest][unittest] for each function/feature you include. 3. The [CodeCoverage][codecov] is 100%. 4. All [UnitTests][pytest] are passing. -5. [MyPy][mypy] is passing 100%. +5. [Type Checking][ty] is passing 100%. +6. [Complexity][complexipy] is within the required standard. +7. [Docstrings][dfc] are correctly formatted. #### Testing -- Run them all together +- Run them all together: ```sh - poetry run make check + uv run src/utils/scripts.py check ``` - Or run them individually: - - [Black][black] - ```pysh - poetry run make check-black + - [Black][black]: + ```sh + uv run src/utils/scripts.py check-black ``` - [PyTests][pytest]: ```sh - poetry run make ckeck-pytest + uv run src/utils/scripts.py check-pytest + ``` + + - [Type Checking][ty]: + ```sh + uv run src/utils/scripts.py check-ty + ``` + + - [Complexity][complexipy]: + ```sh + uv run src/utils/scripts.py check-complexity ``` - - [MyPy][mypy]: + - [Docstrings][dfc]: ```sh - poetry run make check-mypy + uv run src/utils/scripts.py check-docstrings ``` @@ -251,8 +239,9 @@ To ensure that the package is working as expected, please ensure that: [github-license]: https://github.com/data-science-extensions/toolbox-python/blob/main/LICENSE [codecov-repo]: https://codecov.io/gh/data-science-extensions/toolbox-python [pypi]: https://pypi.org/project/toolbox-python -[docs]: ... +[docs]: https://data-science-extensions.com/python-toolbox/ [pip]: https://pypi.org/project/pip +[uv]: https://docs.astral.sh/uv/ [pipenv]: https://github.com/pypa/pipenv [poetry]: https://python-poetry.org [github-fork]: https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/working-with-forks/fork-a-repo @@ -263,5 +252,7 @@ To ensure that the package is working as expected, please ensure that: [unittest]: https://docs.python.org/3/library/unittest.html [codecov]: https://codecov.io/ [pytest]: https://docs.pytest.org -[mypy]: http://www.mypy-lang.org/ +[ty]: https://github.com/alexpovel/ty +[complexipy]: https://github.com/rohaquinlop/complexipy +[dfc]: https://github.com/data-science-extensions/docstring-format-checker [black]: https://black.readthedocs.io/ From 9a8bc4626b156190a5e3204810036d6d93b6306e Mon Sep 17 00:00:00 2001 From: Chris Mahoney <44449504+chrimaho@users.noreply.github.com> Date: Sun, 28 Dec 2025 20:37:12 +1100 Subject: [PATCH 40/43] Delete `requirements/` directory in preference for `pyproject.toml` config --- requirements/dev.txt | 9 --------- requirements/docs.txt | 10 ---------- requirements/root.txt | 2 -- requirements/test.txt | 9 --------- 4 files changed, 30 deletions(-) delete mode 100644 requirements/dev.txt delete mode 100644 requirements/docs.txt delete mode 100644 requirements/root.txt delete mode 100644 requirements/test.txt diff --git a/requirements/dev.txt b/requirements/dev.txt deleted file mode 100644 index fd6c9c0..0000000 --- a/requirements/dev.txt +++ /dev/null @@ -1,9 +0,0 @@ -black==25.* -blacken-docs==1.* -pre-commit==4.* -isort==6.* -codespell==2.* -pyupgrade==3.* -pylint==3.* -pycln==2.* -ipykernel==6.* diff --git a/requirements/docs.txt b/requirements/docs.txt deleted file mode 100644 index 2039696..0000000 --- a/requirements/docs.txt +++ /dev/null @@ -1,10 +0,0 @@ -mkdocs==1.* -mkdocs-material==9.* -mkdocs-coverage==1.* -mkdocs-autorefs==1.* -mkdocstrings==0.* -mkdocstrings-python==1.* -livereload==2.* -mike==2.* -black==25.* -docstring-inheritance==2.* diff --git a/requirements/root.txt b/requirements/root.txt deleted file mode 100644 index 2b49917..0000000 --- a/requirements/root.txt +++ /dev/null @@ -1,2 +0,0 @@ -typeguard==4.* -more-itertools==10.* diff --git a/requirements/test.txt b/requirements/test.txt deleted file mode 100644 index b26246f..0000000 --- a/requirements/test.txt +++ /dev/null @@ -1,9 +0,0 @@ -requests==2.* -pytest==8.* -pytest-clarity==1.* -pytest-cov==6.* -pytest-sugar==1.* -pytest-icdiff==0.* -pytest-xdist==3.* -mypy==1.* -parameterized==0.* From aad6e43c751757f532505972fa7eff0322f01ceb Mon Sep 17 00:00:00 2001 From: Chris Mahoney <44449504+chrimaho@users.noreply.github.com> Date: Sun, 28 Dec 2025 21:09:18 +1100 Subject: [PATCH 41/43] Simplify directory removal flags - Use `-r` instead of `--recursive` in `check_build()` and `check_mkdocs()` functions - Standardise directory removal commands for consistency - Ensure commands can be run on all OS's consistently --- src/utils/scripts.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/utils/scripts.py b/src/utils/scripts.py index ac02a7e..d802096 100644 --- a/src/utils/scripts.py +++ b/src/utils/scripts.py @@ -129,12 +129,12 @@ def check_pycln() -> None: def check_build() -> None: run("uv build --out-dir=dist") - run("rm --recursive dist") + run("rm -r dist") def check_mkdocs() -> None: run("mkdocs build --site-dir=temp") - run("rm --recursive temp") + run("rm -r temp") def check_pytest() -> None: From 57c8f796fbcc5572260603a3df983b738ac2b33c Mon Sep 17 00:00:00 2001 From: Chris Mahoney <44449504+chrimaho@users.noreply.github.com> Date: Sun, 28 Dec 2025 21:29:16 +1100 Subject: [PATCH 42/43] Expose methods in the `Validators()` class - Expose the `.value_is_between()` method, `.assert_value_is_between()` method, `.all_values_are_between()` method, and `.assert_all_values_are_between()` method to make them public. - Add a class-level docstring to the `Validators()` class to summarise its purpose and available methods. - Update unit tests and internal method calls to reference the renamed public methods. --- src/tests/test_validators.py | 14 +++++++------- src/toolbox_python/validators.py | 24 +++++++++++++++++------- 2 files changed, 24 insertions(+), 14 deletions(-) diff --git a/src/tests/test_validators.py b/src/tests/test_validators.py index 5e11afa..068aa46 100644 --- a/src/tests/test_validators.py +++ b/src/tests/test_validators.py @@ -50,22 +50,22 @@ def setUp(self) -> None: name_func=name_func_predefined_name, ) def test_value_is_between(self, _name, value, min_val, max_val, expected) -> None: - assert Validators._value_is_between(value, min_val, max_val) == expected + assert Validators.value_is_between(value, min_val, max_val) == expected def test_value_is_between_raises(self) -> None: with raises(ValueError, match="Invalid range"): - Validators._value_is_between(5, 10, 0) + Validators.value_is_between(5, 10, 0) ## ----------------------------------------------------------------------- # ## _assert_value_is_between #### ## ----------------------------------------------------------------------- # def test_assert_value_is_between_valid(self) -> None: - Validators._assert_value_is_between(5, 0, 10) + Validators.assert_value_is_between(5, 0, 10) def test_assert_value_is_between_invalid(self) -> None: with raises(AssertionError, match="Invalid Value"): - Validators._assert_value_is_between(11, 0, 10) + Validators.assert_value_is_between(11, 0, 10) ## ----------------------------------------------------------------------- # ## _all_values_are_between #### @@ -81,15 +81,15 @@ def test_assert_value_is_between_invalid(self) -> None: name_func=name_func_predefined_name, ) def test_all_values_are_between(self, _name, values, min_val, max_val, expected) -> None: - assert Validators._all_values_are_between(values, min_val, max_val) == expected + assert Validators.all_values_are_between(values, min_val, max_val) == expected ## ----------------------------------------------------------------------- # ## _assert_all_values_are_between #### ## ----------------------------------------------------------------------- # def test_assert_all_values_are_between_valid(self) -> None: - Validators._assert_all_values_are_between([1, 2, 3], 0, 5) + Validators.assert_all_values_are_between([1, 2, 3], 0, 5) def test_assert_all_values_are_between_invalid(self) -> None: with raises(AssertionError, match="Values not between"): - Validators._assert_all_values_are_between([1, 6, -1], 0, 5) + Validators.assert_all_values_are_between([1, 6, -1], 0, 5) diff --git a/src/toolbox_python/validators.py b/src/toolbox_python/validators.py index e024309..496e034 100644 --- a/src/toolbox_python/validators.py +++ b/src/toolbox_python/validators.py @@ -45,9 +45,19 @@ class Validators: + """ + !!! note "Summary" + A class containing various validation methods. + + Methods: + value_is_between(): Check if a value is between two other values. + assert_value_is_between(): Assert that a value is between two other values. + all_values_are_between(): Check if all values in an array are between two other values. + assert_all_values_are_between(): Assert that all values in an array are between two other values + """ @staticmethod - def _value_is_between(value: Real, min_value: Real, max_value: Real) -> bool: + def value_is_between(value: Real, min_value: Real, max_value: Real) -> bool: """ !!! note "Summary" Check if a value is between two other values. @@ -72,7 +82,7 @@ def _value_is_between(value: Real, min_value: Real, max_value: Real) -> bool: return result @staticmethod - def _assert_value_is_between( + def assert_value_is_between( value: Real, min_value: Real, max_value: Real, @@ -93,11 +103,11 @@ def _assert_value_is_between( (AssertionError): If the value is not between the minimum and maximum values. """ - if not Validators._value_is_between(value, min_value, max_value): + if not Validators.value_is_between(value, min_value, max_value): raise AssertionError(f"Invalid Value: `{value}`. Must be between `{min_value}` and `{max_value}`") @staticmethod - def _all_values_are_between( + def all_values_are_between( values: Sequence[Real], min_value: Real, max_value: Real, @@ -118,10 +128,10 @@ def _all_values_are_between( (bool): True if all values are between the minimum and maximum values, False otherwise. """ - return all(Validators._value_is_between(value, min_value, max_value) for value in values) + return all(Validators.value_is_between(value, min_value, max_value) for value in values) @staticmethod - def _assert_all_values_are_between( + def assert_all_values_are_between( values: Sequence[Real], min_value: Real, max_value: Real, @@ -143,7 +153,7 @@ def _assert_all_values_are_between( If any value is not between the minimum and maximum values. """ values_not_between: list[Real] = [ - value for value in values if not Validators._value_is_between(value, min_value, max_value) + value for value in values if not Validators.value_is_between(value, min_value, max_value) ] if not len(values_not_between) == 0: raise AssertionError(f"Values not between `{min_value}` and `{max_value}`: {values_not_between}") From 4b24e77b8c9096c69fd5e9448e7056caf47a1d25 Mon Sep 17 00:00:00 2001 From: Chris Mahoney <44449504+chrimaho@users.noreply.github.com> Date: Sun, 28 Dec 2025 21:30:48 +1100 Subject: [PATCH 43/43] Add explicit return to `print_or_log_output()` - Optimise function readability and ensure explicit return behaviour for the `print_or_log_output()` function. --- src/toolbox_python/output.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/toolbox_python/output.py b/src/toolbox_python/output.py index c4df8ca..8a0a38f 100644 --- a/src/toolbox_python/output.py +++ b/src/toolbox_python/output.py @@ -261,6 +261,9 @@ def print_or_log_output( msg=message, ) + # Return + return None + @overload @typechecked