From ea698cd2b721b226bee8aac6e77b85856841fca3 Mon Sep 17 00:00:00 2001 From: Daniel McKnight <34697904+NeonDaniel@users.noreply.github.com> Date: Wed, 27 Aug 2025 09:02:34 -0700 Subject: [PATCH 01/25] Update Skill API to support outside applications (#554) * Implement `_register_public_api` with additional API method metadata * Remove `packages-exclude` from license tests to troubleshoot failure * Update license tests to address test failure * Remove bad `time` reference and extra API method log --- .github/workflows/license_tests.yml | 2 +- neon_utils/skills/neon_skill.py | 96 +++++++++++++++++++++++++++++ 2 files changed, 97 insertions(+), 1 deletion(-) diff --git a/.github/workflows/license_tests.yml b/.github/workflows/license_tests.yml index 011f7454..7adc134e 100644 --- a/.github/workflows/license_tests.yml +++ b/.github/workflows/license_tests.yml @@ -11,4 +11,4 @@ jobs: uses: neongeckocom/.github/.github/workflows/license_tests.yml@master with: package-extras: audio,configuration,networking - packages-exclude: '^(precise-runner|fann2|tqdm|bs4|ovos-phal-plugin|ovos-skill|neon-core|nvidia|neon-phal-plugin|bitstruct|audioread|RapidFuzz|click|setuptools|typing_extensions|urllib).*' \ No newline at end of file + packages-exclude: '^(precise-runner|fann2|tqdm|bs4|ovos-phal-plugin|ovos-skill|neon-core|nvidia|neon-phal-plugin|bitstruct|audioread|RapidFuzz|click|setuptools|typing_extensions|urllib|marisa-trie).*' diff --git a/neon_utils/skills/neon_skill.py b/neon_utils/skills/neon_skill.py index 8197d486..b9ce3c66 100644 --- a/neon_utils/skills/neon_skill.py +++ b/neon_utils/skills/neon_skill.py @@ -227,6 +227,102 @@ def update_profile(self, new_preferences: dict, message: Message = None): except Exception as x: LOG.error(x) + # Duplicated from OVOSSkill for backwards-compat with skills using ovos-workshop 0.X + def _register_public_api(self): + """ + Find and register API methods decorated with `@api_method` and create a + messagebus handler for fetching the api info if any handlers exist. + """ + + def wrap_method(fn, arg_model=None): + """Boilerplate for returning the response to the sender.""" + + def wrapper(message): + result = None + error = None + try: + if arg_model: + result = fn(arg_model(*message.data['args'], + **message.data['kwargs'])) + else: + result = fn(*message.data.get('args', []), + **message.data.get('kwargs', {})) + try: + result = result.model_dump() + except AttributeError: + # Response is not a Pydantic model + pass + except Exception as e: + error = str(e) + message.context["skill_id"] = self.skill_id + self.bus.emit(message.response(data={'result': result, + 'error': error})) + return wrapper + + from ovos_utils.skills import get_non_properties + methods = [attr_name for attr_name in get_non_properties(self) + if hasattr(getattr(self, attr_name), '__name__')] + + for attr_name in methods: + method = getattr(self, attr_name) + + if hasattr(method, 'api_method'): + doc = method.__doc__ or '' + name = method.__name__ + + # Extract method signature and return type + import inspect + signature = inspect.signature(method) + schema = None + return_schema = None + request_class = None + try: + from pydantic import BaseModel + parameters = signature.parameters + + for arg_name, param in parameters.items(): + if arg_name == 'self': + continue + if issubclass(param.annotation, BaseModel): + # Get the JSON schema for the BaseModel + schema = param.annotation.model_json_schema() + request_class = param.annotation + break + if signature.return_annotation and issubclass(signature.return_annotation, BaseModel): + # Get the JSON schema for the return type + return_schema = signature.return_annotation.model_json_schema() + except ImportError: + # If pydantic is not installed, there is no schema to extract + pass + + self.public_api[name] = { + 'help': doc, + 'type': f'{self.skill_id}.{name}', + 'func': method, + 'signature': str(signature), + 'request_schema': schema, + 'response_schema': return_schema, + 'request_class': request_class + } + for key in self.public_api: + if ('type' in self.public_api[key] and + 'func' in self.public_api[key]): + self.log.debug(f"Adding api method: " + f"{self.public_api[key]['type']}") + + # remove the function member since it shouldn't be + # reused and can't be sent over the messagebus + func = self.public_api[key].pop('func') + req_class = self.public_api[key].pop('request_class', None) + self.add_event(self.public_api[key]['type'], + wrap_method(func, req_class), speak_errors=False) + + if self.public_api: + # TODO: Think about always registering this, so queries get an + # empty response, rather than waiting for a timeout + self.add_event(f'{self.skill_id}.public_api', + self._send_public_api, speak_errors=False) + @resolve_message def update_skill_settings(self, new_preferences: dict, message: Message = None, skill_global=True): From 31ca5fc3b830eaf9055b99a1fde9fa61d4afc504 Mon Sep 17 00:00:00 2001 From: NeonDaniel Date: Wed, 27 Aug 2025 16:02:50 +0000 Subject: [PATCH 02/25] Increment Version to 1.13.1a1 --- version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.py b/version.py index 1ce310e8..13e226f9 100644 --- a/version.py +++ b/version.py @@ -26,4 +26,4 @@ # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -__version__ = "1.13.0" +__version__ = "1.13.1a1" From 7c1a54516fcbc51380cf707c0eabee4493b6fcda Mon Sep 17 00:00:00 2001 From: NeonDaniel Date: Wed, 27 Aug 2025 16:03:33 +0000 Subject: [PATCH 03/25] Update Changelog --- CHANGELOG.md | 22 +++------------------- 1 file changed, 3 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 50657204..b040df89 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,28 +1,12 @@ # Changelog -## [1.12.2a3](https://github.com/NeonGeckoCom/neon-utils/tree/1.12.2a3) (2025-07-02) +## [1.13.1a1](https://github.com/NeonGeckoCom/neon-utils/tree/1.13.1a1) (2025-08-27) -[Full Changelog](https://github.com/NeonGeckoCom/neon-utils/compare/1.12.2a2...1.12.2a3) +[Full Changelog](https://github.com/NeonGeckoCom/neon-utils/compare/1.13.0...1.13.1a1) **Merged pull requests:** -- Override `/status` server endpoint logging [\#552](https://github.com/NeonGeckoCom/neon-utils/pull/552) ([NeonDaniel](https://github.com/NeonDaniel)) - -## [1.12.2a2](https://github.com/NeonGeckoCom/neon-utils/tree/1.12.2a2) (2025-06-25) - -[Full Changelog](https://github.com/NeonGeckoCom/neon-utils/compare/1.12.2a1...1.12.2a2) - -**Merged pull requests:** - -- Add helper to start a health check server [\#551](https://github.com/NeonGeckoCom/neon-utils/pull/551) ([NeonDaniel](https://github.com/NeonDaniel)) - -## [1.12.2a1](https://github.com/NeonGeckoCom/neon-utils/tree/1.12.2a1) (2025-05-30) - -[Full Changelog](https://github.com/NeonGeckoCom/neon-utils/compare/1.12.1...1.12.2a1) - -**Merged pull requests:** - -- Make HANA SSL optional [\#550](https://github.com/NeonGeckoCom/neon-utils/pull/550) ([NeonDaniel](https://github.com/NeonDaniel)) +- Update Skill API to support outside applications [\#554](https://github.com/NeonGeckoCom/neon-utils/pull/554) ([NeonDaniel](https://github.com/NeonDaniel)) From 72eb112e20751ff1515c2b3c33f6acb7f66702be Mon Sep 17 00:00:00 2001 From: Daniel McKnight <34697904+NeonDaniel@users.noreply.github.com> Date: Wed, 10 Sep 2025 14:00:08 -0700 Subject: [PATCH 04/25] Improve Skill metadata parsing support and remove OSM dependency (#558) * Add `skill_utils` module for skill metadata parsing Methods support removal of OSM dependency * Fix auto-completed invalid import * Add skill util tests to GHA automation * Implement `get_skill_metadata` method with unit test coverage * Update `build_skill_spec` to call new method in `skill_utils` * Roll back changes for backwards-compat and implement stricter README.md parsing * Remove invalid metadata test * Remove OSM test dependency * Add test case for README.md parsing --- .github/workflows/unit_tests.yml | 13 +- neon_utils/file_utils.py | 99 +------- neon_utils/packaging_utils.py | 85 ++----- neon_utils/skill_utils.py | 365 +++++++++++++++++++++++++++++ requirements/skills.txt | 2 + requirements/test_requirements.txt | 3 +- setup.py | 3 +- tests/packaging_util_tests.py | 2 +- tests/skill_util_tests.py | 240 +++++++++++++++++++ 9 files changed, 648 insertions(+), 164 deletions(-) create mode 100644 neon_utils/skill_utils.py create mode 100644 requirements/skills.txt create mode 100644 tests/skill_util_tests.py diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 5219d066..877f823c 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -58,7 +58,7 @@ jobs: python -m pip install --upgrade pip setuptools pip install wheel "cython<3.0.0" # TODO: cython patching https://github.com/yaml/pyyaml/issues/724 pip install --no-build-isolation pyyaml~=5.4 # TODO: patching https://github.com/yaml/pyyaml/issues/724 - pip install .[test,audio,network,configuration] + pip install .[test,audio,network,configuration,skills] - name: Change Test File Permissions run: | sudo chown -R nobody:nogroup tests/configuration/unwritable_path @@ -223,4 +223,13 @@ jobs: uses: actions/upload-artifact@v4 with: name: hana-util-test-results-${{ matrix.python-version }} - path: tests/hana-util-test-results.xml \ No newline at end of file + path: tests/hana-util-test-results.xml + + - name: Test Skill Utils + run: | + pytest tests/skill_util_tests.py --doctest-modules --junitxml=tests/skill-util-test-results.xml + - name: Upload skill utils test results + uses: actions/upload-artifact@v4 + with: + name: skill-util-test-results-${{ matrix.python-version }} + path: tests/skill-util-test-results.xml diff --git a/neon_utils/file_utils.py b/neon_utils/file_utils.py index e85500eb..a23eb181 100644 --- a/neon_utils/file_utils.py +++ b/neon_utils/file_utils.py @@ -267,7 +267,7 @@ def parse_skill_readme_file(readme_path: str) -> dict: :param readme_path: absolute path to README.md file :return: dict of parsed data """ - from neon_utils.parse_utils import clean_quotes + from neon_utils.skill_utils import _get_skill_data_readme # Check passed path if not readme_path: @@ -275,102 +275,9 @@ def parse_skill_readme_file(readme_path: str) -> dict: readme_path = os.path.expanduser(readme_path) if not os.path.isfile(readme_path): raise FileNotFoundError(f"{readme_path} is not a valid file") - - # Initialize parser params - list_sections = ("examples", "incompatible skills", "platforms", - "categories", "tags", "credits") - section = "header" - category = None - parsed_data = {} with open(readme_path, "r") as readme: - lines = readme.readlines() - - def _check_section_start(ln: str): - # Handle section start - if ln.startswith("# ![](https://0000.us/klatchat/app/files/" - "neon_images/icons/neon_paw.png)"): - # Old style title line - parsed_data["title"] = ln.split(')', 1)[1].strip() - parsed_data["icon"] = ln.split('(', 1)[1].split(')', 1)[0].strip() - return - elif section == "header" and ln.startswith("# ", 1)[1].strip() - parsed_data["icon"] = ln.split("src=", 1)[1].split()[0]\ - .strip('"').strip("'").lstrip("./") - return "summary" - elif ln.startswith("# ") or ln.startswith("## "): - # Top-level section - if ln.startswith("## About"): - # Handle Mycroft 'About' section - return "description" - elif ln.startswith("## Category"): - # Handle 'Category' as 'Categories' - return "categories" - else: - return line.lstrip("#").strip().lower() - return - - def _format_readme_line(ln: str): - nonlocal category - if section == "incompatible skills": - if not any((ln.startswith('-'), ln.startswith('*'))): - return None - parsed = clean_quotes(ln.lstrip('-').lstrip('*').lower().strip()) - if parsed.startswith('['): - return parsed.split('(', 1)[1].split(')', 1)[0] - return parsed - if section == "examples": - if not any((ln.startswith('-'), ln.startswith('*'))): - return None - parsed = clean_quotes(ln.lstrip('-').lstrip('*').strip()) - if parsed.split(maxsplit=1)[0].lower() == "neon": - return parsed.split(maxsplit=1)[1] - else: - return parsed - if section == "categories": - parsed = ln.rstrip('\n').strip('*') - if ln.startswith('**'): - category = parsed - return parsed - if section == "credits": - if ln.strip().startswith('['): - return ln.split('[', 1)[1].split(']', 1)[0] - return ln.rstrip('\n').lstrip('@') - if section == "tags": - return ln.lstrip('#').rstrip('\n') - if section in list_sections: - return clean_quotes(ln.lstrip('-').lstrip('*').lower().strip()) - return ln.rstrip('\n').rstrip() - - for line in lines: - new_section = _check_section_start(line) - if new_section: - section = new_section - elif line.strip(): - parsed_line = _format_readme_line(line) - if not parsed_line: - # Nothing to parse in this line - continue - if section in list_sections: - if section not in parsed_data: - parsed_data[section] = list() - parsed_data[section].append(parsed_line) - else: - if section not in parsed_data: - parsed_data[section] = parsed_line - else: - parsed_data[section] = " ".join((parsed_data[section], - parsed_line)) - parsed_data["category"] = category or parsed_data.get("categories", - [""])[0] - if parsed_data.get("incompatible skills"): - parsed_data["incompatible_skills"] = \ - parsed_data.pop("incompatible skills") - if parsed_data.get("credits") and len(parsed_data["credits"]) == 1: - parsed_data["credits"] = parsed_data["credits"][0].split(' ') - - return parsed_data + lines = readme.read() + return _get_skill_data_readme(lines) def check_path_permissions(path: str) -> tuple: diff --git a/neon_utils/packaging_utils.py b/neon_utils/packaging_utils.py index 884d830f..12a6bef2 100644 --- a/neon_utils/packaging_utils.py +++ b/neon_utils/packaging_utils.py @@ -35,7 +35,7 @@ import pkg_resources import sysconfig -from os.path import exists, join, expanduser, isdir +from os.path import exists, join, isfile from ovos_utils.log import LOG, deprecated @@ -180,18 +180,19 @@ def get_mycroft_core_root(): raise FileNotFoundError("Could not determine core directory") +@deprecated("Use neon_utils.skill_utils.get_skill_metadata", "2.0.0") def build_skill_spec(skill_dir: str) -> dict: """ Build dict contents of a skill.json file. :param skill_dir: path to skill directory to parse :returns: dict skill.json spec """ - import shutil - from ovos_skills_manager.local_skill import get_skill_data_from_directory - from neon_utils.file_utils import parse_skill_readme_file - from neon_utils.configuration_utils import dict_merge + from neon_utils.skill_utils import get_skill_metadata - def get_skill_license(): # TODO: Implement OSM version of this + # Non-packaged skills are deprecated. Support is patched in here for + # Backwards-compatibility only + + def get_skill_license(): try: with open(join(skill_dir, "LICENSE.md")) as f: contents = f.read() @@ -209,63 +210,23 @@ def get_skill_license(): # TODO: Implement OSM version of this if "Neon AI Non-commercial Friendly License" in contents: return "Neon 1.0" - _invalid_skill_data_keys = ("appstore", "appstore_url", "credits", - "skill_id") - _invalid_readme_keys = ("contact support", "details") - default_skill = {"title": "", - "url": "", - "summary": "", - "short_description": "", - "description": "", - "examples": [], - "desktopFile": False, - "warning": "", - "systemDeps": False, - "requirements": { - "python": [], - "system": {}, - "skill": [] - }, - "incompatible_skills": [], - "platforms": ["i386", - "x86_64", - "ia64", - "arm64", - "arm"], - "branch": "master", - "license": "", - "icon": "", - "category": "", - "categories": [], - "tags": [], - "credits": [], - "skillname": "", - "authorname": "", - "foldername": None} + skill_meta = get_skill_metadata(skill_dir) + if skill_meta.get("license") is None: + skill_meta["license"] = get_skill_license() + + if skill_meta["requirements"].get("python") is None and \ + isfile(join(skill_dir, "requirements.txt")): + try: + with open(join(skill_dir, "requirements.txt")) as f: + requirements = f.read().split('\n') + requirements = [r for r in requirements + if r and not r.startswith('#')] + skill_meta["requirements"]["python"] = requirements + except Exception as e: + LOG.error(e) + skill_meta["requirements"]["python"] = [] - skill_dir = expanduser(skill_dir) - if not isdir(skill_dir): - raise FileNotFoundError(f"Not a Directory: {skill_dir}") - LOG.debug(f"skill_dir={skill_dir}") - skill_json = join(skill_dir, "skill.json") - backup = join(skill_dir, "skill_json.bak") - shutil.move(skill_json, backup) - skill_data = get_skill_data_from_directory(skill_dir) - shutil.move(backup, skill_json) - skill_data['foldername'] = None - for key in _invalid_skill_data_keys: - if key in skill_data: - skill_data.pop(key) - readme_data = parse_skill_readme_file(join(skill_dir, "README.md")) - for key in _invalid_readme_keys: - if key in readme_data: - readme_data.pop(key) - readme_data["short_description"] = readme_data.get("summary") - readme_data["license"] = get_skill_license() - readme_data["branch"] = "master" - skill_data = dict_merge(default_skill, skill_data) - skill_data["requirements"]["python"].sort() - return dict(dict_merge(skill_data, readme_data)) + return skill_meta def install_packages_from_pip(core_module: str, packages: List[str]) -> int: """ diff --git a/neon_utils/skill_utils.py b/neon_utils/skill_utils.py new file mode 100644 index 00000000..0f7c5be3 --- /dev/null +++ b/neon_utils/skill_utils.py @@ -0,0 +1,365 @@ +# NEON AI (TM) SOFTWARE, Software Development Kit & Application Framework +# All trademark and other rights reserved by their respective owners +# Copyright 2008-2025 Neongecko.com Inc. +# Contributors: Daniel McKnight, Guy Daniels, Elon Gasper, Richard Leeds, +# Regina Bloomstine, Casimiro Ferreira, Andrii Pernatii, Kirill Hrymailo +# BSD-3 License +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from this +# software without specific prior written permission. +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, +# OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from os import getcwd, chdir +from os.path import isfile, dirname, abspath, join +from mock import Mock, patch +from importlib.util import spec_from_file_location, module_from_spec + +from ovos_utils.log import LOG + + +def get_skill_metadata(skill_dir: str) -> dict: + """ + Get a metadata object for a skill in a given directory. + """ + setup_py = join(skill_dir, "setup.py") + pyproject_toml = join(skill_dir, "pyproject.toml") + readme_md = join(skill_dir, "README.md") + + if isfile(pyproject_toml): + meta = _get_skill_data_poetry(pyproject_toml) + elif isfile(setup_py): + meta = _get_skill_data_setuptools(setup_py) + elif isfile(readme_md): + with open(readme_md, encoding="utf-8") as f: + readme_data = f.read() + meta = _get_skill_data_readme(readme_data) + else: + raise FileNotFoundError( + f"No setup.py or pyproject.toml found in {skill_dir}" + ) + + # Adding params for backwards-compat. + meta["desktopFile"] = False + meta["warning"] = "" + meta["systemDeps"] = False + meta.setdefault("requirements", {}) + meta["requirements"].setdefault("system", {}) + meta["requirements"].setdefault("skill", []) + meta.setdefault("incompatible_skills", []) + meta["platforms"] = ["i386", "x86_64", "ia64", "arm64", "arm"] + meta["branch"] = "master" + meta["foldername"] = None + meta.setdefault("short_description", meta.get("summary")) + + return meta + + +def _get_skill_data_poetry(pyproject: str) -> dict: + """ + Get skill metadata from a pyproject.toml file + Args: + pyproject: pyproject.toml file to evaluate + Returns: + dict: Skill metadata + """ + from toml import load + + if not isfile(pyproject): + raise FileNotFoundError(f"Not a Directory: {pyproject}") + with open(pyproject, encoding="utf-8") as f: + data = load(f) + skill_data = { + "package_name": data["tool"]["poetry"].get("name", "Unknown"), + "name": data["tool"]["poetry"].get("name", "Unknown"), + "description": data["tool"]["poetry"].get("name", "description"), + "pip_spec": data["tool"]["poetry"].get("name", "Unknown"), + "license": data["tool"]["poetry"].get("license", "Unknown"), + "author": data["tool"]["poetry"].get( + "authors", [""] + ), # List of authors + "tags": data["tool"]["poetry"].get("keywords", []), + "version": data["tool"]["poetry"].get("version", ""), + } + print(data["tool"]["poetry"]["plugins"]) + skill_data["skill_id"] = list( + data["tool"]["poetry"] + .get("plugins", {}) + .get("ovos.plugin.skill", {}) + .keys() + )[0].strip('"') + + # Below match existing Neon skill metadata + skill_data["title"] = skill_data["name"] + skill_data["summary"] = skill_data["description"] + skill_data["short_description"] = skill_data["description"] + skill_data["credits"] = [skill_data["author"]] + skill_data["skillname"] = skill_data["name"] + + python_requirements = data["tool"]["poetry"].get("dependencies", {}) + # Parse requirements into expected string format + python_requirements.pop("python", None) # Remove python version spec + skill_data["requirements"] = { + "python": [f"{k}{v}" for k, v in python_requirements.items()] + } + + if data["tool"]["poetry"].get("readme"): + readme_path = join( + dirname(pyproject), data["tool"]["poetry"]["readme"] + ) + with open(readme_path) as f: + readme_data = f.read() + try: + readme_metadata = _get_skill_data_readme(readme_data) + except Exception as e: + LOG.error(f"Failed to parse README for {skill_data['name']}: {e}") + readme_metadata = {} + return {**readme_metadata, **skill_data} + + +def _get_skill_data_setuptools(setup_py: str) -> dict: + """ + Get skill metadata from a setup.py file by executing it with a mocked setuptools.setup + Args: + setup_py: setup.py file to evaluate + Returns: + dict: Skill metadata + """ + + if not isfile(setup_py): + raise FileNotFoundError(f"File not found: {setup_py}") + + # Create a mock for setuptools.setup + setup_mock = Mock() + + # Mock setuptools.find_packages as well + find_packages_mock = Mock(return_value=[]) + + # Change to the directory containing setup.py to handle relative paths + original_cwd = getcwd() + setup_dir = dirname(abspath(setup_py)) + + try: + chdir(setup_dir) + + # Mock setuptools imports and execute setup.py + with patch.dict( + "sys.modules", + { + "setuptools": Mock( + setup=setup_mock, find_packages=find_packages_mock + ) + }, + ): + # Load and execute the setup.py file + spec = spec_from_file_location("setup", setup_py) + setup_module = module_from_spec(spec) + spec.loader.exec_module(setup_module) + + # Get the captured kwargs from the mock + if not setup_mock.called: + raise ValueError(f"No setup() call found in {setup_py}") + + captured_kwargs = ( + setup_mock.call_args.kwargs + if setup_mock.call_args.kwargs + else setup_mock.call_args.args[0] + if setup_mock.call_args.args + else {} + ) + + finally: + chdir(original_cwd) + + # Build skill_data matching the expected format + skill_data = { + "package_name": captured_kwargs.get("name"), + "pip_spec": captured_kwargs.get("name"), + "license": captured_kwargs.get("license"), + "author": captured_kwargs.get("author"), + "tags": captured_kwargs.get("keywords", []), + "version": captured_kwargs.get("version"), + "url": captured_kwargs.get("url"), + } + + # Extract skill_id from entry_points if available + skill_data["skill_id"] = captured_kwargs["entry_points"][ + "ovos.plugin.skill" + ] + + if "github.com" in skill_data["url"]: + author, skill = skill_data["url"].split("github.com/")[1].split("/") + skill_data["skillname"] = skill + skill_data["authorname"] = author + + # Set additional fields to match existing Neon skill metadata format + skill_data["name"] = skill_data["package_name"] + skill_data["title"] = skill_data["package_name"] + skill_data["credits"] = ( + [skill_data["author"]] if skill_data["author"] != "Unknown" else [] + ) + + # Handle requirements + install_requires = captured_kwargs.get("install_requires", []) + skill_data["requirements"] = { + "python": install_requires + if isinstance(install_requires, list) + else [] + } + + if "long_description" in captured_kwargs: + readme_data = _get_skill_data_readme( + captured_kwargs["long_description"] + ) + else: + readme_data = {} + return {**readme_data, **skill_data} + + +def _get_skill_data_readme(readme_md: str) -> dict: + """ + Get skill metadata from a README file + Args: + readme_md: README.md file contents to evaluate + Returns: + dict: Skill metadata + """ + from neon_utils.parse_utils import clean_quotes + + lines = readme_md.split("\n") + if not lines or len(lines) <= 1: + raise ValueError("Empty README data") + + # Initialize parser params + list_sections = ( + "examples", + "incompatible skills", + "platforms", + "categories", + "tags", + "credits", + ) + valid_sections = list_sections + ( + "summary", + "short_description", + "description", + "warning", + ) + section = "header" + category = None + parsed_data = {} + + def _check_section_start(ln: str): + # Handle section start + if ln.startswith( + "# ![](https://0000.us/klatchat/app/files/" + "neon_images/icons/neon_paw.png)" + ): + # Old style title line + parsed_data["title"] = ln.split(")", 1)[1].strip() + parsed_data["icon"] = ln.split("(", 1)[1].split(")", 1)[0].strip() + return + elif section == "header" and ln.startswith("# ", 1)[1].strip() + parsed_data["icon"] = ( + ln.split("src=", 1)[1] + .split()[0] + .strip('"') + .strip("'") + .lstrip("./") + ) + return "summary" + elif ln.startswith("# ") or ln.startswith("## "): + # Top-level section + if ln.startswith("## About"): + # Handle Mycroft 'About' section + return "description" + elif ln.startswith("## Category"): + # Handle 'Category' as 'Categories' + return "categories" + else: + return line.lstrip("#").strip().lower() + return + + def _format_readme_line(ln: str): + nonlocal category + if section == "incompatible skills": + if not any((ln.startswith("-"), ln.startswith("*"))): + return None + parsed = clean_quotes(ln.lstrip("-").lstrip("*").lower().strip()) + if parsed.startswith("["): + return parsed.split("(", 1)[1].split(")", 1)[0] + return parsed + if section == "examples": + if not any((ln.startswith("-"), ln.startswith("*"))): + return None + parsed = clean_quotes(ln.lstrip("-").lstrip("*").strip()) + if parsed.split(maxsplit=1)[0].lower() == "neon": + return parsed.split(maxsplit=1)[1] + else: + return parsed + if section == "categories": + parsed = ln.rstrip("\n").strip("*") + if ln.startswith("**"): + category = parsed + return parsed + if section == "credits": + if ln.strip().startswith("["): + return ln.split("[", 1)[1].split("]", 1)[0] + return ln.rstrip("\n").lstrip("@") + if section == "tags": + return ln.lstrip("#").rstrip("\n") + if section in list_sections: + return clean_quotes(ln.lstrip("-").lstrip("*").lower().strip()) + return ln.rstrip("\n").rstrip() + + for line in lines: + new_section = _check_section_start(line) + if new_section: + section = new_section + elif line.strip(): + parsed_line = _format_readme_line(line) + if not parsed_line: + # Nothing to parse in this line + continue + if section in list_sections: + if section not in parsed_data: + parsed_data[section] = list() + parsed_data[section].append(parsed_line) + else: + if section not in valid_sections: + continue + elif section not in parsed_data: + parsed_data[section] = parsed_line + else: + parsed_data[section] = " ".join( + (parsed_data[section], parsed_line) + ) + parsed_data["category"] = ( + category or parsed_data.get("categories", [""])[0] + ) + if parsed_data.get("incompatible skills"): + parsed_data["incompatible_skills"] = parsed_data.pop( + "incompatible skills" + ) + if parsed_data.get("credits") and len(parsed_data["credits"]) == 1: + parsed_data["credits"] = parsed_data["credits"][0].split(" ") + + return parsed_data diff --git a/requirements/skills.txt b/requirements/skills.txt new file mode 100644 index 00000000..d2569f96 --- /dev/null +++ b/requirements/skills.txt @@ -0,0 +1,2 @@ +mock +toml~=0.10 diff --git a/requirements/test_requirements.txt b/requirements/test_requirements.txt index 5197c41e..2ba17f40 100644 --- a/requirements/test_requirements.txt +++ b/requirements/test_requirements.txt @@ -5,5 +5,4 @@ mock # TODO: Deprecate ovos-core test dependency ovos-core~=0.0.7,>=0.0.8a21 ovos-plugin-manager~=0.0.18 -ovos-skills-manager -neon-lang-plugin-libretranslate \ No newline at end of file +neon-lang-plugin-libretranslate diff --git a/setup.py b/setup.py index 0235c578..10c9e0dd 100644 --- a/setup.py +++ b/setup.py @@ -87,7 +87,8 @@ def get_requirements(requirements_filename: str): "network": get_requirements("network.txt"), "configuration": get_requirements("configuration.txt"), "sentry": get_requirements("sentry.txt"), - "signal": get_requirements("signal.txt") + "signal": get_requirements("signal.txt"), + "skills": get_requirements("skills.txt"), }, entry_points={ 'console_scripts': [ diff --git a/tests/packaging_util_tests.py b/tests/packaging_util_tests.py index ffbdbf08..9c9a1792 100644 --- a/tests/packaging_util_tests.py +++ b/tests/packaging_util_tests.py @@ -143,7 +143,7 @@ def test_build_skill_spec(self): skill_spec["url"] = "https://github.com/NeonGeckoCom/skill-alerts" with open(join(test_dir, "skill.json")) as f: valid_spec = json.load(f) - self.assertEqual(valid_spec, skill_spec) + self.assertEqual(valid_spec, skill_spec, f"actual={skill_spec}") # TODO: Actually validate exception cases? DM diff --git a/tests/skill_util_tests.py b/tests/skill_util_tests.py new file mode 100644 index 00000000..2fdde461 --- /dev/null +++ b/tests/skill_util_tests.py @@ -0,0 +1,240 @@ +# NEON AI (TM) SOFTWARE, Software Development Kit & Application Framework +# All trademark and other rights reserved by their respective owners +# Copyright 2008-2025 Neongecko.com Inc. +# Contributors: Daniel McKnight, Guy Daniels, Elon Gasper, Richard Leeds, +# Regina Bloomstine, Casimiro Ferreira, Andrii Pernatii, Kirill Hrymailo +# BSD-3 License +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from this +# software without specific prior written permission. +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, +# OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import os +import unittest +import tempfile +import shutil +import subprocess + +from os.path import join, dirname +from mock import patch + + +class SkillUtilTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + """Set up test fixtures, downloading the test skill repository""" + cls.test_dir = tempfile.mkdtemp(prefix="skill_util_tests_") + + # Set up setuptools test repository + cls.setuptools_repo_url = ( + "https://github.com/NeonGeckoCom/skill-caffeinewiz.git" + ) + cls.setuptools_skill_branch = "0.3.1" + cls.setuptools_skill_repo_path = os.path.join( + cls.test_dir, "skill-caffeinewiz" + ) + subprocess.run( + [ + "git", + "clone", + "--branch", + cls.setuptools_skill_branch, + "--depth", + "1", + cls.setuptools_repo_url, + cls.setuptools_skill_repo_path, + ], + check=True, + capture_output=True, + text=True, + timeout=30, + ) + cls.setup_py_path = os.path.join( + cls.setuptools_skill_repo_path, "setup.py" + ) + + # Set up poetry test repository + cls.poetry_repo_url = ( + "https://github.com/OscillateLabsLLC/skill-homeassistant.git" + ) + cls.poetry_skill_branch = "v0.5.1" + cls.poetry_skill_repo_path = os.path.join( + cls.test_dir, "skill-homeassistant" + ) + subprocess.run( + [ + "git", + "clone", + "--branch", + cls.poetry_skill_branch, + "--depth", + "1", + cls.poetry_repo_url, + cls.poetry_skill_repo_path, + ], + check=True, + capture_output=True, + text=True, + timeout=30, + ) + cls.pyproject_toml_path = os.path.join( + cls.poetry_skill_repo_path, "pyproject.toml" + ) + + @classmethod + def tearDownClass(cls): + """Clean up test fixtures""" + if hasattr(cls, "test_dir") and os.path.exists(cls.test_dir): + shutil.rmtree(cls.test_dir) + + @patch("neon_utils.skill_utils._get_skill_data_poetry") + @patch("neon_utils.skill_utils._get_skill_data_setuptools") + def test_get_skill_metadata(self, mock_setuptools, mock_poetry): + """Test that get_skill_metadata calls appropriate internal methods based on file existence""" + from neon_utils.skill_utils import get_skill_metadata + + # Set up mock return values + mock_poetry_data = {"name": "skill-homeassistant", "source": "poetry"} + mock_setuptools_data = { + "name": "skill-caffeinewiz", + "source": "setuptools", + } + mock_poetry.return_value = mock_poetry_data + mock_setuptools.return_value = mock_setuptools_data + + # Test poetry skill (homeassistant) + result_poetry = get_skill_metadata(self.poetry_skill_repo_path) + mock_poetry.assert_called_with(self.pyproject_toml_path) + mock_setuptools.assert_not_called() + self.assertEqual(result_poetry['name'], mock_poetry_data['name']) + + # Reset mocks + mock_poetry.reset_mock() + mock_setuptools.reset_mock() + + # Test setuptools skill (caffeinewiz) + result_setuptools = get_skill_metadata(self.setuptools_skill_repo_path) + mock_setuptools.assert_called_with(self.setup_py_path) + mock_poetry.assert_not_called() + self.assertEqual(result_setuptools['name'], mock_setuptools_data['name']) + # Test FileNotFoundError for non-existent directory + with self.assertRaises(FileNotFoundError): + get_skill_metadata("/non/existent/directory") + + def test_get_skill_data_poetry(self): + from neon_utils.skill_utils import _get_skill_data_poetry + + # Test with real skill-homeassistant pyproject.toml file + self.assertTrue( + os.path.exists(self.pyproject_toml_path), + f"pyproject.toml not found at {self.pyproject_toml_path}", + ) + + skill_metadata = _get_skill_data_poetry(self.pyproject_toml_path) + self.assertIsInstance(skill_metadata, dict) + self.assertEqual(skill_metadata.get("name"), "skill-homeassistant") + self.assertEqual( + skill_metadata.get("package_name"), "skill-homeassistant" + ) + self.assertEqual(skill_metadata.get("pip_spec"), "skill-homeassistant") + self.assertEqual(skill_metadata.get("version"), "0.5.1") + self.assertEqual(skill_metadata.get("license"), "Apache-2.0") + self.assertIn("Mike Gray", skill_metadata.get("author")[0]) + self.assertEqual(skill_metadata.get("title"), "skill-homeassistant") + self.assertIsInstance(skill_metadata.get("tags"), list) + self.assertIn("neon", skill_metadata.get("tags")) + self.assertIsInstance( + skill_metadata.get("requirements", {}).get("python"), list + ) + self.assertGreaterEqual( + len(skill_metadata["requirements"]["python"]), 1 + ) + + # Test params from README + self.assertIsInstance(skill_metadata["summary"], str) + + # Test FileNotFoundError for non-existent file + with self.assertRaises(FileNotFoundError): + _get_skill_data_poetry("non_existent_file.toml") + + def test_get_skill_data_setuptools(self): + from neon_utils.skill_utils import _get_skill_data_setuptools + + # Test with real skill-caffeinewiz setup.py file + self.assertTrue( + os.path.exists(self.setup_py_path), + f"setup.py not found at {self.setup_py_path}", + ) + + skill_metadata = _get_skill_data_setuptools(self.setup_py_path) + self.assertIsInstance(skill_metadata, dict) + self.assertEqual(skill_metadata.get("authorname"), "NeonGeckoCom") + self.assertEqual(skill_metadata.get("skillname"), "skill-caffeinewiz") + self.assertEqual( + skill_metadata["package_name"], "neon-skill-caffeinewiz" + ) + self.assertEqual(skill_metadata["pip_spec"], "neon-skill-caffeinewiz") + self.assertIn("BSD-3", skill_metadata["license"]) + self.assertIsInstance(skill_metadata["author"], str) + self.assertEqual(skill_metadata["version"], "0.3.1") + self.assertIsInstance(skill_metadata["url"], str) + self.assertEqual(skill_metadata["name"], "neon-skill-caffeinewiz") + self.assertEqual(skill_metadata["title"], "neon-skill-caffeinewiz") + self.assertIn("Neongecko", skill_metadata["credits"]) + self.assertIsInstance(skill_metadata["requirements"]["python"], list) + self.assertGreaterEqual( + len(skill_metadata["requirements"]["python"]), 1 + ) + + # Test params from README + self.assertIsInstance(skill_metadata["summary"], str) + + # Test FileNotFoundError for non-existent file + with self.assertRaises(FileNotFoundError): + _get_skill_data_setuptools("non_existent_setup.py") + + def test_get_skill_data_readme(self): + from neon_utils.skill_utils import _get_skill_data_readme + + valid_readme_file = join(dirname(__file__), "test_skill_json", "README.md") + + with open(valid_readme_file, 'r') as f: + readme_contents = f.read() + + skill_metadata = _get_skill_data_readme(readme_contents) + self.assertIsInstance(skill_metadata, dict) + self.assertIsInstance(skill_metadata['examples'], list) + self.assertIsInstance(skill_metadata['incompatible_skills'], list) + self.assertIsInstance(skill_metadata['categories'], list) + self.assertIsInstance(skill_metadata['tags'], list) + self.assertIsInstance(skill_metadata['credits'], list) + + self.assertIsInstance(skill_metadata['summary'], str) + self.assertIsInstance(skill_metadata['description'], str) + + self.assertIsInstance(skill_metadata['title'], str) + self.assertIsInstance(skill_metadata['icon'], str) + + # Test invalid README contents + with self.assertRaises(ValueError): + _get_skill_data_readme("") + +if __name__ == "__main__": + unittest.main() From 0ab9be687c514a95abe73c4e8e55ad9ed163c71d Mon Sep 17 00:00:00 2001 From: NeonDaniel Date: Wed, 10 Sep 2025 21:00:23 +0000 Subject: [PATCH 05/25] Increment Version to 1.13.1a2 --- version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.py b/version.py index 13e226f9..4a1afad6 100644 --- a/version.py +++ b/version.py @@ -26,4 +26,4 @@ # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -__version__ = "1.13.1a1" +__version__ = "1.13.1a2" From 15696fab8c3863445ff956cf845287c3430d7585 Mon Sep 17 00:00:00 2001 From: NeonDaniel Date: Wed, 10 Sep 2025 21:01:14 +0000 Subject: [PATCH 06/25] Update Changelog --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b040df89..d657ed7a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +## [1.13.1a2](https://github.com/NeonGeckoCom/neon-utils/tree/1.13.1a2) (2025-09-10) + +[Full Changelog](https://github.com/NeonGeckoCom/neon-utils/compare/1.13.1a1...1.13.1a2) + +**Merged pull requests:** + +- Improve Skill metadata parsing support and remove OSM dependency [\#558](https://github.com/NeonGeckoCom/neon-utils/pull/558) ([NeonDaniel](https://github.com/NeonDaniel)) + ## [1.13.1a1](https://github.com/NeonGeckoCom/neon-utils/tree/1.13.1a1) (2025-08-27) [Full Changelog](https://github.com/NeonGeckoCom/neon-utils/compare/1.13.0...1.13.1a1) From b4056161507c3c9aebdc5b3942441970447596a3 Mon Sep 17 00:00:00 2001 From: Daniel McKnight <34697904+NeonDaniel@users.noreply.github.com> Date: Wed, 10 Sep 2025 14:45:07 -0700 Subject: [PATCH 07/25] Remove ovos-core test dependency (#557) * Comment out deprecated config handling failing in tests * Clean up configuration_utils Remove deprecated internal methods with no references Remove invalid test case based on default config from ovos_config Mark unused configuraiton methods as deprecated * Remove deprecated tests of removed internal methods * Remove ovos-core from test requirements * Update ovos-plugin-manager test dependency * Make ovos_plugin_manager import safe in deprecated neon_fallback_skill module --- neon_utils/configuration_utils.py | 83 +++++------------------- neon_utils/skills/neon_fallback_skill.py | 7 +- requirements/test_requirements.txt | 4 +- tests/configuration_util_tests.py | 48 +------------- 4 files changed, 23 insertions(+), 119 deletions(-) diff --git a/neon_utils/configuration_utils.py b/neon_utils/configuration_utils.py index a55f64cd..d6d4834e 100644 --- a/neon_utils/configuration_utils.py +++ b/neon_utils/configuration_utils.py @@ -37,7 +37,7 @@ import yaml from copy import deepcopy -from os.path import * +from os.path import join, isfile, isdir, exists, expanduser, dirname, basename, splitext from collections.abc import MutableMapping from contextlib import suppress @@ -302,9 +302,9 @@ def _load_yaml_file(self) -> dict: from ruamel.yaml import YAML config = _make_loaded_config_safe(YAML().load(f)) except ImportError: - LOG.error(f"ruamel.yaml not available to load " - f"legacy config. " - f"pip install neon-utils[configuration]") + LOG.error("ruamel.yaml not available to load " + "legacy config. " + "pip install neon-utils[configuration]") if not config: LOG.debug(f"Empty config file found at: {self.file_path}") config = dict() @@ -325,7 +325,7 @@ def _write_yaml_file(self) -> bool: """ to_write = deepcopy(self._content) if not to_write: - LOG.error(f"Config content empty! Skipping write to disk and reloading") + LOG.error("Config content empty! Skipping write to disk and reloading") return False if not path_is_read_writable(self.file_path): LOG.warning(f"Insufficient write permissions: {self.file_path}") @@ -347,7 +347,7 @@ def _write_yaml_file(self) -> bool: self._disk_content_hash = hash(repr(self._content)) except Exception as e: LOG.error(e) - LOG.info(f"Restoring config from tmp file backup") + LOG.info("Restoring config from tmp file backup") shutil.copy2(tmp_filename, self.file_path) return True @@ -810,7 +810,7 @@ def get_user_config_from_mycroft_conf(user_config: dict = None) -> dict: ["offset"] / 3600000, 1)) else: - LOG.warning(f"No location in core configuration") + LOG.warning("No location in core configuration") return user_config @@ -941,6 +941,8 @@ def get_mycroft_compatible_location(location: dict) -> dict: return location +@deprecated("Configuration should be managed via ovos_config.Configuration", + "2.0.0") def get_mycroft_compatible_config(mycroft_only=False, neon_config_path=None) -> dict: """ @@ -1022,6 +1024,8 @@ def get_mycroft_compatible_config(mycroft_only=False, return default_config +@deprecated("Configuration should be managed via ovos_config.Configuration", + "2.0.0") def write_mycroft_compatible_config(file_to_write: str = None) -> str: """ Generates a mycroft-like configuration and writes it to the specified file @@ -1036,7 +1040,7 @@ def write_mycroft_compatible_config(file_to_write: str = None) -> str: if isfile(file_path): disk_config = load_commented_json(file_path) if disk_config == configuration: - LOG.debug(f"Configuration already up to date") + LOG.debug("Configuration already up to date") return file_path LOG.warning(f"File exists and will be overwritten: {file_to_write}") elif not isdir(dirname(file_path)): @@ -1149,7 +1153,7 @@ def parse_skill_default_settings(settings_meta: dict) -> dict: LOG.error(settings_meta) raise TypeError(f"Expected a dict, got: {type(settings_meta)}") if not settings_meta: - LOG.debug(f"Empty Settings") + LOG.debug("Empty Settings") return dict() else: settings = dict() @@ -1355,7 +1359,7 @@ def _get_neon_skills_config(neon_config_path=None) -> dict: neon_skills["directory_override"] = neon_skills["directory"] except ModuleNotFoundError: LOG.warning("ovos-core not installed") - neon_skills["directory_override"] = neon_skills["directory"] + # neon_skills["directory_override"] = neon_skills["directory"] neon_skills["disable_osm"] = neon_skills.get("skill_manager", "osm") != "osm" neon_skills["priority_skills"] = neon_skills.get("priority") or [] @@ -1381,45 +1385,13 @@ def _get_neon_skills_config(neon_config_path=None) -> dict: neon_skills["neon_token"] = find_neon_git_token() # populate_github_token_config(neon_skills["neon_token"]) except FileNotFoundError: - LOG.debug(f"No Github token found; skills may fail to install") + LOG.debug("No Github token found; skills may fail to install") neon_skills["neon_token"] = None skills_config = {**mycroft_config.get("skills", {}), **neon_skills} skills_config.setdefault("debug", False) return skills_config -@deprecated("Configuration moved to `ovos_config.Configuration`", "2.0.0") -def _get_neon_transcribe_config(neon_config_path=None) -> dict: - """ - Get a configuration dict for the transcription module. - Returns: - dict of config params used for the Neon transcription module - """ - local_config = _get_neon_local_config(neon_config_path) - user_config = get_neon_user_config(neon_config_path) if \ - isfile(join(neon_config_path or get_config_dir(), - "ngi_user_info.yml")) else {} - neon_transcribe_config = dict() - neon_transcribe_config["transcript_dir"] = \ - local_config.get("dirVars", {}).get("docsDir", "") - neon_transcribe_config["audio_permission"] = \ - user_config.get("privacy", {}).get("save_audio", False) - return neon_transcribe_config - - -@deprecated("Configuration moved to `ovos_config.Configuration`", "2.0.0") -def _get_neon_gui_config(neon_config_path=None) -> dict: - """ - Get a configuration dict for the gui module. - Returns: - dict of config params used for the Neon gui module - """ - local_config = _get_neon_local_config(neon_config_path) - gui_config = dict(local_config.get("gui", {})) - gui_config["base_port"] = gui_config.get("port") - return gui_config - - @deprecated("Configuration moved to `ovos_config.Configuration`", "2.0.0") def _safe_mycroft_config() -> dict: """ @@ -1432,31 +1404,6 @@ def _safe_mycroft_config() -> dict: return dict(config) -@deprecated("Configuration moved to `ovos_config.Configuration`", "2.0.0") -def _get_neon_yaml_config() -> dict: - from ovos_config.meta import get_ovos_config - from ovos_config.locations import get_xdg_config_save_path - from ovos_utils.json_helper import merge_dict - - with open(get_ovos_config()["default_config_path"]) as f: - default = yaml.safe_load(f) - config = dict(default) - system_config = os.environ.get("MYCROFT_SYSTEM_CONFIG", - "/etc/neon/neon.yaml") - user_config = join(get_xdg_config_save_path("neon"), "neon.yaml") - if isfile(system_config): - with open(system_config) as f: - system = yaml.safe_load(f) - config = merge_dict(config, system) - - if isfile(user_config): - with open(user_config) as f: - user = yaml.safe_load(f) - config = merge_dict(config, user) - - return config - - @deprecated("Configuration moved to `ovos_config.Configuration`", "2.0.0") def _get_neon_auth_config(path: Optional[str] = None) -> dict: """ diff --git a/neon_utils/skills/neon_fallback_skill.py b/neon_utils/skills/neon_fallback_skill.py index a3098604..0428c775 100644 --- a/neon_utils/skills/neon_fallback_skill.py +++ b/neon_utils/skills/neon_fallback_skill.py @@ -40,7 +40,6 @@ from dateutil.tz import gettz from json_database import JsonStorage from ovos_bus_client import Message -from ovos_plugin_manager.language import OVOSLangDetectionFactory, OVOSLangTranslationFactory from ovos_utils.gui import is_gui_connected from ovos_utils.log import LOG, log_deprecation, deprecated from ovos_utils.skills import get_non_properties @@ -56,6 +55,12 @@ save_settings from neon_utils.user_utils import get_user_prefs +try: + from ovos_plugin_manager.language import OVOSLangDetectionFactory, OVOSLangTranslationFactory +except ImportError: + OVOSLangDetectionFactory = None + OVOSLangTranslationFactory = None + class NeonFallbackSkill(FallbackSkillV1): """ diff --git a/requirements/test_requirements.txt b/requirements/test_requirements.txt index 2ba17f40..acf0f590 100644 --- a/requirements/test_requirements.txt +++ b/requirements/test_requirements.txt @@ -2,7 +2,5 @@ pytest pytest-timeout pytest-cov mock -# TODO: Deprecate ovos-core test dependency -ovos-core~=0.0.7,>=0.0.8a21 -ovos-plugin-manager~=0.0.18 +ovos-plugin-manager~=0.1 neon-lang-plugin-libretranslate diff --git a/tests/configuration_util_tests.py b/tests/configuration_util_tests.py index 73d1b0e7..9a800405 100644 --- a/tests/configuration_util_tests.py +++ b/tests/configuration_util_tests.py @@ -528,7 +528,7 @@ def test_get_mycroft_compat_config(self): self.assertIsInstance(mycroft_config["gui_websocket"]["host"], str) self.assertIsInstance(mycroft_config["gui_websocket"]["base_port"], int) - self.assertIsInstance(mycroft_config["ready_settings"], list) + # self.assertIsInstance(mycroft_config["ready_settings"], list) self.assertIsInstance(mycroft_config['tts'], dict) self.assertIsInstance(mycroft_config["keys"], dict) # self.assertEqual(mycroft_config["skills"]["directory"], @@ -1017,30 +1017,6 @@ def test_migrate_ngi_config(self): os.remove(new_conf) - @mock.patch('neon_utils.packaging_utils.get_neon_core_root') - def test_get_neon_yaml_config(self, get_core_root): - config_dir = join(dirname(__file__), "configuration", - "get_neon_yaml_config") - get_core_root.return_value = join(config_dir, "default") - os.environ["XDG_CONFIG_HOME"] = config_dir - os.environ["MYCROFT_SYSTEM_CONFIG"] = join(config_dir, "system", - "neon.yaml") - from neon_utils.configuration_utils import _get_neon_yaml_config, \ - init_config_dir - init_config_dir() - config = _get_neon_yaml_config() - self.assertEqual(config, - {"config": "user", - "from_default": True, - "from_system": True, - "user": { - "from_system": True, - "from_default": True, - "from_user": True, - "not_from_user": False - }}) - shutil.rmtree(join(config_dir, os.environ["OVOS_CONFIG_BASE_FOLDER"])) - class DeprecatedConfigTests(unittest.TestCase): def doCleanups(self) -> None: @@ -1065,21 +1041,6 @@ def test_get_audio_config(self): self.assertIsInstance(config["tts"], dict) self.assertIsInstance(config["language"], dict) - def test_get_gui_config(self): - from neon_utils.configuration_utils import _get_neon_gui_config - config = _get_neon_gui_config() - self.assertIsInstance(config, dict) - # self.assertIsInstance(config["lang"], str) - # self.assertIsInstance(config["enclosure"], str) - # self.assertIsInstance(config["host"], str) - # self.assertIsInstance(config["port"], int) - # self.assertIsInstance(config["base_port"], int) - # self.assertIsInstance(config["route"], str) - # self.assertIsInstance(config["ssl"], bool) - # self.assertIsInstance(config["resource_root"], str) - # self.assertIn("file_server", config.keys()) - # self.assertEqual(config["port"], config["base_port"]) - def test_get_lang_config(self): from neon_utils.configuration_utils import _get_neon_lang_config config = _get_neon_lang_config() @@ -1091,13 +1052,6 @@ def test_get_lang_config(self): # self.assertIn("boost", config) # self.assertIsInstance(config["libretranslate"], dict) - def test_get_transcribe_config(self): - from neon_utils.configuration_utils import _get_neon_transcribe_config - config = _get_neon_transcribe_config() - self.assertIsInstance(config, dict) - self.assertIsInstance(config["audio_permission"], bool) - self.assertIsInstance(config["transcript_dir"], str) - def test_get_tts_config(self): from neon_utils.configuration_utils import _get_neon_tts_config config = _get_neon_tts_config() From 8a73b3d9ca1b51186f87f5670b2571a9c7f6c695 Mon Sep 17 00:00:00 2001 From: NeonDaniel Date: Wed, 10 Sep 2025 21:45:23 +0000 Subject: [PATCH 08/25] Increment Version to 1.13.1a3 --- version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.py b/version.py index 4a1afad6..2414b0d5 100644 --- a/version.py +++ b/version.py @@ -26,4 +26,4 @@ # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -__version__ = "1.13.1a2" +__version__ = "1.13.1a3" From 634f4ff4e7b50ee990b0a0e4d018f4b0f4129dc6 Mon Sep 17 00:00:00 2001 From: NeonDaniel Date: Wed, 10 Sep 2025 21:46:14 +0000 Subject: [PATCH 09/25] Update Changelog --- CHANGELOG.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d657ed7a..4c58d5d3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,17 @@ # Changelog +## [1.13.1a3](https://github.com/NeonGeckoCom/neon-utils/tree/1.13.1a3) (2025-09-10) + +[Full Changelog](https://github.com/NeonGeckoCom/neon-utils/compare/1.13.1a2...1.13.1a3) + +**Implemented enhancements:** + +- \[FEAT\] Deprecate ovos-core dependency [\#469](https://github.com/NeonGeckoCom/neon-utils/issues/469) + +**Merged pull requests:** + +- Remove ovos-core test dependency [\#557](https://github.com/NeonGeckoCom/neon-utils/pull/557) ([NeonDaniel](https://github.com/NeonDaniel)) + ## [1.13.1a2](https://github.com/NeonGeckoCom/neon-utils/tree/1.13.1a2) (2025-09-10) [Full Changelog](https://github.com/NeonGeckoCom/neon-utils/compare/1.13.1a1...1.13.1a2) From b8587087f727b2775261d9627a3d1a1e7a66f9d3 Mon Sep 17 00:00:00 2001 From: Daniel McKnight <34697904+NeonDaniel@users.noreply.github.com> Date: Wed, 10 Sep 2025 18:06:41 -0700 Subject: [PATCH 10/25] Update pip package installation method (#556) * Update pip package installation method * Remove variable used only for logging --- neon_utils/packaging_utils.py | 32 ++++++++++++++++++++++---------- tests/packaging_util_tests.py | 13 ++++++------- 2 files changed, 28 insertions(+), 17 deletions(-) diff --git a/neon_utils/packaging_utils.py b/neon_utils/packaging_utils.py index 12a6bef2..a8d7ea42 100644 --- a/neon_utils/packaging_utils.py +++ b/neon_utils/packaging_utils.py @@ -28,6 +28,7 @@ import sys import re +import subprocess import importlib.util from typing import Tuple, Optional, List from tempfile import mkstemp @@ -228,19 +229,26 @@ def get_skill_license(): return skill_meta -def install_packages_from_pip(core_module: str, packages: List[str]) -> int: +def install_packages_from_pip(core_module: str, packages: List[str], + force_reinstall: bool = False) -> int: """ Install a Python package using pip :param core_module: string neon core module to install dependency for :param packages: List(string) list of packages to install + :param force_reinstall: force re-installation of packages :returns: int pip exit code """ - import pip + def _pip_install(command_args: List[str]) -> int: + try: + result = subprocess.check_call([sys.executable, '-m', 'pip'] + command_args) + return result + except subprocess.CalledProcessError as e: + LOG.error(f"Error installing {command_args}: {e}") + return e.returncode + _, tmp_constraints_file = mkstemp() _, tmp_requirements_file = mkstemp() - install_str = " ".join(packages) - with open(tmp_constraints_file, 'w', encoding="utf8") as f: constraints = '\n'.join(get_package_dependencies(core_module)) f.write(constraints) @@ -250,10 +258,14 @@ def install_packages_from_pip(core_module: str, packages: List[str]) -> int: for pkg in packages: f.write(f"{pkg}\n") - LOG.info(f"Requested installation of plugins: {install_str}") + LOG.info(f"Requested installation of plugins: {packages}") pip_args = ['install', '-r', tmp_requirements_file, '-c', tmp_constraints_file] - result = pip.main(pip_args) if hasattr(pip, 'main') else pip._internal.main(pip_args) - - if result != 0: - return result - return 0 + if stat := _pip_install(pip_args) != 0: + return stat + + if force_reinstall: + LOG.info(f"Requested forced re-installation of plugins: {packages}") + pip_args.extend(['--no-deps', '--force-reinstall']) + stat = _pip_install(pip_args) + + return stat diff --git a/tests/packaging_util_tests.py b/tests/packaging_util_tests.py index 9c9a1792..8a8c2077 100644 --- a/tests/packaging_util_tests.py +++ b/tests/packaging_util_tests.py @@ -148,26 +148,25 @@ def test_build_skill_spec(self): # TODO: Actually validate exception cases? DM def test_install_packages_from_pip(self): - import pip + import subprocess from neon_utils.packaging_utils import install_packages_from_pip - with patch.object(pip, 'main', return_value=0) as mock_method: + with patch.object(subprocess, 'check_call', return_value=0) as mock_method: test_result = install_packages_from_pip("neon-utils", ["pip-install-test"]) args, kwargs = mock_method.call_args self.assertEqual(0, test_result) - mock_method.assert_called_once_with(['install', '-r', args[0][2], '-c', args[0][4]]) + mock_method.assert_called_once_with([sys.executable, '-m', 'pip', 'install', '-r', args[0][5], '-c', args[0][7]]) - with open(args[0][2], "r", encoding="utf8") as f: + with open(args[0][5], "r", encoding="utf8") as f: line = f.readline() self.assertEqual("pip-install-test\n", line) mock_method.reset_mock() - test_result = install_packages_from_pip("neon-utils", ["pip-install-test", "pip-install-another"]) + test_result = install_packages_from_pip("neon-utils", ["pip-install-test", "pip-install-another"], force_reinstall=True) self.assertEqual(0, test_result) - mock_method.assert_called_once() - + self.assertEqual(mock_method.call_count, 2) if __name__ == '__main__': From d53f5597831a7f568fc42f06c2bd6fc6e9e9cd57 Mon Sep 17 00:00:00 2001 From: NeonDaniel Date: Thu, 11 Sep 2025 01:06:58 +0000 Subject: [PATCH 11/25] Increment Version to 1.13.1a4 --- version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.py b/version.py index 2414b0d5..02560e93 100644 --- a/version.py +++ b/version.py @@ -26,4 +26,4 @@ # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -__version__ = "1.13.1a3" +__version__ = "1.13.1a4" From 02041b39547676ff35fd5249e19f2e2e6215e02e Mon Sep 17 00:00:00 2001 From: NeonDaniel Date: Thu, 11 Sep 2025 01:07:35 +0000 Subject: [PATCH 12/25] Update Changelog --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4c58d5d3..81371ef7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +## [1.13.1a4](https://github.com/NeonGeckoCom/neon-utils/tree/1.13.1a4) (2025-09-11) + +[Full Changelog](https://github.com/NeonGeckoCom/neon-utils/compare/1.13.1a3...1.13.1a4) + +**Merged pull requests:** + +- Update pip package installation method [\#556](https://github.com/NeonGeckoCom/neon-utils/pull/556) ([NeonDaniel](https://github.com/NeonDaniel)) + ## [1.13.1a3](https://github.com/NeonGeckoCom/neon-utils/tree/1.13.1a3) (2025-09-10) [Full Changelog](https://github.com/NeonGeckoCom/neon-utils/compare/1.13.1a2...1.13.1a3) From 56d7795d54f24eeda937f75a91f0120a28bb485d Mon Sep 17 00:00:00 2001 From: Daniel McKnight <34697904+NeonDaniel@users.noreply.github.com> Date: Thu, 11 Sep 2025 18:12:04 -0700 Subject: [PATCH 13/25] Refactor log aggregators init (#560) * Refactor log aggregators init * More specific log aggregator exception handling * Update logging around log aggregator init errors --- neon_utils/log_aggregators/__init__.py | 13 +++++++++---- neon_utils/log_aggregators/sentry.py | 3 ++- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/neon_utils/log_aggregators/__init__.py b/neon_utils/log_aggregators/__init__.py index 0196509f..7fe7a21d 100644 --- a/neon_utils/log_aggregators/__init__.py +++ b/neon_utils/log_aggregators/__init__.py @@ -28,6 +28,8 @@ import importlib +from ovos_utils.log import LOG + _service_name_to_handler = { 'sentry': 'init_sentry' } @@ -37,10 +39,13 @@ def init_log_aggregators(config: dict = None): from ovos_config.config import Configuration config = config or Configuration() for service_name, handler in _service_name_to_handler.items(): - service_config = _get_log_aggregator_config(config=config, name=service_name) - if bool(service_config.pop('enabled', False)): - service_module = importlib.import_module(f'.{service_name}', __name__) - getattr(service_module, handler)(config=service_config) + try: + service_config = _get_log_aggregator_config(config=config, name=service_name) + if service_config.pop('enabled', False) is True: + service_module = importlib.import_module(f'.{service_name}', __name__) + getattr(service_module, handler)(config=service_config) + except ModuleNotFoundError as e: + LOG.exception(f"Failed to load log aggregator {service_name}: {e}") def _get_log_aggregator_config(config: dict, name: str): diff --git a/neon_utils/log_aggregators/sentry.py b/neon_utils/log_aggregators/sentry.py index f57907a3..a3ad2e8d 100644 --- a/neon_utils/log_aggregators/sentry.py +++ b/neon_utils/log_aggregators/sentry.py @@ -26,12 +26,13 @@ # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -import sentry_sdk SENTRY_SDK_REQUIRED_KEYS = {'dsn'} def init_sentry(config: dict): + import sentry_sdk + missing_required_keys = SENTRY_SDK_REQUIRED_KEYS.difference(config.keys()) if missing_required_keys: raise KeyError(f'Sentry SDK configuration missing required keys: {missing_required_keys}') From eaaf58553964b08701e93e763537da54db4961cd Mon Sep 17 00:00:00 2001 From: NeonDaniel Date: Fri, 12 Sep 2025 01:12:16 +0000 Subject: [PATCH 14/25] Increment Version to 1.13.1a5 --- version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.py b/version.py index 02560e93..33c2f2d4 100644 --- a/version.py +++ b/version.py @@ -26,4 +26,4 @@ # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -__version__ = "1.13.1a4" +__version__ = "1.13.1a5" From cff6a5b8b1456320215c63566e00a171ddf7788d Mon Sep 17 00:00:00 2001 From: NeonDaniel Date: Fri, 12 Sep 2025 01:12:58 +0000 Subject: [PATCH 15/25] Update Changelog --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 81371ef7..8068f43d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +## [1.13.1a5](https://github.com/NeonGeckoCom/neon-utils/tree/1.13.1a5) (2025-09-12) + +[Full Changelog](https://github.com/NeonGeckoCom/neon-utils/compare/1.13.1a4...1.13.1a5) + +**Merged pull requests:** + +- Refactor log aggregators init [\#560](https://github.com/NeonGeckoCom/neon-utils/pull/560) ([NeonDaniel](https://github.com/NeonDaniel)) + ## [1.13.1a4](https://github.com/NeonGeckoCom/neon-utils/tree/1.13.1a4) (2025-09-11) [Full Changelog](https://github.com/NeonGeckoCom/neon-utils/compare/1.13.1a3...1.13.1a4) From f9d6cd118e70ad5220a860218eef97a7418a859f Mon Sep 17 00:00:00 2001 From: Daniel McKnight <34697904+NeonDaniel@users.noreply.github.com> Date: Thu, 11 Sep 2025 19:13:11 -0700 Subject: [PATCH 16/25] Add support for authenticated HANA (#537) * Add support for configured hana username/password Add unit test case for invalid login * Fix typo in test config patching * Refactor token name to be URL-safe Fix logical error in unit test * Update test case to account for refreshed token expiration rounding to the same value --- neon_utils/hana_utils.py | 17 +++++++++++------ tests/hana_util_tests.py | 18 +++++++++++++----- 2 files changed, 24 insertions(+), 11 deletions(-) diff --git a/neon_utils/hana_utils.py b/neon_utils/hana_utils.py index 91caeb80..66c09b61 100644 --- a/neon_utils/hana_utils.py +++ b/neon_utils/hana_utils.py @@ -25,6 +25,8 @@ # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +from _socket import gethostname +from datetime import datetime import requests import json @@ -33,6 +35,7 @@ from os import makedirs, remove from os.path import join, isfile, isdir, dirname from time import time + from ovos_utils.log import LOG from ovos_utils.xdg_utils import xdg_cache_home @@ -92,21 +95,23 @@ def _init_client(backend_address: str, ssl_verify: bool = True): _headers = {"Authorization": f"Bearer {_client_config['access_token']}"} -def _get_token(backend_address: str, username: str = "guest", - password: str = "password", +def _get_token(backend_address: str, ssl_verify: bool = True): """ Get new auth tokens from the specified server. This will cache the returned token, overwriting any previous data at the cache path. @param backend_address: Hana server URL to connect to - @param username: Username to authorize - @param password: Password for specified username """ + from ovos_config.config import Configuration global _client_config - # TODO: username/password from configuration + hana_config = Configuration().get('hana', {}) + username = hana_config.get("username") or "guest" + password = hana_config.get("password") or "password" + token_name = f"{gethostname()}_{datetime.utcnow().isoformat()}" resp = requests.post(f"{backend_address}/auth/login", json={"username": username, - "password": password}, + "password": password, + "token_name": token_name}, verify=ssl_verify) if not resp.ok: raise ServerException(f"Error logging into {backend_address}. " diff --git a/tests/hana_util_tests.py b/tests/hana_util_tests.py index 3ac38989..27ece7af 100644 --- a/tests/hana_util_tests.py +++ b/tests/hana_util_tests.py @@ -121,18 +121,26 @@ def test_request_backend_refresh_token(self, refresh_token, config_path): neon_utils.hana_utils._client_config = real_client_config @patch("neon_utils.hana_utils._get_client_config_path") - def test_00_get_token(self, config_path): + @patch("ovos_config.config.Configuration") + def test_00_get_token(self, config, config_path): + config.return_value = {} config_path.return_value = self.test_path from neon_utils.hana_utils import _get_token - # Test valid request + # Test valid default request _get_token(self.test_server) from neon_utils.hana_utils import _client_config self.assertTrue(isfile(self.test_path)) with open(self.test_path) as f: credentials_on_disk = json.load(f) self.assertEqual(credentials_on_disk, _client_config) - # TODO: Test invalid request, rate-limited request + + # Test with configured invalid login + config.return_value = {"hana": {"username": "guest", + "password": "fake_password"}} + from neon_utils.hana_utils import ServerException + with self.assertRaises(ServerException): + _get_token(self.test_server) @patch("neon_utils.hana_utils._get_client_config_path") @patch("neon_utils.hana_utils._get_token") @@ -170,8 +178,8 @@ def _write_token(*_, **__): new_credentials['client_id']) self.assertEqual(credentials_on_disk['username'], new_credentials['username']) - self.assertGreater(new_credentials['expiration'], - credentials_on_disk['expiration']) + self.assertGreaterEqual(new_credentials['expiration'], + credentials_on_disk['expiration']) self.assertNotEqual(credentials_on_disk['access_token'], new_credentials['access_token']) self.assertNotEqual(credentials_on_disk['refresh_token'], From a177594000f6e85e0178cfb63f68ed4bebe69df2 Mon Sep 17 00:00:00 2001 From: NeonDaniel Date: Fri, 12 Sep 2025 02:13:26 +0000 Subject: [PATCH 17/25] Increment Version to 1.13.1a6 --- version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.py b/version.py index 33c2f2d4..a4daefd3 100644 --- a/version.py +++ b/version.py @@ -26,4 +26,4 @@ # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -__version__ = "1.13.1a5" +__version__ = "1.13.1a6" From 8889182d2135730a2ee7de07990ffe27aa54bf23 Mon Sep 17 00:00:00 2001 From: NeonDaniel Date: Fri, 12 Sep 2025 02:13:58 +0000 Subject: [PATCH 18/25] Update Changelog --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8068f43d..28901dc6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +## [1.13.1a6](https://github.com/NeonGeckoCom/neon-utils/tree/1.13.1a6) (2025-09-12) + +[Full Changelog](https://github.com/NeonGeckoCom/neon-utils/compare/1.13.1a5...1.13.1a6) + +**Merged pull requests:** + +- Add support for authenticated HANA [\#537](https://github.com/NeonGeckoCom/neon-utils/pull/537) ([NeonDaniel](https://github.com/NeonDaniel)) + ## [1.13.1a5](https://github.com/NeonGeckoCom/neon-utils/tree/1.13.1a5) (2025-09-12) [Full Changelog](https://github.com/NeonGeckoCom/neon-utils/compare/1.13.1a4...1.13.1a5) From 668420f9d5d5736eaafcb56f1818a19e522c4bd5 Mon Sep 17 00:00:00 2001 From: Daniel McKnight <34697904+NeonDaniel@users.noreply.github.com> Date: Fri, 12 Sep 2025 09:37:39 -0700 Subject: [PATCH 19/25] Refactor to remove use of deprecated `pkg_resources` module (#555) * Refactor to use importlib instead of pkg_resources in `packaging_utils` * Update unit test to account for valid behavior change * Update packaging_utils and tests for backwards-compat * Set `max-parallel` jobs to prevent HANA rate-limiting errors --- .github/workflows/unit_tests.yml | 1 + neon_utils/packaging_utils.py | 33 +++++++++++++++++++------------- tests/packaging_util_tests.py | 13 +++++++++---- 3 files changed, 30 insertions(+), 17 deletions(-) diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 877f823c..59dfcc38 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -10,6 +10,7 @@ jobs: uses: neongeckocom/.github/.github/workflows/python_build_tests.yml@master skill_object_tests: strategy: + max-parallel: 2 matrix: python-version: [ 3.9, '3.10', '3.11', '3.12' ] runs-on: ubuntu-latest diff --git a/neon_utils/packaging_utils.py b/neon_utils/packaging_utils.py index a8d7ea42..cde4d034 100644 --- a/neon_utils/packaging_utils.py +++ b/neon_utils/packaging_utils.py @@ -33,7 +33,6 @@ from typing import Tuple, Optional, List from tempfile import mkstemp -import pkg_resources import sysconfig from os.path import exists, join, isfile @@ -63,13 +62,16 @@ def get_package_version_spec(pkg: str): """ Locate an installed package and return its reported version :param pkg: string package name to locate - :returns: Version string as reported by pkg_resources - :raises: ModuleNotFoundError if requested package isn't installed + :returns: Version string as reported by importlib.metadata + :raises: PackageNotFoundError if requested package isn't installed """ try: - return pkg_resources.get_distribution(pkg).version - except pkg_resources.DistributionNotFound: - raise ModuleNotFoundError(f"{pkg} not found") + from importlib.metadata import version + except ImportError: + # Fallback for Python < 3.8 + from importlib_metadata import version + + return version(pkg) def get_package_dependencies(pkg: str): @@ -77,15 +79,20 @@ def get_package_dependencies(pkg: str): Get the dependencies for an installed package :param pkg: string package name to evaluate :returns: list of string dependencies (equivalent to requirements.txt) - :raises ModuleNotFoundError if requested package isn't installed + :raises PackageNotFoundError if requested package isn't installed """ try: - constraints = pkg_resources.working_set.by_key[pkg].requires() - constraints_spec = [str(c).split('[', 1)[0] for c in constraints] - LOG.debug(constraints_spec) - return constraints_spec - except KeyError: - raise ModuleNotFoundError(f"{pkg} not found") + from importlib.metadata import requires + except ImportError: + # Fallback for Python < 3.8 + from importlib_metadata import requires + + requirements = requires(pkg) + if requirements is None: + return [] + constraints_spec = [req.split('[', 1)[0].split(';', 1)[0] for req in requirements] + LOG.debug(constraints_spec) + return constraints_spec @deprecated("Reference `neon_core.version.__version__` directly", "2.0.0") diff --git a/tests/packaging_util_tests.py b/tests/packaging_util_tests.py index 8a8c2077..6a7e67a5 100644 --- a/tests/packaging_util_tests.py +++ b/tests/packaging_util_tests.py @@ -114,13 +114,18 @@ def test_get_packaged_core_version(self): def test_get_package_dependencies(self): self_deps = get_package_dependencies("neon-utils") - requirements_file = join(os.path.dirname(os.path.dirname(__file__)), - "requirements", "requirements.txt") - with open(requirements_file) as f: - spec_requirements = f.read().split('\n') + requirements_dir = os.path.join(os.path.dirname(os.path.dirname(__file__)), "requirements") + spec_requirements = [] + for _, _, files in os.walk(requirements_dir): + for file in files: + with open(os.path.join(requirements_dir, file)) as f: + spec_requirements += f.read().split('\n') + spec_requirements = [r for r in spec_requirements if r and not r.startswith('#')] + # Version specs aren't order-dependent, so they can't be compared + # Also, constraints are normalized so elements cannot be compared self.assertEqual(len(self_deps), len(spec_requirements)) with self.assertRaises(ModuleNotFoundError): get_package_dependencies("fakeneongeckopackage") From d50c45aa67127c65fcd7cb0e1294d3deb52cf17c Mon Sep 17 00:00:00 2001 From: NeonDaniel Date: Fri, 12 Sep 2025 16:37:53 +0000 Subject: [PATCH 20/25] Increment Version to 1.13.1a7 --- version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.py b/version.py index a4daefd3..c58a9884 100644 --- a/version.py +++ b/version.py @@ -26,4 +26,4 @@ # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -__version__ = "1.13.1a6" +__version__ = "1.13.1a7" From 23d0b4674a2c89d8341599cbbc2cba102268f89a Mon Sep 17 00:00:00 2001 From: NeonDaniel Date: Fri, 12 Sep 2025 16:38:44 +0000 Subject: [PATCH 21/25] Update Changelog --- CHANGELOG.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 28901dc6..5b2d2563 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,18 @@ # Changelog +## [1.13.1a7](https://github.com/NeonGeckoCom/neon-utils/tree/1.13.1a7) (2025-09-12) + +[Full Changelog](https://github.com/NeonGeckoCom/neon-utils/compare/1.13.1a6...1.13.1a7) + +**Implemented enhancements:** + +- \[FEAT\] Allow self-signed certificates in hana-utils [\#536](https://github.com/NeonGeckoCom/neon-utils/issues/536) +- \[FEAT\] Docker handling of status hooks [\#498](https://github.com/NeonGeckoCom/neon-utils/issues/498) + +**Merged pull requests:** + +- Refactor to remove use of deprecated `pkg_resources` module [\#555](https://github.com/NeonGeckoCom/neon-utils/pull/555) ([NeonDaniel](https://github.com/NeonDaniel)) + ## [1.13.1a6](https://github.com/NeonGeckoCom/neon-utils/tree/1.13.1a6) (2025-09-12) [Full Changelog](https://github.com/NeonGeckoCom/neon-utils/compare/1.13.1a5...1.13.1a6) From d57ec2d6a26928f6d4dd3ebfadc919c6b43421c4 Mon Sep 17 00:00:00 2001 From: Daniel McKnight <34697904+NeonDaniel@users.noreply.github.com> Date: Fri, 12 Sep 2025 13:17:12 -0700 Subject: [PATCH 22/25] Add function to get installed prerelease packages (#517) * Add function to get installed prerelease packages * Handle `post` releases with updated test * Update to remove column headers with updated unit test case --- neon_utils/packaging_utils.py | 24 ++++++++++++++++++++++++ tests/packaging_util_tests.py | 17 +++++++++++++++++ 2 files changed, 41 insertions(+) diff --git a/neon_utils/packaging_utils.py b/neon_utils/packaging_utils.py index cde4d034..ae6381a9 100644 --- a/neon_utils/packaging_utils.py +++ b/neon_utils/packaging_utils.py @@ -276,3 +276,27 @@ def _pip_install(command_args: List[str]) -> int: stat = _pip_install(pip_args) return stat + + +def get_installed_prereleases() -> List[Tuple[str, str]]: + """ + Get a list of installed pre-release packages. + @return: List of tuple (pkg_name, version) + """ + from subprocess import run + packages = run(["pip", "list"], + capture_output=True).stdout.decode("utf-8") + prerelease_pkgs = list() + for line in packages.split('\n'): + if not line: + continue + name, version = line.split() + if name == "Package" or not name.replace('-', ''): + continue + if not version.replace('.', '').isnumeric(): + if "post" in version: + LOG.debug(f"post release {name}:{version}") + continue + prerelease_pkgs.append((name, version)) + return prerelease_pkgs + diff --git a/tests/packaging_util_tests.py b/tests/packaging_util_tests.py index 6a7e67a5..10fc7607 100644 --- a/tests/packaging_util_tests.py +++ b/tests/packaging_util_tests.py @@ -173,6 +173,23 @@ def test_install_packages_from_pip(self): self.assertEqual(0, test_result) self.assertEqual(mock_method.call_count, 2) + @patch("subprocess.run") + def test_get_installed_prereleases(self, run): + run.return_value.stdout = """Package Version +----------------------- ------------ +stable_package 1.0.0 +beta_package 0.2.2b3 +alpha_package 0.0.0a0 +date_package 24.4.30 +post_package 2.0.0post10 +""".encode("utf-8") + from neon_utils.packaging_utils import get_installed_prereleases + prereleases = get_installed_prereleases() + self.assertEqual(len(prereleases), 2) + for pkg in prereleases: + self.assertTrue(pkg[0].endswith("_package")) + self.assertEqual(len(pkg[1].split('.')), 3) + if __name__ == '__main__': unittest.main() From 7a92606abd6d91751066a60ca4e9543680be76d8 Mon Sep 17 00:00:00 2001 From: NeonDaniel Date: Fri, 12 Sep 2025 20:17:30 +0000 Subject: [PATCH 23/25] Increment Version to 1.13.1a8 --- version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.py b/version.py index c58a9884..724bc00b 100644 --- a/version.py +++ b/version.py @@ -26,4 +26,4 @@ # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -__version__ = "1.13.1a7" +__version__ = "1.13.1a8" From 406e811bbde2bf186fb738f355c30558bdcd95e5 Mon Sep 17 00:00:00 2001 From: NeonDaniel Date: Fri, 12 Sep 2025 20:18:08 +0000 Subject: [PATCH 24/25] Update Changelog --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5b2d2563..1544e5a9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +## [1.13.1a8](https://github.com/NeonGeckoCom/neon-utils/tree/1.13.1a8) (2025-09-12) + +[Full Changelog](https://github.com/NeonGeckoCom/neon-utils/compare/1.13.1a7...1.13.1a8) + +**Merged pull requests:** + +- Add function to get installed prerelease packages [\#517](https://github.com/NeonGeckoCom/neon-utils/pull/517) ([NeonDaniel](https://github.com/NeonDaniel)) + ## [1.13.1a7](https://github.com/NeonGeckoCom/neon-utils/tree/1.13.1a7) (2025-09-12) [Full Changelog](https://github.com/NeonGeckoCom/neon-utils/compare/1.13.1a6...1.13.1a7) From cadff3dd046711df635cb03811afda3c85783957 Mon Sep 17 00:00:00 2001 From: NeonDaniel Date: Fri, 12 Sep 2025 20:45:50 +0000 Subject: [PATCH 25/25] Increment Version to 1.14.0 --- version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.py b/version.py index 724bc00b..f6ec4d9c 100644 --- a/version.py +++ b/version.py @@ -26,4 +26,4 @@ # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -__version__ = "1.13.1a8" +__version__ = "1.14.0"