diff --git a/.github/workflows/license_tests.yml b/.github/workflows/license_tests.yml
index 011f7454..7adc134e 100644
--- a/.github/workflows/license_tests.yml
+++ b/.github/workflows/license_tests.yml
@@ -11,4 +11,4 @@ jobs:
uses: neongeckocom/.github/.github/workflows/license_tests.yml@master
with:
package-extras: audio,configuration,networking
- packages-exclude: '^(precise-runner|fann2|tqdm|bs4|ovos-phal-plugin|ovos-skill|neon-core|nvidia|neon-phal-plugin|bitstruct|audioread|RapidFuzz|click|setuptools|typing_extensions|urllib).*'
\ No newline at end of file
+ packages-exclude: '^(precise-runner|fann2|tqdm|bs4|ovos-phal-plugin|ovos-skill|neon-core|nvidia|neon-phal-plugin|bitstruct|audioread|RapidFuzz|click|setuptools|typing_extensions|urllib|marisa-trie).*'
diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml
index 5219d066..59dfcc38 100644
--- a/.github/workflows/unit_tests.yml
+++ b/.github/workflows/unit_tests.yml
@@ -10,6 +10,7 @@ jobs:
uses: neongeckocom/.github/.github/workflows/python_build_tests.yml@master
skill_object_tests:
strategy:
+ max-parallel: 2
matrix:
python-version: [ 3.9, '3.10', '3.11', '3.12' ]
runs-on: ubuntu-latest
@@ -58,7 +59,7 @@ jobs:
python -m pip install --upgrade pip setuptools
pip install wheel "cython<3.0.0" # TODO: cython patching https://github.com/yaml/pyyaml/issues/724
pip install --no-build-isolation pyyaml~=5.4 # TODO: patching https://github.com/yaml/pyyaml/issues/724
- pip install .[test,audio,network,configuration]
+ pip install .[test,audio,network,configuration,skills]
- name: Change Test File Permissions
run: |
sudo chown -R nobody:nogroup tests/configuration/unwritable_path
@@ -223,4 +224,13 @@ jobs:
uses: actions/upload-artifact@v4
with:
name: hana-util-test-results-${{ matrix.python-version }}
- path: tests/hana-util-test-results.xml
\ No newline at end of file
+ path: tests/hana-util-test-results.xml
+
+ - name: Test Skill Utils
+ run: |
+ pytest tests/skill_util_tests.py --doctest-modules --junitxml=tests/skill-util-test-results.xml
+ - name: Upload skill utils test results
+ uses: actions/upload-artifact@v4
+ with:
+ name: skill-util-test-results-${{ matrix.python-version }}
+ path: tests/skill-util-test-results.xml
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 50657204..1544e5a9 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,28 +1,77 @@
# Changelog
-## [1.12.2a3](https://github.com/NeonGeckoCom/neon-utils/tree/1.12.2a3) (2025-07-02)
+## [1.13.1a8](https://github.com/NeonGeckoCom/neon-utils/tree/1.13.1a8) (2025-09-12)
-[Full Changelog](https://github.com/NeonGeckoCom/neon-utils/compare/1.12.2a2...1.12.2a3)
+[Full Changelog](https://github.com/NeonGeckoCom/neon-utils/compare/1.13.1a7...1.13.1a8)
**Merged pull requests:**
-- Override `/status` server endpoint logging [\#552](https://github.com/NeonGeckoCom/neon-utils/pull/552) ([NeonDaniel](https://github.com/NeonDaniel))
+- Add function to get installed prerelease packages [\#517](https://github.com/NeonGeckoCom/neon-utils/pull/517) ([NeonDaniel](https://github.com/NeonDaniel))
-## [1.12.2a2](https://github.com/NeonGeckoCom/neon-utils/tree/1.12.2a2) (2025-06-25)
+## [1.13.1a7](https://github.com/NeonGeckoCom/neon-utils/tree/1.13.1a7) (2025-09-12)
-[Full Changelog](https://github.com/NeonGeckoCom/neon-utils/compare/1.12.2a1...1.12.2a2)
+[Full Changelog](https://github.com/NeonGeckoCom/neon-utils/compare/1.13.1a6...1.13.1a7)
+
+**Implemented enhancements:**
+
+- \[FEAT\] Allow self-signed certificates in hana-utils [\#536](https://github.com/NeonGeckoCom/neon-utils/issues/536)
+- \[FEAT\] Docker handling of status hooks [\#498](https://github.com/NeonGeckoCom/neon-utils/issues/498)
+
+**Merged pull requests:**
+
+- Refactor to remove use of deprecated `pkg_resources` module [\#555](https://github.com/NeonGeckoCom/neon-utils/pull/555) ([NeonDaniel](https://github.com/NeonDaniel))
+
+## [1.13.1a6](https://github.com/NeonGeckoCom/neon-utils/tree/1.13.1a6) (2025-09-12)
+
+[Full Changelog](https://github.com/NeonGeckoCom/neon-utils/compare/1.13.1a5...1.13.1a6)
+
+**Merged pull requests:**
+
+- Add support for authenticated HANA [\#537](https://github.com/NeonGeckoCom/neon-utils/pull/537) ([NeonDaniel](https://github.com/NeonDaniel))
+
+## [1.13.1a5](https://github.com/NeonGeckoCom/neon-utils/tree/1.13.1a5) (2025-09-12)
+
+[Full Changelog](https://github.com/NeonGeckoCom/neon-utils/compare/1.13.1a4...1.13.1a5)
+
+**Merged pull requests:**
+
+- Refactor log aggregators init [\#560](https://github.com/NeonGeckoCom/neon-utils/pull/560) ([NeonDaniel](https://github.com/NeonDaniel))
+
+## [1.13.1a4](https://github.com/NeonGeckoCom/neon-utils/tree/1.13.1a4) (2025-09-11)
+
+[Full Changelog](https://github.com/NeonGeckoCom/neon-utils/compare/1.13.1a3...1.13.1a4)
+
+**Merged pull requests:**
+
+- Update pip package installation method [\#556](https://github.com/NeonGeckoCom/neon-utils/pull/556) ([NeonDaniel](https://github.com/NeonDaniel))
+
+## [1.13.1a3](https://github.com/NeonGeckoCom/neon-utils/tree/1.13.1a3) (2025-09-10)
+
+[Full Changelog](https://github.com/NeonGeckoCom/neon-utils/compare/1.13.1a2...1.13.1a3)
+
+**Implemented enhancements:**
+
+- \[FEAT\] Deprecate ovos-core dependency [\#469](https://github.com/NeonGeckoCom/neon-utils/issues/469)
+
+**Merged pull requests:**
+
+- Remove ovos-core test dependency [\#557](https://github.com/NeonGeckoCom/neon-utils/pull/557) ([NeonDaniel](https://github.com/NeonDaniel))
+
+## [1.13.1a2](https://github.com/NeonGeckoCom/neon-utils/tree/1.13.1a2) (2025-09-10)
+
+[Full Changelog](https://github.com/NeonGeckoCom/neon-utils/compare/1.13.1a1...1.13.1a2)
**Merged pull requests:**
-- Add helper to start a health check server [\#551](https://github.com/NeonGeckoCom/neon-utils/pull/551) ([NeonDaniel](https://github.com/NeonDaniel))
+- Improve Skill metadata parsing support and remove OSM dependency [\#558](https://github.com/NeonGeckoCom/neon-utils/pull/558) ([NeonDaniel](https://github.com/NeonDaniel))
-## [1.12.2a1](https://github.com/NeonGeckoCom/neon-utils/tree/1.12.2a1) (2025-05-30)
+## [1.13.1a1](https://github.com/NeonGeckoCom/neon-utils/tree/1.13.1a1) (2025-08-27)
-[Full Changelog](https://github.com/NeonGeckoCom/neon-utils/compare/1.12.1...1.12.2a1)
+[Full Changelog](https://github.com/NeonGeckoCom/neon-utils/compare/1.13.0...1.13.1a1)
**Merged pull requests:**
-- Make HANA SSL optional [\#550](https://github.com/NeonGeckoCom/neon-utils/pull/550) ([NeonDaniel](https://github.com/NeonDaniel))
+- Update Skill API to support outside applications [\#554](https://github.com/NeonGeckoCom/neon-utils/pull/554) ([NeonDaniel](https://github.com/NeonDaniel))
diff --git a/neon_utils/configuration_utils.py b/neon_utils/configuration_utils.py
index a55f64cd..d6d4834e 100644
--- a/neon_utils/configuration_utils.py
+++ b/neon_utils/configuration_utils.py
@@ -37,7 +37,7 @@
import yaml
from copy import deepcopy
-from os.path import *
+from os.path import join, isfile, isdir, exists, expanduser, dirname, basename, splitext
from collections.abc import MutableMapping
from contextlib import suppress
@@ -302,9 +302,9 @@ def _load_yaml_file(self) -> dict:
from ruamel.yaml import YAML
config = _make_loaded_config_safe(YAML().load(f))
except ImportError:
- LOG.error(f"ruamel.yaml not available to load "
- f"legacy config. "
- f"pip install neon-utils[configuration]")
+ LOG.error("ruamel.yaml not available to load "
+ "legacy config. "
+ "pip install neon-utils[configuration]")
if not config:
LOG.debug(f"Empty config file found at: {self.file_path}")
config = dict()
@@ -325,7 +325,7 @@ def _write_yaml_file(self) -> bool:
"""
to_write = deepcopy(self._content)
if not to_write:
- LOG.error(f"Config content empty! Skipping write to disk and reloading")
+ LOG.error("Config content empty! Skipping write to disk and reloading")
return False
if not path_is_read_writable(self.file_path):
LOG.warning(f"Insufficient write permissions: {self.file_path}")
@@ -347,7 +347,7 @@ def _write_yaml_file(self) -> bool:
self._disk_content_hash = hash(repr(self._content))
except Exception as e:
LOG.error(e)
- LOG.info(f"Restoring config from tmp file backup")
+ LOG.info("Restoring config from tmp file backup")
shutil.copy2(tmp_filename, self.file_path)
return True
@@ -810,7 +810,7 @@ def get_user_config_from_mycroft_conf(user_config: dict = None) -> dict:
["offset"] / 3600000, 1))
else:
- LOG.warning(f"No location in core configuration")
+ LOG.warning("No location in core configuration")
return user_config
@@ -941,6 +941,8 @@ def get_mycroft_compatible_location(location: dict) -> dict:
return location
+@deprecated("Configuration should be managed via ovos_config.Configuration",
+ "2.0.0")
def get_mycroft_compatible_config(mycroft_only=False,
neon_config_path=None) -> dict:
"""
@@ -1022,6 +1024,8 @@ def get_mycroft_compatible_config(mycroft_only=False,
return default_config
+@deprecated("Configuration should be managed via ovos_config.Configuration",
+ "2.0.0")
def write_mycroft_compatible_config(file_to_write: str = None) -> str:
"""
Generates a mycroft-like configuration and writes it to the specified file
@@ -1036,7 +1040,7 @@ def write_mycroft_compatible_config(file_to_write: str = None) -> str:
if isfile(file_path):
disk_config = load_commented_json(file_path)
if disk_config == configuration:
- LOG.debug(f"Configuration already up to date")
+ LOG.debug("Configuration already up to date")
return file_path
LOG.warning(f"File exists and will be overwritten: {file_to_write}")
elif not isdir(dirname(file_path)):
@@ -1149,7 +1153,7 @@ def parse_skill_default_settings(settings_meta: dict) -> dict:
LOG.error(settings_meta)
raise TypeError(f"Expected a dict, got: {type(settings_meta)}")
if not settings_meta:
- LOG.debug(f"Empty Settings")
+ LOG.debug("Empty Settings")
return dict()
else:
settings = dict()
@@ -1355,7 +1359,7 @@ def _get_neon_skills_config(neon_config_path=None) -> dict:
neon_skills["directory_override"] = neon_skills["directory"]
except ModuleNotFoundError:
LOG.warning("ovos-core not installed")
- neon_skills["directory_override"] = neon_skills["directory"]
+ # neon_skills["directory_override"] = neon_skills["directory"]
neon_skills["disable_osm"] = neon_skills.get("skill_manager", "osm") != "osm"
neon_skills["priority_skills"] = neon_skills.get("priority") or []
@@ -1381,45 +1385,13 @@ def _get_neon_skills_config(neon_config_path=None) -> dict:
neon_skills["neon_token"] = find_neon_git_token()
# populate_github_token_config(neon_skills["neon_token"])
except FileNotFoundError:
- LOG.debug(f"No Github token found; skills may fail to install")
+ LOG.debug("No Github token found; skills may fail to install")
neon_skills["neon_token"] = None
skills_config = {**mycroft_config.get("skills", {}), **neon_skills}
skills_config.setdefault("debug", False)
return skills_config
-@deprecated("Configuration moved to `ovos_config.Configuration`", "2.0.0")
-def _get_neon_transcribe_config(neon_config_path=None) -> dict:
- """
- Get a configuration dict for the transcription module.
- Returns:
- dict of config params used for the Neon transcription module
- """
- local_config = _get_neon_local_config(neon_config_path)
- user_config = get_neon_user_config(neon_config_path) if \
- isfile(join(neon_config_path or get_config_dir(),
- "ngi_user_info.yml")) else {}
- neon_transcribe_config = dict()
- neon_transcribe_config["transcript_dir"] = \
- local_config.get("dirVars", {}).get("docsDir", "")
- neon_transcribe_config["audio_permission"] = \
- user_config.get("privacy", {}).get("save_audio", False)
- return neon_transcribe_config
-
-
-@deprecated("Configuration moved to `ovos_config.Configuration`", "2.0.0")
-def _get_neon_gui_config(neon_config_path=None) -> dict:
- """
- Get a configuration dict for the gui module.
- Returns:
- dict of config params used for the Neon gui module
- """
- local_config = _get_neon_local_config(neon_config_path)
- gui_config = dict(local_config.get("gui", {}))
- gui_config["base_port"] = gui_config.get("port")
- return gui_config
-
-
@deprecated("Configuration moved to `ovos_config.Configuration`", "2.0.0")
def _safe_mycroft_config() -> dict:
"""
@@ -1432,31 +1404,6 @@ def _safe_mycroft_config() -> dict:
return dict(config)
-@deprecated("Configuration moved to `ovos_config.Configuration`", "2.0.0")
-def _get_neon_yaml_config() -> dict:
- from ovos_config.meta import get_ovos_config
- from ovos_config.locations import get_xdg_config_save_path
- from ovos_utils.json_helper import merge_dict
-
- with open(get_ovos_config()["default_config_path"]) as f:
- default = yaml.safe_load(f)
- config = dict(default)
- system_config = os.environ.get("MYCROFT_SYSTEM_CONFIG",
- "/etc/neon/neon.yaml")
- user_config = join(get_xdg_config_save_path("neon"), "neon.yaml")
- if isfile(system_config):
- with open(system_config) as f:
- system = yaml.safe_load(f)
- config = merge_dict(config, system)
-
- if isfile(user_config):
- with open(user_config) as f:
- user = yaml.safe_load(f)
- config = merge_dict(config, user)
-
- return config
-
-
@deprecated("Configuration moved to `ovos_config.Configuration`", "2.0.0")
def _get_neon_auth_config(path: Optional[str] = None) -> dict:
"""
diff --git a/neon_utils/file_utils.py b/neon_utils/file_utils.py
index e85500eb..a23eb181 100644
--- a/neon_utils/file_utils.py
+++ b/neon_utils/file_utils.py
@@ -267,7 +267,7 @@ def parse_skill_readme_file(readme_path: str) -> dict:
:param readme_path: absolute path to README.md file
:return: dict of parsed data
"""
- from neon_utils.parse_utils import clean_quotes
+ from neon_utils.skill_utils import _get_skill_data_readme
# Check passed path
if not readme_path:
@@ -275,102 +275,9 @@ def parse_skill_readme_file(readme_path: str) -> dict:
readme_path = os.path.expanduser(readme_path)
if not os.path.isfile(readme_path):
raise FileNotFoundError(f"{readme_path} is not a valid file")
-
- # Initialize parser params
- list_sections = ("examples", "incompatible skills", "platforms",
- "categories", "tags", "credits")
- section = "header"
- category = None
- parsed_data = {}
with open(readme_path, "r") as readme:
- lines = readme.readlines()
-
- def _check_section_start(ln: str):
- # Handle section start
- if ln.startswith("# ![](https://0000.us/klatchat/app/files/"
- "neon_images/icons/neon_paw.png)"):
- # Old style title line
- parsed_data["title"] = ln.split(')', 1)[1].strip()
- parsed_data["icon"] = ln.split('(', 1)[1].split(')', 1)[0].strip()
- return
- elif section == "header" and ln.startswith("# ![]():
- # Title line
- parsed_data[)
", 1)[1].strip()
- parsed_data["icon"] = ln.split("src=", 1)[1].split()[0]\
- .strip('"').strip("'").lstrip("./")
- return "summary"
- elif ln.startswith("# ") or ln.startswith("## "):
- # Top-level section
- if ln.startswith("## About"):
- # Handle Mycroft 'About' section
- return "description"
- elif ln.startswith("## Category"):
- # Handle 'Category' as 'Categories'
- return "categories"
- else:
- return line.lstrip("#").strip().lower()
- return
-
- def _format_readme_line(ln: str):
- nonlocal category
- if section == "incompatible skills":
- if not any((ln.startswith('-'), ln.startswith('*'))):
- return None
- parsed = clean_quotes(ln.lstrip('-').lstrip('*').lower().strip())
- if parsed.startswith('['):
- return parsed.split('(', 1)[1].split(')', 1)[0]
- return parsed
- if section == "examples":
- if not any((ln.startswith('-'), ln.startswith('*'))):
- return None
- parsed = clean_quotes(ln.lstrip('-').lstrip('*').strip())
- if parsed.split(maxsplit=1)[0].lower() == "neon":
- return parsed.split(maxsplit=1)[1]
- else:
- return parsed
- if section == "categories":
- parsed = ln.rstrip('\n').strip('*')
- if ln.startswith('**'):
- category = parsed
- return parsed
- if section == "credits":
- if ln.strip().startswith('['):
- return ln.split('[', 1)[1].split(']', 1)[0]
- return ln.rstrip('\n').lstrip('@')
- if section == "tags":
- return ln.lstrip('#').rstrip('\n')
- if section in list_sections:
- return clean_quotes(ln.lstrip('-').lstrip('*').lower().strip())
- return ln.rstrip('\n').rstrip()
-
- for line in lines:
- new_section = _check_section_start(line)
- if new_section:
- section = new_section
- elif line.strip():
- parsed_line = _format_readme_line(line)
- if not parsed_line:
- # Nothing to parse in this line
- continue
- if section in list_sections:
- if section not in parsed_data:
- parsed_data[section] = list()
- parsed_data[section].append(parsed_line)
- else:
- if section not in parsed_data:
- parsed_data[section] = parsed_line
- else:
- parsed_data[section] = " ".join((parsed_data[section],
- parsed_line))
- parsed_data["category"] = category or parsed_data.get("categories",
- [""])[0]
- if parsed_data.get("incompatible skills"):
- parsed_data["incompatible_skills"] = \
- parsed_data.pop("incompatible skills")
- if parsed_data.get("credits") and len(parsed_data["credits"]) == 1:
- parsed_data["credits"] = parsed_data["credits"][0].split(' ')
-
- return parsed_data
+ lines = readme.read()
+ return _get_skill_data_readme(lines)
def check_path_permissions(path: str) -> tuple:
diff --git a/neon_utils/hana_utils.py b/neon_utils/hana_utils.py
index 91caeb80..66c09b61 100644
--- a/neon_utils/hana_utils.py
+++ b/neon_utils/hana_utils.py
@@ -25,6 +25,8 @@
# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+from _socket import gethostname
+from datetime import datetime
import requests
import json
@@ -33,6 +35,7 @@
from os import makedirs, remove
from os.path import join, isfile, isdir, dirname
from time import time
+
from ovos_utils.log import LOG
from ovos_utils.xdg_utils import xdg_cache_home
@@ -92,21 +95,23 @@ def _init_client(backend_address: str, ssl_verify: bool = True):
_headers = {"Authorization": f"Bearer {_client_config['access_token']}"}
-def _get_token(backend_address: str, username: str = "guest",
- password: str = "password",
+def _get_token(backend_address: str,
ssl_verify: bool = True):
"""
Get new auth tokens from the specified server. This will cache the returned
token, overwriting any previous data at the cache path.
@param backend_address: Hana server URL to connect to
- @param username: Username to authorize
- @param password: Password for specified username
"""
+ from ovos_config.config import Configuration
global _client_config
- # TODO: username/password from configuration
+ hana_config = Configuration().get('hana', {})
+ username = hana_config.get("username") or "guest"
+ password = hana_config.get("password") or "password"
+ token_name = f"{gethostname()}_{datetime.utcnow().isoformat()}"
resp = requests.post(f"{backend_address}/auth/login",
json={"username": username,
- "password": password},
+ "password": password,
+ "token_name": token_name},
verify=ssl_verify)
if not resp.ok:
raise ServerException(f"Error logging into {backend_address}. "
diff --git a/neon_utils/log_aggregators/__init__.py b/neon_utils/log_aggregators/__init__.py
index 0196509f..7fe7a21d 100644
--- a/neon_utils/log_aggregators/__init__.py
+++ b/neon_utils/log_aggregators/__init__.py
@@ -28,6 +28,8 @@
import importlib
+from ovos_utils.log import LOG
+
_service_name_to_handler = {
'sentry': 'init_sentry'
}
@@ -37,10 +39,13 @@ def init_log_aggregators(config: dict = None):
from ovos_config.config import Configuration
config = config or Configuration()
for service_name, handler in _service_name_to_handler.items():
- service_config = _get_log_aggregator_config(config=config, name=service_name)
- if bool(service_config.pop('enabled', False)):
- service_module = importlib.import_module(f'.{service_name}', __name__)
- getattr(service_module, handler)(config=service_config)
+ try:
+ service_config = _get_log_aggregator_config(config=config, name=service_name)
+ if service_config.pop('enabled', False) is True:
+ service_module = importlib.import_module(f'.{service_name}', __name__)
+ getattr(service_module, handler)(config=service_config)
+ except ModuleNotFoundError as e:
+ LOG.exception(f"Failed to load log aggregator {service_name}: {e}")
def _get_log_aggregator_config(config: dict, name: str):
diff --git a/neon_utils/log_aggregators/sentry.py b/neon_utils/log_aggregators/sentry.py
index f57907a3..a3ad2e8d 100644
--- a/neon_utils/log_aggregators/sentry.py
+++ b/neon_utils/log_aggregators/sentry.py
@@ -26,12 +26,13 @@
# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-import sentry_sdk
SENTRY_SDK_REQUIRED_KEYS = {'dsn'}
def init_sentry(config: dict):
+ import sentry_sdk
+
missing_required_keys = SENTRY_SDK_REQUIRED_KEYS.difference(config.keys())
if missing_required_keys:
raise KeyError(f'Sentry SDK configuration missing required keys: {missing_required_keys}')
diff --git a/neon_utils/packaging_utils.py b/neon_utils/packaging_utils.py
index 884d830f..ae6381a9 100644
--- a/neon_utils/packaging_utils.py
+++ b/neon_utils/packaging_utils.py
@@ -28,14 +28,14 @@
import sys
import re
+import subprocess
import importlib.util
from typing import Tuple, Optional, List
from tempfile import mkstemp
-import pkg_resources
import sysconfig
-from os.path import exists, join, expanduser, isdir
+from os.path import exists, join, isfile
from ovos_utils.log import LOG, deprecated
@@ -62,13 +62,16 @@ def get_package_version_spec(pkg: str):
"""
Locate an installed package and return its reported version
:param pkg: string package name to locate
- :returns: Version string as reported by pkg_resources
- :raises: ModuleNotFoundError if requested package isn't installed
+ :returns: Version string as reported by importlib.metadata
+ :raises: PackageNotFoundError if requested package isn't installed
"""
try:
- return pkg_resources.get_distribution(pkg).version
- except pkg_resources.DistributionNotFound:
- raise ModuleNotFoundError(f"{pkg} not found")
+ from importlib.metadata import version
+ except ImportError:
+ # Fallback for Python < 3.8
+ from importlib_metadata import version
+
+ return version(pkg)
def get_package_dependencies(pkg: str):
@@ -76,15 +79,20 @@ def get_package_dependencies(pkg: str):
Get the dependencies for an installed package
:param pkg: string package name to evaluate
:returns: list of string dependencies (equivalent to requirements.txt)
- :raises ModuleNotFoundError if requested package isn't installed
+ :raises PackageNotFoundError if requested package isn't installed
"""
try:
- constraints = pkg_resources.working_set.by_key[pkg].requires()
- constraints_spec = [str(c).split('[', 1)[0] for c in constraints]
- LOG.debug(constraints_spec)
- return constraints_spec
- except KeyError:
- raise ModuleNotFoundError(f"{pkg} not found")
+ from importlib.metadata import requires
+ except ImportError:
+ # Fallback for Python < 3.8
+ from importlib_metadata import requires
+
+ requirements = requires(pkg)
+ if requirements is None:
+ return []
+ constraints_spec = [req.split('[', 1)[0].split(';', 1)[0] for req in requirements]
+ LOG.debug(constraints_spec)
+ return constraints_spec
@deprecated("Reference `neon_core.version.__version__` directly", "2.0.0")
@@ -180,18 +188,19 @@ def get_mycroft_core_root():
raise FileNotFoundError("Could not determine core directory")
+@deprecated("Use neon_utils.skill_utils.get_skill_metadata", "2.0.0")
def build_skill_spec(skill_dir: str) -> dict:
"""
Build dict contents of a skill.json file.
:param skill_dir: path to skill directory to parse
:returns: dict skill.json spec
"""
- import shutil
- from ovos_skills_manager.local_skill import get_skill_data_from_directory
- from neon_utils.file_utils import parse_skill_readme_file
- from neon_utils.configuration_utils import dict_merge
+ from neon_utils.skill_utils import get_skill_metadata
+
+ # Non-packaged skills are deprecated. Support is patched in here for
+ # Backwards-compatibility only
- def get_skill_license(): # TODO: Implement OSM version of this
+ def get_skill_license():
try:
with open(join(skill_dir, "LICENSE.md")) as f:
contents = f.read()
@@ -209,77 +218,44 @@ def get_skill_license(): # TODO: Implement OSM version of this
if "Neon AI Non-commercial Friendly License" in contents:
return "Neon 1.0"
- _invalid_skill_data_keys = ("appstore", "appstore_url", "credits",
- "skill_id")
- _invalid_readme_keys = ("contact support", "details")
- default_skill = {"title": "",
- "url": "",
- "summary": "",
- "short_description": "",
- "description": "",
- "examples": [],
- "desktopFile": False,
- "warning": "",
- "systemDeps": False,
- "requirements": {
- "python": [],
- "system": {},
- "skill": []
- },
- "incompatible_skills": [],
- "platforms": ["i386",
- "x86_64",
- "ia64",
- "arm64",
- "arm"],
- "branch": "master",
- "license": "",
- "icon": "",
- "category": "",
- "categories": [],
- "tags": [],
- "credits": [],
- "skillname": "",
- "authorname": "",
- "foldername": None}
-
- skill_dir = expanduser(skill_dir)
- if not isdir(skill_dir):
- raise FileNotFoundError(f"Not a Directory: {skill_dir}")
- LOG.debug(f"skill_dir={skill_dir}")
- skill_json = join(skill_dir, "skill.json")
- backup = join(skill_dir, "skill_json.bak")
- shutil.move(skill_json, backup)
- skill_data = get_skill_data_from_directory(skill_dir)
- shutil.move(backup, skill_json)
- skill_data['foldername'] = None
- for key in _invalid_skill_data_keys:
- if key in skill_data:
- skill_data.pop(key)
- readme_data = parse_skill_readme_file(join(skill_dir, "README.md"))
- for key in _invalid_readme_keys:
- if key in readme_data:
- readme_data.pop(key)
- readme_data["short_description"] = readme_data.get("summary")
- readme_data["license"] = get_skill_license()
- readme_data["branch"] = "master"
- skill_data = dict_merge(default_skill, skill_data)
- skill_data["requirements"]["python"].sort()
- return dict(dict_merge(skill_data, readme_data))
-
-def install_packages_from_pip(core_module: str, packages: List[str]) -> int:
+ skill_meta = get_skill_metadata(skill_dir)
+ if skill_meta.get("license") is None:
+ skill_meta["license"] = get_skill_license()
+
+ if skill_meta["requirements"].get("python") is None and \
+ isfile(join(skill_dir, "requirements.txt")):
+ try:
+ with open(join(skill_dir, "requirements.txt")) as f:
+ requirements = f.read().split('\n')
+ requirements = [r for r in requirements
+ if r and not r.startswith('#')]
+ skill_meta["requirements"]["python"] = requirements
+ except Exception as e:
+ LOG.error(e)
+ skill_meta["requirements"]["python"] = []
+
+ return skill_meta
+
+def install_packages_from_pip(core_module: str, packages: List[str],
+ force_reinstall: bool = False) -> int:
"""
Install a Python package using pip
:param core_module: string neon core module to install dependency for
:param packages: List(string) list of packages to install
+ :param force_reinstall: force re-installation of packages
:returns: int pip exit code
"""
- import pip
+ def _pip_install(command_args: List[str]) -> int:
+ try:
+ result = subprocess.check_call([sys.executable, '-m', 'pip'] + command_args)
+ return result
+ except subprocess.CalledProcessError as e:
+ LOG.error(f"Error installing {command_args}: {e}")
+ return e.returncode
+
_, tmp_constraints_file = mkstemp()
_, tmp_requirements_file = mkstemp()
- install_str = " ".join(packages)
-
with open(tmp_constraints_file, 'w', encoding="utf8") as f:
constraints = '\n'.join(get_package_dependencies(core_module))
f.write(constraints)
@@ -289,10 +265,38 @@ def install_packages_from_pip(core_module: str, packages: List[str]) -> int:
for pkg in packages:
f.write(f"{pkg}\n")
- LOG.info(f"Requested installation of plugins: {install_str}")
+ LOG.info(f"Requested installation of plugins: {packages}")
pip_args = ['install', '-r', tmp_requirements_file, '-c', tmp_constraints_file]
- result = pip.main(pip_args) if hasattr(pip, 'main') else pip._internal.main(pip_args)
-
- if result != 0:
- return result
- return 0
+ if stat := _pip_install(pip_args) != 0:
+ return stat
+
+ if force_reinstall:
+ LOG.info(f"Requested forced re-installation of plugins: {packages}")
+ pip_args.extend(['--no-deps', '--force-reinstall'])
+ stat = _pip_install(pip_args)
+
+ return stat
+
+
+def get_installed_prereleases() -> List[Tuple[str, str]]:
+ """
+ Get a list of installed pre-release packages.
+ @return: List of tuple (pkg_name, version)
+ """
+ from subprocess import run
+ packages = run(["pip", "list"],
+ capture_output=True).stdout.decode("utf-8")
+ prerelease_pkgs = list()
+ for line in packages.split('\n'):
+ if not line:
+ continue
+ name, version = line.split()
+ if name == "Package" or not name.replace('-', ''):
+ continue
+ if not version.replace('.', '').isnumeric():
+ if "post" in version:
+ LOG.debug(f"post release {name}:{version}")
+ continue
+ prerelease_pkgs.append((name, version))
+ return prerelease_pkgs
+
diff --git a/neon_utils/skill_utils.py b/neon_utils/skill_utils.py
new file mode 100644
index 00000000..0f7c5be3
--- /dev/null
+++ b/neon_utils/skill_utils.py
@@ -0,0 +1,365 @@
+# NEON AI (TM) SOFTWARE, Software Development Kit & Application Framework
+# All trademark and other rights reserved by their respective owners
+# Copyright 2008-2025 Neongecko.com Inc.
+# Contributors: Daniel McKnight, Guy Daniels, Elon Gasper, Richard Leeds,
+# Regina Bloomstine, Casimiro Ferreira, Andrii Pernatii, Kirill Hrymailo
+# BSD-3 License
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 1. Redistributions of source code must retain the above copyright notice,
+# this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright notice,
+# this list of conditions and the following disclaimer in the documentation
+# and/or other materials provided with the distribution.
+# 3. Neither the name of the copyright holder nor the names of its
+# contributors may be used to endorse or promote products derived from this
+# software without specific prior written permission.
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA,
+# OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+from os import getcwd, chdir
+from os.path import isfile, dirname, abspath, join
+from mock import Mock, patch
+from importlib.util import spec_from_file_location, module_from_spec
+
+from ovos_utils.log import LOG
+
+
+def get_skill_metadata(skill_dir: str) -> dict:
+ """
+ Get a metadata object for a skill in a given directory.
+ """
+ setup_py = join(skill_dir, "setup.py")
+ pyproject_toml = join(skill_dir, "pyproject.toml")
+ readme_md = join(skill_dir, "README.md")
+
+ if isfile(pyproject_toml):
+ meta = _get_skill_data_poetry(pyproject_toml)
+ elif isfile(setup_py):
+ meta = _get_skill_data_setuptools(setup_py)
+ elif isfile(readme_md):
+ with open(readme_md, encoding="utf-8") as f:
+ readme_data = f.read()
+ meta = _get_skill_data_readme(readme_data)
+ else:
+ raise FileNotFoundError(
+ f"No setup.py or pyproject.toml found in {skill_dir}"
+ )
+
+ # Adding params for backwards-compat.
+ meta["desktopFile"] = False
+ meta["warning"] = ""
+ meta["systemDeps"] = False
+ meta.setdefault("requirements", {})
+ meta["requirements"].setdefault("system", {})
+ meta["requirements"].setdefault("skill", [])
+ meta.setdefault("incompatible_skills", [])
+ meta["platforms"] = ["i386", "x86_64", "ia64", "arm64", "arm"]
+ meta["branch"] = "master"
+ meta["foldername"] = None
+ meta.setdefault("short_description", meta.get("summary"))
+
+ return meta
+
+
+def _get_skill_data_poetry(pyproject: str) -> dict:
+ """
+ Get skill metadata from a pyproject.toml file
+ Args:
+ pyproject: pyproject.toml file to evaluate
+ Returns:
+ dict: Skill metadata
+ """
+ from toml import load
+
+ if not isfile(pyproject):
+ raise FileNotFoundError(f"Not a Directory: {pyproject}")
+ with open(pyproject, encoding="utf-8") as f:
+ data = load(f)
+ skill_data = {
+ "package_name": data["tool"]["poetry"].get("name", "Unknown"),
+ "name": data["tool"]["poetry"].get("name", "Unknown"),
+ "description": data["tool"]["poetry"].get("name", "description"),
+ "pip_spec": data["tool"]["poetry"].get("name", "Unknown"),
+ "license": data["tool"]["poetry"].get("license", "Unknown"),
+ "author": data["tool"]["poetry"].get(
+ "authors", [""]
+ ), # List of authors
+ "tags": data["tool"]["poetry"].get("keywords", []),
+ "version": data["tool"]["poetry"].get("version", ""),
+ }
+ print(data["tool"]["poetry"]["plugins"])
+ skill_data["skill_id"] = list(
+ data["tool"]["poetry"]
+ .get("plugins", {})
+ .get("ovos.plugin.skill", {})
+ .keys()
+ )[0].strip('"')
+
+ # Below match existing Neon skill metadata
+ skill_data["title"] = skill_data["name"]
+ skill_data["summary"] = skill_data["description"]
+ skill_data["short_description"] = skill_data["description"]
+ skill_data["credits"] = [skill_data["author"]]
+ skill_data["skillname"] = skill_data["name"]
+
+ python_requirements = data["tool"]["poetry"].get("dependencies", {})
+ # Parse requirements into expected string format
+ python_requirements.pop("python", None) # Remove python version spec
+ skill_data["requirements"] = {
+ "python": [f"{k}{v}" for k, v in python_requirements.items()]
+ }
+
+ if data["tool"]["poetry"].get("readme"):
+ readme_path = join(
+ dirname(pyproject), data["tool"]["poetry"]["readme"]
+ )
+ with open(readme_path) as f:
+ readme_data = f.read()
+ try:
+ readme_metadata = _get_skill_data_readme(readme_data)
+ except Exception as e:
+ LOG.error(f"Failed to parse README for {skill_data['name']}: {e}")
+ readme_metadata = {}
+ return {**readme_metadata, **skill_data}
+
+
+def _get_skill_data_setuptools(setup_py: str) -> dict:
+ """
+ Get skill metadata from a setup.py file by executing it with a mocked setuptools.setup
+ Args:
+ setup_py: setup.py file to evaluate
+ Returns:
+ dict: Skill metadata
+ """
+
+ if not isfile(setup_py):
+ raise FileNotFoundError(f"File not found: {setup_py}")
+
+ # Create a mock for setuptools.setup
+ setup_mock = Mock()
+
+ # Mock setuptools.find_packages as well
+ find_packages_mock = Mock(return_value=[])
+
+ # Change to the directory containing setup.py to handle relative paths
+ original_cwd = getcwd()
+ setup_dir = dirname(abspath(setup_py))
+
+ try:
+ chdir(setup_dir)
+
+ # Mock setuptools imports and execute setup.py
+ with patch.dict(
+ "sys.modules",
+ {
+ "setuptools": Mock(
+ setup=setup_mock, find_packages=find_packages_mock
+ )
+ },
+ ):
+ # Load and execute the setup.py file
+ spec = spec_from_file_location("setup", setup_py)
+ setup_module = module_from_spec(spec)
+ spec.loader.exec_module(setup_module)
+
+ # Get the captured kwargs from the mock
+ if not setup_mock.called:
+ raise ValueError(f"No setup() call found in {setup_py}")
+
+ captured_kwargs = (
+ setup_mock.call_args.kwargs
+ if setup_mock.call_args.kwargs
+ else setup_mock.call_args.args[0]
+ if setup_mock.call_args.args
+ else {}
+ )
+
+ finally:
+ chdir(original_cwd)
+
+ # Build skill_data matching the expected format
+ skill_data = {
+ "package_name": captured_kwargs.get("name"),
+ "pip_spec": captured_kwargs.get("name"),
+ "license": captured_kwargs.get("license"),
+ "author": captured_kwargs.get("author"),
+ "tags": captured_kwargs.get("keywords", []),
+ "version": captured_kwargs.get("version"),
+ "url": captured_kwargs.get("url"),
+ }
+
+ # Extract skill_id from entry_points if available
+ skill_data["skill_id"] = captured_kwargs["entry_points"][
+ "ovos.plugin.skill"
+ ]
+
+ if "github.com" in skill_data["url"]:
+ author, skill = skill_data["url"].split("github.com/")[1].split("/")
+ skill_data["skillname"] = skill
+ skill_data["authorname"] = author
+
+ # Set additional fields to match existing Neon skill metadata format
+ skill_data["name"] = skill_data["package_name"]
+ skill_data["title"] = skill_data["package_name"]
+ skill_data["credits"] = (
+ [skill_data["author"]] if skill_data["author"] != "Unknown" else []
+ )
+
+ # Handle requirements
+ install_requires = captured_kwargs.get("install_requires", [])
+ skill_data["requirements"] = {
+ "python": install_requires
+ if isinstance(install_requires, list)
+ else []
+ }
+
+ if "long_description" in captured_kwargs:
+ readme_data = _get_skill_data_readme(
+ captured_kwargs["long_description"]
+ )
+ else:
+ readme_data = {}
+ return {**readme_data, **skill_data}
+
+
+def _get_skill_data_readme(readme_md: str) -> dict:
+ """
+ Get skill metadata from a README file
+ Args:
+ readme_md: README.md file contents to evaluate
+ Returns:
+ dict: Skill metadata
+ """
+ from neon_utils.parse_utils import clean_quotes
+
+ lines = readme_md.split("\n")
+ if not lines or len(lines) <= 1:
+ raise ValueError("Empty README data")
+
+ # Initialize parser params
+ list_sections = (
+ "examples",
+ "incompatible skills",
+ "platforms",
+ "categories",
+ "tags",
+ "credits",
+ )
+ valid_sections = list_sections + (
+ "summary",
+ "short_description",
+ "description",
+ "warning",
+ )
+ section = "header"
+ category = None
+ parsed_data = {}
+
+ def _check_section_start(ln: str):
+ # Handle section start
+ if ln.startswith(
+ "# ![](https://0000.us/klatchat/app/files/"
+ "neon_images/icons/neon_paw.png)"
+ ):
+ # Old style title line
+ parsed_data["title"] = ln.split(")", 1)[1].strip()
+ parsed_data["icon"] = ln.split("(", 1)[1].split(")", 1)[0].strip()
+ return
+ elif section == "header" and ln.startswith("# ![]():
+ # Title line
+ parsed_data[)
", 1)[1].strip()
+ parsed_data["icon"] = (
+ ln.split("src=", 1)[1]
+ .split()[0]
+ .strip('"')
+ .strip("'")
+ .lstrip("./")
+ )
+ return "summary"
+ elif ln.startswith("# ") or ln.startswith("## "):
+ # Top-level section
+ if ln.startswith("## About"):
+ # Handle Mycroft 'About' section
+ return "description"
+ elif ln.startswith("## Category"):
+ # Handle 'Category' as 'Categories'
+ return "categories"
+ else:
+ return line.lstrip("#").strip().lower()
+ return
+
+ def _format_readme_line(ln: str):
+ nonlocal category
+ if section == "incompatible skills":
+ if not any((ln.startswith("-"), ln.startswith("*"))):
+ return None
+ parsed = clean_quotes(ln.lstrip("-").lstrip("*").lower().strip())
+ if parsed.startswith("["):
+ return parsed.split("(", 1)[1].split(")", 1)[0]
+ return parsed
+ if section == "examples":
+ if not any((ln.startswith("-"), ln.startswith("*"))):
+ return None
+ parsed = clean_quotes(ln.lstrip("-").lstrip("*").strip())
+ if parsed.split(maxsplit=1)[0].lower() == "neon":
+ return parsed.split(maxsplit=1)[1]
+ else:
+ return parsed
+ if section == "categories":
+ parsed = ln.rstrip("\n").strip("*")
+ if ln.startswith("**"):
+ category = parsed
+ return parsed
+ if section == "credits":
+ if ln.strip().startswith("["):
+ return ln.split("[", 1)[1].split("]", 1)[0]
+ return ln.rstrip("\n").lstrip("@")
+ if section == "tags":
+ return ln.lstrip("#").rstrip("\n")
+ if section in list_sections:
+ return clean_quotes(ln.lstrip("-").lstrip("*").lower().strip())
+ return ln.rstrip("\n").rstrip()
+
+ for line in lines:
+ new_section = _check_section_start(line)
+ if new_section:
+ section = new_section
+ elif line.strip():
+ parsed_line = _format_readme_line(line)
+ if not parsed_line:
+ # Nothing to parse in this line
+ continue
+ if section in list_sections:
+ if section not in parsed_data:
+ parsed_data[section] = list()
+ parsed_data[section].append(parsed_line)
+ else:
+ if section not in valid_sections:
+ continue
+ elif section not in parsed_data:
+ parsed_data[section] = parsed_line
+ else:
+ parsed_data[section] = " ".join(
+ (parsed_data[section], parsed_line)
+ )
+ parsed_data["category"] = (
+ category or parsed_data.get("categories", [""])[0]
+ )
+ if parsed_data.get("incompatible skills"):
+ parsed_data["incompatible_skills"] = parsed_data.pop(
+ "incompatible skills"
+ )
+ if parsed_data.get("credits") and len(parsed_data["credits"]) == 1:
+ parsed_data["credits"] = parsed_data["credits"][0].split(" ")
+
+ return parsed_data
diff --git a/neon_utils/skills/neon_fallback_skill.py b/neon_utils/skills/neon_fallback_skill.py
index a3098604..0428c775 100644
--- a/neon_utils/skills/neon_fallback_skill.py
+++ b/neon_utils/skills/neon_fallback_skill.py
@@ -40,7 +40,6 @@
from dateutil.tz import gettz
from json_database import JsonStorage
from ovos_bus_client import Message
-from ovos_plugin_manager.language import OVOSLangDetectionFactory, OVOSLangTranslationFactory
from ovos_utils.gui import is_gui_connected
from ovos_utils.log import LOG, log_deprecation, deprecated
from ovos_utils.skills import get_non_properties
@@ -56,6 +55,12 @@
save_settings
from neon_utils.user_utils import get_user_prefs
+try:
+ from ovos_plugin_manager.language import OVOSLangDetectionFactory, OVOSLangTranslationFactory
+except ImportError:
+ OVOSLangDetectionFactory = None
+ OVOSLangTranslationFactory = None
+
class NeonFallbackSkill(FallbackSkillV1):
"""
diff --git a/neon_utils/skills/neon_skill.py b/neon_utils/skills/neon_skill.py
index 8197d486..b9ce3c66 100644
--- a/neon_utils/skills/neon_skill.py
+++ b/neon_utils/skills/neon_skill.py
@@ -227,6 +227,102 @@ def update_profile(self, new_preferences: dict, message: Message = None):
except Exception as x:
LOG.error(x)
+ # Duplicated from OVOSSkill for backwards-compat with skills using ovos-workshop 0.X
+ def _register_public_api(self):
+ """
+ Find and register API methods decorated with `@api_method` and create a
+ messagebus handler for fetching the api info if any handlers exist.
+ """
+
+ def wrap_method(fn, arg_model=None):
+ """Boilerplate for returning the response to the sender."""
+
+ def wrapper(message):
+ result = None
+ error = None
+ try:
+ if arg_model:
+ result = fn(arg_model(*message.data['args'],
+ **message.data['kwargs']))
+ else:
+ result = fn(*message.data.get('args', []),
+ **message.data.get('kwargs', {}))
+ try:
+ result = result.model_dump()
+ except AttributeError:
+ # Response is not a Pydantic model
+ pass
+ except Exception as e:
+ error = str(e)
+ message.context["skill_id"] = self.skill_id
+ self.bus.emit(message.response(data={'result': result,
+ 'error': error}))
+ return wrapper
+
+ from ovos_utils.skills import get_non_properties
+ methods = [attr_name for attr_name in get_non_properties(self)
+ if hasattr(getattr(self, attr_name), '__name__')]
+
+ for attr_name in methods:
+ method = getattr(self, attr_name)
+
+ if hasattr(method, 'api_method'):
+ doc = method.__doc__ or ''
+ name = method.__name__
+
+ # Extract method signature and return type
+ import inspect
+ signature = inspect.signature(method)
+ schema = None
+ return_schema = None
+ request_class = None
+ try:
+ from pydantic import BaseModel
+ parameters = signature.parameters
+
+ for arg_name, param in parameters.items():
+ if arg_name == 'self':
+ continue
+ if issubclass(param.annotation, BaseModel):
+ # Get the JSON schema for the BaseModel
+ schema = param.annotation.model_json_schema()
+ request_class = param.annotation
+ break
+ if signature.return_annotation and issubclass(signature.return_annotation, BaseModel):
+ # Get the JSON schema for the return type
+ return_schema = signature.return_annotation.model_json_schema()
+ except ImportError:
+ # If pydantic is not installed, there is no schema to extract
+ pass
+
+ self.public_api[name] = {
+ 'help': doc,
+ 'type': f'{self.skill_id}.{name}',
+ 'func': method,
+ 'signature': str(signature),
+ 'request_schema': schema,
+ 'response_schema': return_schema,
+ 'request_class': request_class
+ }
+ for key in self.public_api:
+ if ('type' in self.public_api[key] and
+ 'func' in self.public_api[key]):
+ self.log.debug(f"Adding api method: "
+ f"{self.public_api[key]['type']}")
+
+ # remove the function member since it shouldn't be
+ # reused and can't be sent over the messagebus
+ func = self.public_api[key].pop('func')
+ req_class = self.public_api[key].pop('request_class', None)
+ self.add_event(self.public_api[key]['type'],
+ wrap_method(func, req_class), speak_errors=False)
+
+ if self.public_api:
+ # TODO: Think about always registering this, so queries get an
+ # empty response, rather than waiting for a timeout
+ self.add_event(f'{self.skill_id}.public_api',
+ self._send_public_api, speak_errors=False)
+
@resolve_message
def update_skill_settings(self, new_preferences: dict,
message: Message = None, skill_global=True):
diff --git a/requirements/skills.txt b/requirements/skills.txt
new file mode 100644
index 00000000..d2569f96
--- /dev/null
+++ b/requirements/skills.txt
@@ -0,0 +1,2 @@
+mock
+toml~=0.10
diff --git a/requirements/test_requirements.txt b/requirements/test_requirements.txt
index 5197c41e..acf0f590 100644
--- a/requirements/test_requirements.txt
+++ b/requirements/test_requirements.txt
@@ -2,8 +2,5 @@ pytest
pytest-timeout
pytest-cov
mock
-# TODO: Deprecate ovos-core test dependency
-ovos-core~=0.0.7,>=0.0.8a21
-ovos-plugin-manager~=0.0.18
-ovos-skills-manager
-neon-lang-plugin-libretranslate
\ No newline at end of file
+ovos-plugin-manager~=0.1
+neon-lang-plugin-libretranslate
diff --git a/setup.py b/setup.py
index 0235c578..10c9e0dd 100644
--- a/setup.py
+++ b/setup.py
@@ -87,7 +87,8 @@ def get_requirements(requirements_filename: str):
"network": get_requirements("network.txt"),
"configuration": get_requirements("configuration.txt"),
"sentry": get_requirements("sentry.txt"),
- "signal": get_requirements("signal.txt")
+ "signal": get_requirements("signal.txt"),
+ "skills": get_requirements("skills.txt"),
},
entry_points={
'console_scripts': [
diff --git a/tests/configuration_util_tests.py b/tests/configuration_util_tests.py
index 73d1b0e7..9a800405 100644
--- a/tests/configuration_util_tests.py
+++ b/tests/configuration_util_tests.py
@@ -528,7 +528,7 @@ def test_get_mycroft_compat_config(self):
self.assertIsInstance(mycroft_config["gui_websocket"]["host"], str)
self.assertIsInstance(mycroft_config["gui_websocket"]["base_port"],
int)
- self.assertIsInstance(mycroft_config["ready_settings"], list)
+ # self.assertIsInstance(mycroft_config["ready_settings"], list)
self.assertIsInstance(mycroft_config['tts'], dict)
self.assertIsInstance(mycroft_config["keys"], dict)
# self.assertEqual(mycroft_config["skills"]["directory"],
@@ -1017,30 +1017,6 @@ def test_migrate_ngi_config(self):
os.remove(new_conf)
- @mock.patch('neon_utils.packaging_utils.get_neon_core_root')
- def test_get_neon_yaml_config(self, get_core_root):
- config_dir = join(dirname(__file__), "configuration",
- "get_neon_yaml_config")
- get_core_root.return_value = join(config_dir, "default")
- os.environ["XDG_CONFIG_HOME"] = config_dir
- os.environ["MYCROFT_SYSTEM_CONFIG"] = join(config_dir, "system",
- "neon.yaml")
- from neon_utils.configuration_utils import _get_neon_yaml_config, \
- init_config_dir
- init_config_dir()
- config = _get_neon_yaml_config()
- self.assertEqual(config,
- {"config": "user",
- "from_default": True,
- "from_system": True,
- "user": {
- "from_system": True,
- "from_default": True,
- "from_user": True,
- "not_from_user": False
- }})
- shutil.rmtree(join(config_dir, os.environ["OVOS_CONFIG_BASE_FOLDER"]))
-
class DeprecatedConfigTests(unittest.TestCase):
def doCleanups(self) -> None:
@@ -1065,21 +1041,6 @@ def test_get_audio_config(self):
self.assertIsInstance(config["tts"], dict)
self.assertIsInstance(config["language"], dict)
- def test_get_gui_config(self):
- from neon_utils.configuration_utils import _get_neon_gui_config
- config = _get_neon_gui_config()
- self.assertIsInstance(config, dict)
- # self.assertIsInstance(config["lang"], str)
- # self.assertIsInstance(config["enclosure"], str)
- # self.assertIsInstance(config["host"], str)
- # self.assertIsInstance(config["port"], int)
- # self.assertIsInstance(config["base_port"], int)
- # self.assertIsInstance(config["route"], str)
- # self.assertIsInstance(config["ssl"], bool)
- # self.assertIsInstance(config["resource_root"], str)
- # self.assertIn("file_server", config.keys())
- # self.assertEqual(config["port"], config["base_port"])
-
def test_get_lang_config(self):
from neon_utils.configuration_utils import _get_neon_lang_config
config = _get_neon_lang_config()
@@ -1091,13 +1052,6 @@ def test_get_lang_config(self):
# self.assertIn("boost", config)
# self.assertIsInstance(config["libretranslate"], dict)
- def test_get_transcribe_config(self):
- from neon_utils.configuration_utils import _get_neon_transcribe_config
- config = _get_neon_transcribe_config()
- self.assertIsInstance(config, dict)
- self.assertIsInstance(config["audio_permission"], bool)
- self.assertIsInstance(config["transcript_dir"], str)
-
def test_get_tts_config(self):
from neon_utils.configuration_utils import _get_neon_tts_config
config = _get_neon_tts_config()
diff --git a/tests/hana_util_tests.py b/tests/hana_util_tests.py
index 3ac38989..27ece7af 100644
--- a/tests/hana_util_tests.py
+++ b/tests/hana_util_tests.py
@@ -121,18 +121,26 @@ def test_request_backend_refresh_token(self, refresh_token, config_path):
neon_utils.hana_utils._client_config = real_client_config
@patch("neon_utils.hana_utils._get_client_config_path")
- def test_00_get_token(self, config_path):
+ @patch("ovos_config.config.Configuration")
+ def test_00_get_token(self, config, config_path):
+ config.return_value = {}
config_path.return_value = self.test_path
from neon_utils.hana_utils import _get_token
- # Test valid request
+ # Test valid default request
_get_token(self.test_server)
from neon_utils.hana_utils import _client_config
self.assertTrue(isfile(self.test_path))
with open(self.test_path) as f:
credentials_on_disk = json.load(f)
self.assertEqual(credentials_on_disk, _client_config)
- # TODO: Test invalid request, rate-limited request
+
+ # Test with configured invalid login
+ config.return_value = {"hana": {"username": "guest",
+ "password": "fake_password"}}
+ from neon_utils.hana_utils import ServerException
+ with self.assertRaises(ServerException):
+ _get_token(self.test_server)
@patch("neon_utils.hana_utils._get_client_config_path")
@patch("neon_utils.hana_utils._get_token")
@@ -170,8 +178,8 @@ def _write_token(*_, **__):
new_credentials['client_id'])
self.assertEqual(credentials_on_disk['username'],
new_credentials['username'])
- self.assertGreater(new_credentials['expiration'],
- credentials_on_disk['expiration'])
+ self.assertGreaterEqual(new_credentials['expiration'],
+ credentials_on_disk['expiration'])
self.assertNotEqual(credentials_on_disk['access_token'],
new_credentials['access_token'])
self.assertNotEqual(credentials_on_disk['refresh_token'],
diff --git a/tests/packaging_util_tests.py b/tests/packaging_util_tests.py
index ffbdbf08..10fc7607 100644
--- a/tests/packaging_util_tests.py
+++ b/tests/packaging_util_tests.py
@@ -114,13 +114,18 @@ def test_get_packaged_core_version(self):
def test_get_package_dependencies(self):
self_deps = get_package_dependencies("neon-utils")
- requirements_file = join(os.path.dirname(os.path.dirname(__file__)),
- "requirements", "requirements.txt")
- with open(requirements_file) as f:
- spec_requirements = f.read().split('\n')
+ requirements_dir = os.path.join(os.path.dirname(os.path.dirname(__file__)), "requirements")
+ spec_requirements = []
+ for _, _, files in os.walk(requirements_dir):
+ for file in files:
+ with open(os.path.join(requirements_dir, file)) as f:
+ spec_requirements += f.read().split('\n')
+
spec_requirements = [r for r in spec_requirements
if r and not r.startswith('#')]
+
# Version specs aren't order-dependent, so they can't be compared
+ # Also, constraints are normalized so elements cannot be compared
self.assertEqual(len(self_deps), len(spec_requirements))
with self.assertRaises(ModuleNotFoundError):
get_package_dependencies("fakeneongeckopackage")
@@ -143,31 +148,47 @@ def test_build_skill_spec(self):
skill_spec["url"] = "https://github.com/NeonGeckoCom/skill-alerts"
with open(join(test_dir, "skill.json")) as f:
valid_spec = json.load(f)
- self.assertEqual(valid_spec, skill_spec)
+ self.assertEqual(valid_spec, skill_spec, f"actual={skill_spec}")
# TODO: Actually validate exception cases? DM
def test_install_packages_from_pip(self):
- import pip
+ import subprocess
from neon_utils.packaging_utils import install_packages_from_pip
- with patch.object(pip, 'main', return_value=0) as mock_method:
+ with patch.object(subprocess, 'check_call', return_value=0) as mock_method:
test_result = install_packages_from_pip("neon-utils", ["pip-install-test"])
args, kwargs = mock_method.call_args
self.assertEqual(0, test_result)
- mock_method.assert_called_once_with(['install', '-r', args[0][2], '-c', args[0][4]])
+ mock_method.assert_called_once_with([sys.executable, '-m', 'pip', 'install', '-r', args[0][5], '-c', args[0][7]])
- with open(args[0][2], "r", encoding="utf8") as f:
+ with open(args[0][5], "r", encoding="utf8") as f:
line = f.readline()
self.assertEqual("pip-install-test\n", line)
mock_method.reset_mock()
- test_result = install_packages_from_pip("neon-utils", ["pip-install-test", "pip-install-another"])
+ test_result = install_packages_from_pip("neon-utils", ["pip-install-test", "pip-install-another"], force_reinstall=True)
self.assertEqual(0, test_result)
- mock_method.assert_called_once()
-
+ self.assertEqual(mock_method.call_count, 2)
+
+ @patch("subprocess.run")
+ def test_get_installed_prereleases(self, run):
+ run.return_value.stdout = """Package Version
+----------------------- ------------
+stable_package 1.0.0
+beta_package 0.2.2b3
+alpha_package 0.0.0a0
+date_package 24.4.30
+post_package 2.0.0post10
+""".encode("utf-8")
+ from neon_utils.packaging_utils import get_installed_prereleases
+ prereleases = get_installed_prereleases()
+ self.assertEqual(len(prereleases), 2)
+ for pkg in prereleases:
+ self.assertTrue(pkg[0].endswith("_package"))
+ self.assertEqual(len(pkg[1].split('.')), 3)
if __name__ == '__main__':
diff --git a/tests/skill_util_tests.py b/tests/skill_util_tests.py
new file mode 100644
index 00000000..2fdde461
--- /dev/null
+++ b/tests/skill_util_tests.py
@@ -0,0 +1,240 @@
+# NEON AI (TM) SOFTWARE, Software Development Kit & Application Framework
+# All trademark and other rights reserved by their respective owners
+# Copyright 2008-2025 Neongecko.com Inc.
+# Contributors: Daniel McKnight, Guy Daniels, Elon Gasper, Richard Leeds,
+# Regina Bloomstine, Casimiro Ferreira, Andrii Pernatii, Kirill Hrymailo
+# BSD-3 License
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 1. Redistributions of source code must retain the above copyright notice,
+# this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright notice,
+# this list of conditions and the following disclaimer in the documentation
+# and/or other materials provided with the distribution.
+# 3. Neither the name of the copyright holder nor the names of its
+# contributors may be used to endorse or promote products derived from this
+# software without specific prior written permission.
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA,
+# OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+import os
+import unittest
+import tempfile
+import shutil
+import subprocess
+
+from os.path import join, dirname
+from mock import patch
+
+
+class SkillUtilTests(unittest.TestCase):
+ @classmethod
+ def setUpClass(cls):
+ """Set up test fixtures, downloading the test skill repository"""
+ cls.test_dir = tempfile.mkdtemp(prefix="skill_util_tests_")
+
+ # Set up setuptools test repository
+ cls.setuptools_repo_url = (
+ "https://github.com/NeonGeckoCom/skill-caffeinewiz.git"
+ )
+ cls.setuptools_skill_branch = "0.3.1"
+ cls.setuptools_skill_repo_path = os.path.join(
+ cls.test_dir, "skill-caffeinewiz"
+ )
+ subprocess.run(
+ [
+ "git",
+ "clone",
+ "--branch",
+ cls.setuptools_skill_branch,
+ "--depth",
+ "1",
+ cls.setuptools_repo_url,
+ cls.setuptools_skill_repo_path,
+ ],
+ check=True,
+ capture_output=True,
+ text=True,
+ timeout=30,
+ )
+ cls.setup_py_path = os.path.join(
+ cls.setuptools_skill_repo_path, "setup.py"
+ )
+
+ # Set up poetry test repository
+ cls.poetry_repo_url = (
+ "https://github.com/OscillateLabsLLC/skill-homeassistant.git"
+ )
+ cls.poetry_skill_branch = "v0.5.1"
+ cls.poetry_skill_repo_path = os.path.join(
+ cls.test_dir, "skill-homeassistant"
+ )
+ subprocess.run(
+ [
+ "git",
+ "clone",
+ "--branch",
+ cls.poetry_skill_branch,
+ "--depth",
+ "1",
+ cls.poetry_repo_url,
+ cls.poetry_skill_repo_path,
+ ],
+ check=True,
+ capture_output=True,
+ text=True,
+ timeout=30,
+ )
+ cls.pyproject_toml_path = os.path.join(
+ cls.poetry_skill_repo_path, "pyproject.toml"
+ )
+
+ @classmethod
+ def tearDownClass(cls):
+ """Clean up test fixtures"""
+ if hasattr(cls, "test_dir") and os.path.exists(cls.test_dir):
+ shutil.rmtree(cls.test_dir)
+
+ @patch("neon_utils.skill_utils._get_skill_data_poetry")
+ @patch("neon_utils.skill_utils._get_skill_data_setuptools")
+ def test_get_skill_metadata(self, mock_setuptools, mock_poetry):
+ """Test that get_skill_metadata calls appropriate internal methods based on file existence"""
+ from neon_utils.skill_utils import get_skill_metadata
+
+ # Set up mock return values
+ mock_poetry_data = {"name": "skill-homeassistant", "source": "poetry"}
+ mock_setuptools_data = {
+ "name": "skill-caffeinewiz",
+ "source": "setuptools",
+ }
+ mock_poetry.return_value = mock_poetry_data
+ mock_setuptools.return_value = mock_setuptools_data
+
+ # Test poetry skill (homeassistant)
+ result_poetry = get_skill_metadata(self.poetry_skill_repo_path)
+ mock_poetry.assert_called_with(self.pyproject_toml_path)
+ mock_setuptools.assert_not_called()
+ self.assertEqual(result_poetry['name'], mock_poetry_data['name'])
+
+ # Reset mocks
+ mock_poetry.reset_mock()
+ mock_setuptools.reset_mock()
+
+ # Test setuptools skill (caffeinewiz)
+ result_setuptools = get_skill_metadata(self.setuptools_skill_repo_path)
+ mock_setuptools.assert_called_with(self.setup_py_path)
+ mock_poetry.assert_not_called()
+ self.assertEqual(result_setuptools['name'], mock_setuptools_data['name'])
+ # Test FileNotFoundError for non-existent directory
+ with self.assertRaises(FileNotFoundError):
+ get_skill_metadata("/non/existent/directory")
+
+ def test_get_skill_data_poetry(self):
+ from neon_utils.skill_utils import _get_skill_data_poetry
+
+ # Test with real skill-homeassistant pyproject.toml file
+ self.assertTrue(
+ os.path.exists(self.pyproject_toml_path),
+ f"pyproject.toml not found at {self.pyproject_toml_path}",
+ )
+
+ skill_metadata = _get_skill_data_poetry(self.pyproject_toml_path)
+ self.assertIsInstance(skill_metadata, dict)
+ self.assertEqual(skill_metadata.get("name"), "skill-homeassistant")
+ self.assertEqual(
+ skill_metadata.get("package_name"), "skill-homeassistant"
+ )
+ self.assertEqual(skill_metadata.get("pip_spec"), "skill-homeassistant")
+ self.assertEqual(skill_metadata.get("version"), "0.5.1")
+ self.assertEqual(skill_metadata.get("license"), "Apache-2.0")
+ self.assertIn("Mike Gray", skill_metadata.get("author")[0])
+ self.assertEqual(skill_metadata.get("title"), "skill-homeassistant")
+ self.assertIsInstance(skill_metadata.get("tags"), list)
+ self.assertIn("neon", skill_metadata.get("tags"))
+ self.assertIsInstance(
+ skill_metadata.get("requirements", {}).get("python"), list
+ )
+ self.assertGreaterEqual(
+ len(skill_metadata["requirements"]["python"]), 1
+ )
+
+ # Test params from README
+ self.assertIsInstance(skill_metadata["summary"], str)
+
+ # Test FileNotFoundError for non-existent file
+ with self.assertRaises(FileNotFoundError):
+ _get_skill_data_poetry("non_existent_file.toml")
+
+ def test_get_skill_data_setuptools(self):
+ from neon_utils.skill_utils import _get_skill_data_setuptools
+
+ # Test with real skill-caffeinewiz setup.py file
+ self.assertTrue(
+ os.path.exists(self.setup_py_path),
+ f"setup.py not found at {self.setup_py_path}",
+ )
+
+ skill_metadata = _get_skill_data_setuptools(self.setup_py_path)
+ self.assertIsInstance(skill_metadata, dict)
+ self.assertEqual(skill_metadata.get("authorname"), "NeonGeckoCom")
+ self.assertEqual(skill_metadata.get("skillname"), "skill-caffeinewiz")
+ self.assertEqual(
+ skill_metadata["package_name"], "neon-skill-caffeinewiz"
+ )
+ self.assertEqual(skill_metadata["pip_spec"], "neon-skill-caffeinewiz")
+ self.assertIn("BSD-3", skill_metadata["license"])
+ self.assertIsInstance(skill_metadata["author"], str)
+ self.assertEqual(skill_metadata["version"], "0.3.1")
+ self.assertIsInstance(skill_metadata["url"], str)
+ self.assertEqual(skill_metadata["name"], "neon-skill-caffeinewiz")
+ self.assertEqual(skill_metadata["title"], "neon-skill-caffeinewiz")
+ self.assertIn("Neongecko", skill_metadata["credits"])
+ self.assertIsInstance(skill_metadata["requirements"]["python"], list)
+ self.assertGreaterEqual(
+ len(skill_metadata["requirements"]["python"]), 1
+ )
+
+ # Test params from README
+ self.assertIsInstance(skill_metadata["summary"], str)
+
+ # Test FileNotFoundError for non-existent file
+ with self.assertRaises(FileNotFoundError):
+ _get_skill_data_setuptools("non_existent_setup.py")
+
+ def test_get_skill_data_readme(self):
+ from neon_utils.skill_utils import _get_skill_data_readme
+
+ valid_readme_file = join(dirname(__file__), "test_skill_json", "README.md")
+
+ with open(valid_readme_file, 'r') as f:
+ readme_contents = f.read()
+
+ skill_metadata = _get_skill_data_readme(readme_contents)
+ self.assertIsInstance(skill_metadata, dict)
+ self.assertIsInstance(skill_metadata['examples'], list)
+ self.assertIsInstance(skill_metadata['incompatible_skills'], list)
+ self.assertIsInstance(skill_metadata['categories'], list)
+ self.assertIsInstance(skill_metadata['tags'], list)
+ self.assertIsInstance(skill_metadata['credits'], list)
+
+ self.assertIsInstance(skill_metadata['summary'], str)
+ self.assertIsInstance(skill_metadata['description'], str)
+
+ self.assertIsInstance(skill_metadata['title'], str)
+ self.assertIsInstance(skill_metadata['icon'], str)
+
+ # Test invalid README contents
+ with self.assertRaises(ValueError):
+ _get_skill_data_readme("")
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/version.py b/version.py
index 1ce310e8..f6ec4d9c 100644
--- a/version.py
+++ b/version.py
@@ -26,4 +26,4 @@
# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-__version__ = "1.13.0"
+__version__ = "1.14.0"