From 7c5e0d4bd6e8e1e1d77f53716c3eb59f9848ea8d Mon Sep 17 00:00:00 2001 From: Daniel McKnight Date: Fri, 29 Aug 2025 09:03:00 -0700 Subject: [PATCH 1/4] Refactor to use importlib instead of pkg_resources in `packaging_utils` --- neon_utils/packaging_utils.py | 33 ++++++++++++++++++++------------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/neon_utils/packaging_utils.py b/neon_utils/packaging_utils.py index a8d7ea42..4be190dc 100644 --- a/neon_utils/packaging_utils.py +++ b/neon_utils/packaging_utils.py @@ -33,7 +33,6 @@ from typing import Tuple, Optional, List from tempfile import mkstemp -import pkg_resources import sysconfig from os.path import exists, join, isfile @@ -63,13 +62,16 @@ def get_package_version_spec(pkg: str): """ Locate an installed package and return its reported version :param pkg: string package name to locate - :returns: Version string as reported by pkg_resources - :raises: ModuleNotFoundError if requested package isn't installed + :returns: Version string as reported by importlib.metadata + :raises: PackageNotFoundError if requested package isn't installed """ try: - return pkg_resources.get_distribution(pkg).version - except pkg_resources.DistributionNotFound: - raise ModuleNotFoundError(f"{pkg} not found") + from importlib.metadata import version + except ImportError: + # Fallback for Python < 3.8 + from importlib_metadata import version + + return version(pkg) def get_package_dependencies(pkg: str): @@ -77,15 +79,20 @@ def get_package_dependencies(pkg: str): Get the dependencies for an installed package :param pkg: string package name to evaluate :returns: list of string dependencies (equivalent to requirements.txt) - :raises ModuleNotFoundError if requested package isn't installed + :raises PackageNotFoundError if requested package isn't installed """ try: - constraints = pkg_resources.working_set.by_key[pkg].requires() - constraints_spec = [str(c).split('[', 1)[0] for c in constraints] - LOG.debug(constraints_spec) - return constraints_spec - except KeyError: - raise ModuleNotFoundError(f"{pkg} not found") + from importlib.metadata import requires + except ImportError: + # Fallback for Python < 3.8 + from importlib_metadata import requires + + requirements = requires(pkg) + if requirements is None: + return [] + constraints_spec = [req.split('[', 1)[0] for req in requirements] + LOG.debug(constraints_spec) + return constraints_spec @deprecated("Reference `neon_core.version.__version__` directly", "2.0.0") From 5c697675d56ec3223326861d79db1fe3043fa2af Mon Sep 17 00:00:00 2001 From: Daniel McKnight Date: Fri, 29 Aug 2025 09:15:57 -0700 Subject: [PATCH 2/4] Update unit test to account for valid behavior change --- tests/packaging_util_tests.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/packaging_util_tests.py b/tests/packaging_util_tests.py index 8a8c2077..e8d6fd61 100644 --- a/tests/packaging_util_tests.py +++ b/tests/packaging_util_tests.py @@ -114,14 +114,16 @@ def test_get_packaged_core_version(self): def test_get_package_dependencies(self): self_deps = get_package_dependencies("neon-utils") - requirements_file = join(os.path.dirname(os.path.dirname(__file__)), + requirements_file = os.path.join(os.path.dirname(os.path.dirname(__file__)), "requirements", "requirements.txt") with open(requirements_file) as f: spec_requirements = f.read().split('\n') spec_requirements = [r for r in spec_requirements if r and not r.startswith('#')] - # Version specs aren't order-dependent, so they can't be compared - self.assertEqual(len(self_deps), len(spec_requirements)) + # Extra dependencies may change by environment, so just spot check the + # required dependencies are present + for dep in spec_requirements: + self.assertIn(dep, self_deps) with self.assertRaises(ModuleNotFoundError): get_package_dependencies("fakeneongeckopackage") From 6aee6308a5504bd2766cf6dc82673af3486973af Mon Sep 17 00:00:00 2001 From: Daniel McKnight Date: Fri, 29 Aug 2025 09:34:06 -0700 Subject: [PATCH 3/4] Update packaging_utils and tests for backwards-compat --- neon_utils/packaging_utils.py | 2 +- tests/packaging_util_tests.py | 19 +++++++++++-------- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/neon_utils/packaging_utils.py b/neon_utils/packaging_utils.py index 4be190dc..cde4d034 100644 --- a/neon_utils/packaging_utils.py +++ b/neon_utils/packaging_utils.py @@ -90,7 +90,7 @@ def get_package_dependencies(pkg: str): requirements = requires(pkg) if requirements is None: return [] - constraints_spec = [req.split('[', 1)[0] for req in requirements] + constraints_spec = [req.split('[', 1)[0].split(';', 1)[0] for req in requirements] LOG.debug(constraints_spec) return constraints_spec diff --git a/tests/packaging_util_tests.py b/tests/packaging_util_tests.py index e8d6fd61..6a7e67a5 100644 --- a/tests/packaging_util_tests.py +++ b/tests/packaging_util_tests.py @@ -114,16 +114,19 @@ def test_get_packaged_core_version(self): def test_get_package_dependencies(self): self_deps = get_package_dependencies("neon-utils") - requirements_file = os.path.join(os.path.dirname(os.path.dirname(__file__)), - "requirements", "requirements.txt") - with open(requirements_file) as f: - spec_requirements = f.read().split('\n') + requirements_dir = os.path.join(os.path.dirname(os.path.dirname(__file__)), "requirements") + spec_requirements = [] + for _, _, files in os.walk(requirements_dir): + for file in files: + with open(os.path.join(requirements_dir, file)) as f: + spec_requirements += f.read().split('\n') + spec_requirements = [r for r in spec_requirements if r and not r.startswith('#')] - # Extra dependencies may change by environment, so just spot check the - # required dependencies are present - for dep in spec_requirements: - self.assertIn(dep, self_deps) + + # Version specs aren't order-dependent, so they can't be compared + # Also, constraints are normalized so elements cannot be compared + self.assertEqual(len(self_deps), len(spec_requirements)) with self.assertRaises(ModuleNotFoundError): get_package_dependencies("fakeneongeckopackage") From e2f2d64ae5814b74330188999e6c8cd730e9a531 Mon Sep 17 00:00:00 2001 From: Daniel McKnight Date: Fri, 29 Aug 2025 09:48:54 -0700 Subject: [PATCH 4/4] Set `max-parallel` jobs to prevent HANA rate-limiting errors --- .github/workflows/unit_tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 877f823c..59dfcc38 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -10,6 +10,7 @@ jobs: uses: neongeckocom/.github/.github/workflows/python_build_tests.yml@master skill_object_tests: strategy: + max-parallel: 2 matrix: python-version: [ 3.9, '3.10', '3.11', '3.12' ] runs-on: ubuntu-latest