diff --git a/colcon_core/python_project/hook_caller/__init__.py b/colcon_core/python_project/hook_caller/__init__.py new file mode 100644 index 00000000..854d52e0 --- /dev/null +++ b/colcon_core/python_project/hook_caller/__init__.py @@ -0,0 +1,141 @@ +# Copyright 2022 Open Source Robotics Foundation, Inc. +# Licensed under the Apache License, Version 2.0 + +from contextlib import AbstractContextManager +import os +import pickle +import sys + +from colcon_core.python_project.hook_caller import _call_hook +from colcon_core.python_project.hook_caller import _list_hooks +from colcon_core.python_project.spec import load_and_cache_spec +from colcon_core.subprocess import run + + +class _SubprocessTransport(AbstractContextManager): + + def __enter__(self): + self.child_in, self.parent_out = os.pipe() + self.parent_in, self.child_out = os.pipe() + + try: + import msvcrt + except ImportError: + os.set_inheritable(self.child_in, True) + self.pass_in = self.child_in + os.set_inheritable(self.child_out, True) + self.pass_out = self.child_out + else: + self.pass_in = msvcrt.get_osfhandle(self.child_in) + os.set_handle_inheritable(self.pass_in, True) + self.pass_out = msvcrt.get_osfhandle(self.child_out) + os.set_handle_inheritable(self.pass_out, True) + + return self + + def __exit__(self, exc_type, exc_value, traceback): + os.close(self.parent_out) + os.close(self.parent_in) + os.close(self.child_out) + os.close(self.child_in) + + +class AsyncHookCaller: + """Calls PEP 517 style hooks asynchronously in a new process.""" + + def __init__( + self, backend_name, *, project_path=None, env=None, + stdout_callback=None, stderr_callback=None, + ): + """ + Initialize a new AsyncHookCaller. + + :param backend_name: The name of the PEP 517 build backend. + :param project_path: Path to the project's root directory. + :param env: Environment variables to use when invoking hooks. + :param stdout_callback: Callback for stdout from the hook invocation. + :param stderr_callback: Callback for stderr from the hook invocation. + """ + self._backend_name = backend_name + self._project_path = str(project_path) if project_path else None + self._env = dict(env if env is not None else os.environ) + self._stdout_callback = stdout_callback + self._stderr_callback = stderr_callback + + @property + def backend_name(self): + """Get the name of the backend to call hooks on.""" + return self._backend_name + + @property + def env(self): + """Get the environment variables to use when invoking hooks.""" + return self._env + + async def list_hooks(self): + """ + Call into the backend to list implemented hooks. + + This function lists all callable methods on the backend, which may + include more than just the hook names. + + :returns: List of hook names. + """ + args = [ + sys.executable, _list_hooks.__file__, + self._backend_name] + process = await run( + args, None, self._stderr_callback, + cwd=self._project_path, env=self.env, + capture_output=True) + process.check_returncode() + hook_names = [ + line.strip().decode() for line in process.stdout.splitlines()] + return [ + hook for hook in hook_names if hook and not hook.startswith('_')] + + async def call_hook(self, hook_name, **kwargs): + """ + Call the given hook with given arguments. + + :param hook_name: Name of the hook to call. + """ + with _SubprocessTransport() as transport: + args = [ + sys.executable, _call_hook.__file__, + self._backend_name, hook_name, + str(transport.pass_in), str(transport.pass_out)] + with os.fdopen(os.dup(transport.parent_out), 'wb') as f: + pickle.dump(kwargs, f) + have_callbacks = self._stdout_callback or self._stderr_callback + process = await run( + args, self._stdout_callback, self._stderr_callback, + cwd=self._project_path, env=self.env, close_fds=False, + capture_output=not have_callbacks) + process.check_returncode() + with os.fdopen(os.dup(transport.parent_in), 'rb') as f: + res = pickle.load(f) + return res + + +def get_hook_caller(desc, **kwargs): + """ + Create a new AsyncHookCaller instance for a package descriptor. + + :param desc: The package descriptor + """ + spec = load_and_cache_spec(desc) + backend_path = spec['build-system'].get('backend-path') + if backend_path: + # TODO: This isn't *technically* the beginning of sys.path + # as PEP 517 calls for, but it's pretty darn close. + kwargs['env'] = { + **kwargs.get('env', os.environ), + 'PYTHONDONTWRITEBYTECODE': '1', + } + pythonpath = kwargs['env'].get('PYTHONPATH', '') + kwargs['env']['PYTHONPATH'] = os.pathsep.join( + backend_path + ([pythonpath] if pythonpath else [])) + return AsyncHookCaller( + spec['build-system']['build-backend'], + project_path=desc.path, **kwargs) diff --git a/colcon_core/python_project/hook_caller/_call_hook.py b/colcon_core/python_project/hook_caller/_call_hook.py new file mode 100644 index 00000000..e9e86aa6 --- /dev/null +++ b/colcon_core/python_project/hook_caller/_call_hook.py @@ -0,0 +1,29 @@ +# Copyright 2022 Open Source Robotics Foundation, Inc. +# Licensed under the Apache License, Version 2.0 + +from importlib import import_module +import os +import pickle +import sys + + +if __name__ == '__main__': + backend_name, hook_name, child_in, child_out = sys.argv[1:] + try: + import msvcrt + except ImportError: + pass + else: + child_in = msvcrt.open_osfhandle(int(child_in), os.O_RDONLY) + child_out = msvcrt.open_osfhandle(int(child_out), 0) + if ':' in backend_name: + backend_module_name, backend_object_name = backend_name.split(':', 2) + backend_module = import_module(backend_module_name) + backend = getattr(backend_module, backend_object_name) + else: + backend = import_module(backend_name) + with os.fdopen(int(child_in), 'rb') as f: + kwargs = pickle.load(f) or {} + res = getattr(backend, hook_name)(**kwargs) + with os.fdopen(int(child_out), 'wb') as f: + pickle.dump(res, f) diff --git a/colcon_core/python_project/hook_caller/_list_hooks.py b/colcon_core/python_project/hook_caller/_list_hooks.py new file mode 100644 index 00000000..c5551e86 --- /dev/null +++ b/colcon_core/python_project/hook_caller/_list_hooks.py @@ -0,0 +1,19 @@ +# Copyright 2023 Open Source Robotics Foundation, Inc. +# Licensed under the Apache License, Version 2.0 + +from importlib import import_module +import sys + + +if __name__ == '__main__': + backend_name = sys.argv[1] + if ':' in backend_name: + backend_module_name, backend_object_name = backend_name.split(':', 2) + backend_module = import_module(backend_module_name) + backend = getattr(backend_module, backend_object_name) + else: + backend = import_module(backend_name) + + for attr in dir(backend): + if callable(getattr(backend, attr)): + print(attr) diff --git a/colcon_core/python_project/hook_caller_decorator/__init__.py b/colcon_core/python_project/hook_caller_decorator/__init__.py new file mode 100644 index 00000000..014cfb10 --- /dev/null +++ b/colcon_core/python_project/hook_caller_decorator/__init__.py @@ -0,0 +1,93 @@ +# Copyright 2022 Open Source Robotics Foundation, Inc. +# Licensed under the Apache License, Version 2.0 + +import traceback + +from colcon_core.logging import colcon_logger +from colcon_core.plugin_system import instantiate_extensions +from colcon_core.plugin_system import order_extensions_by_priority +from colcon_core.python_project.hook_caller import get_hook_caller + +logger = colcon_logger.getChild(__name__) + + +class HookCallerDecoratorExtensionPoint: + """ + The interface for PEP 517 hook caller decorator extensions. + + For each instance the attribute `HOOK_CALLER_DECORATOR_NAME` is being + set to the basename of the entry point registering the extension. + """ + + """The version of the hook caller decorator extension interface.""" + EXTENSION_POINT_VERSION = '1.0' + + """The default priority of hook caller decorator extensions.""" + PRIORITY = 100 + + def decorate_hook_caller(self, *, hook_caller): + """ + Decorate a hook caller to perform additional functionality. + + This method must be overridden in a subclass. + + :param hook_caller: The hook caller + :returns: A decorator + """ + raise NotImplementedError() + + +def get_hook_caller_extensions(): + """ + Get the available hook caller decorator extensions. + + The extensions are ordered by their priority and entry point name. + + :rtype: OrderedDict + """ + extensions = instantiate_extensions(__name__) + for name, extension in extensions.items(): + extension.HOOK_CALLER_DECORATOR_NAME = name + return order_extensions_by_priority(extensions) + + +def decorate_hook_caller(hook_caller): + """ + Decorate the hook caller using hook caller decorator extensions. + + :param hook_caller: The hook caller + + :returns: The decorated parser + """ + extensions = get_hook_caller_extensions() + for extension in extensions.values(): + logger.log( + 1, 'decorate_hook_caller() %s', + extension.HOOK_CALLER_DECORATOR_NAME) + try: + decorated_hook_caller = extension.decorate_hook_caller( + hook_caller=hook_caller) + assert hasattr(decorated_hook_caller, 'call_hook'), \ + 'decorate_hook_caller() should return something to call hooks' + except Exception as e: # noqa: F841 + # catch exceptions raised in decorator extension + exc = traceback.format_exc() + logger.error( + 'Exception in hook caller decorator extension ' + f"'{extension.HOOK_CALLER_DECORATOR_NAME}': {e}\n{exc}") + # skip failing extension, continue with next one + else: + hook_caller = decorated_hook_caller + + return hook_caller + + +def get_decorated_hook_caller(desc, **kwargs): + """ + Create and decorate a hook caller instance for a package descriptor. + + :param desc: The package descriptor + """ + hook_caller = get_hook_caller(desc, **kwargs) + decorated_hook_caller = decorate_hook_caller(hook_caller) + return decorated_hook_caller diff --git a/test/spell_check.words b/test/spell_check.words index 1dce75b3..f24e6afa 100644 --- a/test/spell_check.words +++ b/test/spell_check.words @@ -5,6 +5,7 @@ argparse asyncio autouse backend +backends backported basepath bazqux @@ -75,6 +76,7 @@ noqa notestscollected openpty optionxform +osfhandle pathlib pkgname pkgs diff --git a/test/test_flake8.py b/test/test_flake8.py index c7214ed6..8eb9cd60 100644 --- a/test/test_flake8.py +++ b/test/test_flake8.py @@ -25,7 +25,9 @@ def test_flake8(): show_source=True, ) style_guide_tests = get_style_guide( - extend_ignore=['D100', 'D101', 'D102', 'D103', 'D104', 'D105', 'D107'], + extend_ignore=[ + 'D100', 'D101', 'D102', 'D103', 'D104', 'D105', 'D106', 'D107', + ], show_source=True, ) diff --git a/test/test_hook_caller.py b/test/test_hook_caller.py new file mode 100644 index 00000000..131c4dac --- /dev/null +++ b/test/test_hook_caller.py @@ -0,0 +1,199 @@ +# Copyright 2026 Open Source Robotics Foundation, Inc. +# Licensed under the Apache License, Version 2.0 + +import os +from pathlib import Path + +from colcon_core.generic_decorator import GenericDecorator +from colcon_core.package_descriptor import PackageDescriptor +from colcon_core.python_project.hook_caller import AsyncHookCaller +from colcon_core.python_project.hook_caller import get_hook_caller +from colcon_core.python_project.hook_caller_decorator import \ + decorate_hook_caller +from colcon_core.python_project.hook_caller_decorator import \ + HookCallerDecoratorExtensionPoint + +from .extension_point_context import ExtensionPointContext +from .run_until_complete import run_until_complete + + +TEST_ARTIFACTS_PATH = Path(__file__).parent / 'test_hook_caller' + + +def test_async_hook_caller_list_hooks(): + env = { + **os.environ, + 'PYTHONPATH': os.pathsep.join(( + str(TEST_ARTIFACTS_PATH / 'mock_backends'), + os.environ.get('PYTHONPATH', ''), + )), + } + + caller = AsyncHookCaller('mock_backend', env=env) + + hooks = run_until_complete(caller.list_hooks()) + assert sorted(hooks) == [ + 'build_sdist', + 'build_wheel', + 'custom_hook', + ] + + +def test_async_hook_caller_list_hooks_object(): + env = { + **os.environ, + 'PYTHONPATH': os.pathsep.join(( + str(TEST_ARTIFACTS_PATH / 'mock_backends'), + os.environ.get('PYTHONPATH', ''), + )), + } + + caller = AsyncHookCaller('mock_backend_obj:backend_instance', env=env) + + hooks = run_until_complete(caller.list_hooks()) + assert sorted(hooks) == [ + 'build_sdist', + 'build_wheel', + 'custom_hook', + ] + + +def test_async_hook_caller_call_hook(): + env = { + **os.environ, + 'PYTHONPATH': os.pathsep.join(( + str(TEST_ARTIFACTS_PATH / 'mock_backends'), + os.environ.get('PYTHONPATH', ''), + )), + } + + stdout_log = [] + stderr_log = [] + + def stdout_callback(data): + stdout_log.append(data) + + def stderr_callback(data): + stderr_log.append(data) + + caller = AsyncHookCaller( + 'mock_backend', env=env, + stdout_callback=stdout_callback, stderr_callback=stderr_callback) + + res = run_until_complete(caller.call_hook('custom_hook', a=3, b=4)) + assert res == 7 + + res = run_until_complete(caller.call_hook('custom_hook', a=3)) + assert res == 5 + + +def test_async_hook_caller_call_hook_object(): + env = { + **os.environ, + 'PYTHONPATH': os.pathsep.join(( + str(TEST_ARTIFACTS_PATH / 'mock_backends'), + os.environ.get('PYTHONPATH', ''), + )), + } + + caller = AsyncHookCaller('mock_backend_obj:backend_instance', env=env) + + res = run_until_complete(caller.call_hook('custom_hook', a=5, b=5)) + assert res == 10 + + +def test_get_hook_caller_default_spec(): + # An empty directory has no pyproject.toml, so it uses default spec + empty_dir = TEST_ARTIFACTS_PATH / 'empty' + desc = PackageDescriptor(str(empty_dir)) + + caller = get_hook_caller(desc) + + assert caller.backend_name == 'setuptools.build_meta:__legacy__' + + +def test_get_hook_caller_with_backend_path(): + proj_dir = TEST_ARTIFACTS_PATH / 'with_backend_path' + desc = PackageDescriptor(str(proj_dir)) + + env = {'PYTHONPATH': 'initial_path'} + caller = get_hook_caller(desc, env=env) + + assert caller.backend_name == 'mock_backend' + + # The backend path items are added to the beginning of PYTHONPATH + assert caller.env['PYTHONPATH'] == os.pathsep.join(( + 'mock_backend_dir', + 'another_dir', + 'initial_path', + )) + + +def test_get_hook_caller_with_backend_path_no_pythonpath(): + proj_dir = TEST_ARTIFACTS_PATH / 'with_backend_path_no_pythonpath' + desc = PackageDescriptor(str(proj_dir)) + + caller = get_hook_caller(desc, env={}) + + assert caller.backend_name == 'mock_backend' + + # 'mock_backend_dir' is the only element, no trailing path separator + assert caller.env['PYTHONPATH'] == 'mock_backend_dir' + + +class MockDecoratorExtension(HookCallerDecoratorExtensionPoint): + """ + Testing extension point for hook caller decoration. + + This extension demonstrates how to apply a decorator to a specific backend. + """ + + class DecoratedHookCaller(GenericDecorator): + """ + Testing class for hook caller decoration. + + This decorator modifies the default value of the ``b`` argument of the + ``custom_hook`` function. + """ + + async def call_hook(self, hook_name, **kwargs): + if hook_name == 'custom_hook' and 'b' not in kwargs: + kwargs['b'] = 10 + return await self._decoree.call_hook(hook_name, **kwargs) + + def decorate_hook_caller(self, *, hook_caller): + if hook_caller.backend_name != 'mock_backend_obj:backend_instance': + return hook_caller + + return self.DecoratedHookCaller(hook_caller) + + +def test_decorate_hook_caller(): + env = { + **os.environ, + 'PYTHONPATH': os.pathsep.join(( + str(TEST_ARTIFACTS_PATH / 'mock_backends'), + os.environ.get('PYTHONPATH', ''), + )), + } + + caller_obj = AsyncHookCaller('mock_backend_obj:backend_instance', env=env) + caller_mod = AsyncHookCaller('mock_backend', env=env) + + with ExtensionPointContext(mock_decorator=MockDecoratorExtension): + decorated_obj = decorate_hook_caller(caller_obj) + decorated_mod = decorate_hook_caller(caller_mod) + + # Object backend custom_hook: b=5 -> 10, b=default(10) -> 15 + res = run_until_complete(decorated_obj.call_hook('custom_hook', a=5, b=5)) + assert res == 10 + + res = run_until_complete(decorated_obj.call_hook('custom_hook', a=5)) + assert res == 15 + + # Module backend custom_hook: unaffected, b=5 -> 10, b=default(2) -> 7 + res = run_until_complete(decorated_mod.call_hook('custom_hook', a=5, b=5)) + assert res == 10 + + res = run_until_complete(decorated_mod.call_hook('custom_hook', a=5)) + assert res == 7 diff --git a/test/test_hook_caller/empty/.gitkeep b/test/test_hook_caller/empty/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/test/test_hook_caller/mock_backends/mock_backend.py b/test/test_hook_caller/mock_backends/mock_backend.py new file mode 100644 index 00000000..4c89bed6 --- /dev/null +++ b/test/test_hook_caller/mock_backends/mock_backend.py @@ -0,0 +1,19 @@ +# Copyright 2026 Open Source Robotics Foundation, Inc. +# Licensed under the Apache License, Version 2.0 + +def build_wheel( + wheel_directory, config_settings=None, metadata_directory=None +): + return 'mock_wheel.whl' + + +def build_sdist(sdist_directory, config_settings=None): + return 'mock_sdist.tar.gz' + + +def custom_hook(a, b=2): + return a + b + + +def _private_hook(): + return 'hidden' diff --git a/test/test_hook_caller/mock_backends/mock_backend_obj.py b/test/test_hook_caller/mock_backends/mock_backend_obj.py new file mode 100644 index 00000000..0ee080cb --- /dev/null +++ b/test/test_hook_caller/mock_backends/mock_backend_obj.py @@ -0,0 +1,18 @@ +# Copyright 2026 Open Source Robotics Foundation, Inc. +# Licensed under the Apache License, Version 2.0 + +class Backend: + + def build_wheel( + self, wheel_directory, config_settings=None, metadata_directory=None + ): + return 'mock_wheel.whl' + + def build_sdist(self, sdist_directory, config_settings=None): + return 'mock_sdist.tar.gz' + + def custom_hook(self, a, b=2): + return a + b + + +backend_instance = Backend() diff --git a/test/test_hook_caller/with_backend_path/pyproject.toml b/test/test_hook_caller/with_backend_path/pyproject.toml new file mode 100644 index 00000000..4d394aeb --- /dev/null +++ b/test/test_hook_caller/with_backend_path/pyproject.toml @@ -0,0 +1,4 @@ +[build-system] +requires = [] +build-backend = 'mock_backend' +backend-path = ['mock_backend_dir', 'another_dir'] diff --git a/test/test_hook_caller/with_backend_path_no_pythonpath/pyproject.toml b/test/test_hook_caller/with_backend_path_no_pythonpath/pyproject.toml new file mode 100644 index 00000000..fa76cb9c --- /dev/null +++ b/test/test_hook_caller/with_backend_path_no_pythonpath/pyproject.toml @@ -0,0 +1,4 @@ +[build-system] +requires = [] +build-backend = 'mock_backend' +backend-path = ['mock_backend_dir']