|
13 | 13 | # See the License for the specific language governing permissions and |
14 | 14 | # limitations under the License. |
15 | 15 |
|
| 16 | +from __future__ import annotations |
| 17 | + |
16 | 18 | import importlib |
17 | 19 | import inspect |
| 20 | +import logging |
18 | 21 | import os |
19 | 22 | import pkgutil |
20 | 23 | import sys |
| 24 | +from types import ModuleType |
| 25 | + |
| 26 | +from EngineLoader.base import CompilerEngine |
| 27 | +from EngineLoader import registry as engine_registry |
21 | 28 |
|
| 29 | +logger = logging.getLogger(__name__) |
22 | 30 |
|
23 | | -def _import_engine_modules(base_path: str, project_root: str | None = None) -> list[object]: |
24 | | - """Import all engine packages and submodules found under ENGINES/ dynamically.""" |
25 | | - imported: list[object] = [] |
| 31 | + |
| 32 | +def _iter_engine_module_names(base_path: str, namespace_package: str) -> list[str]: |
| 33 | + """Return top-level engine package names under the configured namespace.""" |
26 | 34 | if not os.path.isdir(base_path): |
27 | | - return imported |
| 35 | + return [] |
| 36 | + prefix = f"{namespace_package.rstrip('.') }." |
| 37 | + return [ |
| 38 | + name |
| 39 | + for _finder, name, ispkg in pkgutil.iter_modules([base_path], prefix=prefix) |
| 40 | + if ispkg |
| 41 | + ] |
28 | 42 |
|
29 | | - # Prefer project-root namespace imports (ENGINES.<name>) to avoid collisions. |
30 | | - if project_root and project_root not in sys.path: |
31 | | - sys.path.insert(0, project_root) |
32 | | - if base_path not in sys.path: |
33 | | - sys.path.insert(0, base_path) |
34 | 43 |
|
35 | | - for _finder, name, ispkg in pkgutil.iter_modules([base_path]): |
36 | | - if not ispkg: |
| 44 | +def _register_engine_classes(module: ModuleType) -> list[str]: |
| 45 | + """Register CompilerEngine subclasses declared by an imported module.""" |
| 46 | + registered: list[str] = [] |
| 47 | + for _name, obj in inspect.getmembers(module, inspect.isclass): |
| 48 | + if not issubclass(obj, CompilerEngine) or obj is CompilerEngine: |
37 | 49 | continue |
| 50 | + try: |
| 51 | + if not str(getattr(obj, "__module__", "")).startswith(module.__name__): |
| 52 | + continue |
| 53 | + engine_registry.engine_register(obj) |
| 54 | + registered.append(getattr(obj, "id", obj.__name__)) |
| 55 | + except Exception: |
| 56 | + logger.exception( |
| 57 | + "Failed to register engine class %s from module %s", |
| 58 | + getattr(obj, "__name__", repr(obj)), |
| 59 | + module.__name__, |
| 60 | + ) |
| 61 | + return registered |
38 | 62 |
|
39 | | - module_names: list[str] = [] |
40 | | - if project_root: |
41 | | - module_names.append(f"ENGINES.{name}") |
42 | | - module_names.append(name) |
43 | 63 |
|
44 | | - package = None |
45 | | - for mod_name in module_names: |
46 | | - try: |
47 | | - package = importlib.import_module(mod_name) |
48 | | - imported.append(package) |
49 | | - break |
50 | | - except Exception: |
51 | | - continue |
52 | | - if package is None: |
53 | | - continue |
| 64 | +def _import_module_tree(module_name: str) -> list[ModuleType]: |
| 65 | + """Import a package and its submodules, returning successfully loaded modules.""" |
| 66 | + imported: list[ModuleType] = [] |
| 67 | + root_module = importlib.import_module(module_name) |
| 68 | + imported.append(root_module) |
| 69 | + |
| 70 | + module_path = getattr(root_module, "__path__", None) |
| 71 | + if not module_path: |
| 72 | + return imported |
54 | 73 |
|
55 | | - pkg_name = getattr(package, "__name__", "") |
56 | | - pkg_path = os.path.join(base_path, name) |
| 74 | + for _finder, subname, _ispkg in pkgutil.walk_packages(module_path, prefix=f"{module_name}."): |
57 | 75 | try: |
58 | | - for _f, sub_name, _is_pkg in pkgutil.walk_packages( |
59 | | - [pkg_path], prefix=f"{pkg_name}." |
60 | | - ): |
61 | | - try: |
62 | | - sub = importlib.import_module(sub_name) |
63 | | - imported.append(sub) |
64 | | - except Exception: |
65 | | - continue |
| 76 | + imported.append(importlib.import_module(subname)) |
66 | 77 | except Exception: |
67 | | - continue |
68 | | - |
| 78 | + logger.exception("Failed to import engine submodule %s", subname) |
69 | 79 | return imported |
70 | 80 |
|
71 | 81 |
|
72 | | -def _register_discovered_engines(modules: list[object]) -> None: |
73 | | - """Register all CompilerEngine subclasses discovered in imported modules.""" |
| 82 | +def _sync_engine_sdk_registry() -> None: |
| 83 | + """Keep engine_sdk.registry aligned with the active EngineLoader registry module.""" |
74 | 84 | try: |
75 | | - from EngineLoader import registry as _registry # local import to avoid cycles |
76 | | - from EngineLoader.base import CompilerEngine |
| 85 | + import engine_sdk |
| 86 | + |
| 87 | + engine_sdk.registry = engine_registry |
77 | 88 | except Exception: |
78 | | - return |
| 89 | + logger.exception("Failed to synchronize engine_sdk.registry with EngineLoader.registry") |
79 | 90 |
|
80 | | - for mod in modules: |
81 | | - try: |
82 | | - for _, cls in inspect.getmembers(mod, inspect.isclass): |
83 | | - try: |
84 | | - if not issubclass(cls, CompilerEngine) or cls is CompilerEngine: |
85 | | - continue |
86 | | - except Exception: |
87 | | - continue |
| 91 | + |
| 92 | +def _discover_external_plugins(base_path: str, namespace_package: str = "ENGINES") -> None: |
| 93 | + """Import namespaced engine packages and register CompilerEngine classes dynamically.""" |
| 94 | + try: |
| 95 | + if not os.path.isdir(base_path): |
| 96 | + return |
| 97 | + |
| 98 | + parent_dir = os.path.abspath(os.path.dirname(base_path)) |
| 99 | + if parent_dir not in sys.path: |
| 100 | + sys.path.insert(0, parent_dir) |
| 101 | + |
| 102 | + for module_name in _iter_engine_module_names(base_path, namespace_package): |
| 103 | + try: |
| 104 | + imported_modules = _import_module_tree(module_name) |
| 105 | + except Exception: |
| 106 | + logger.exception("Failed to import engine package %s", module_name) |
| 107 | + continue |
| 108 | + |
| 109 | + registered_any = False |
| 110 | + for module in imported_modules: |
88 | 111 | try: |
89 | | - eid = getattr(cls, "id", None) |
90 | | - if not isinstance(eid, str) or not eid.strip(): |
91 | | - continue |
92 | | - _registry.register(cls) |
| 112 | + if _register_engine_classes(module): |
| 113 | + registered_any = True |
93 | 114 | except Exception: |
94 | | - continue |
95 | | - except Exception: |
96 | | - continue |
| 115 | + logger.exception( |
| 116 | + "Failed while introspecting engine module %s", |
| 117 | + getattr(module, "__name__", module_name), |
| 118 | + ) |
| 119 | + if not registered_any: |
| 120 | + logger.warning( |
| 121 | + "No CompilerEngine subclasses were discovered in engine package %s", |
| 122 | + module_name, |
| 123 | + ) |
| 124 | + except Exception: |
| 125 | + logger.exception("Engine discovery failed for base path %s", base_path) |
| 126 | + finally: |
| 127 | + _sync_engine_sdk_registry() |
97 | 128 |
|
98 | 129 |
|
99 | 130 | def _auto_discover() -> None: |
100 | | - """Discover and register engine plugins dynamically from the ENGINES folder.""" |
| 131 | + """Discover and register external engine plugins from the project ENGINES folder.""" |
101 | 132 | base_dir = os.path.dirname(__file__) |
102 | 133 | try: |
103 | 134 | project_root = os.path.abspath(os.path.join(base_dir, os.pardir, os.pardir)) |
104 | 135 | external_dir = os.path.join(project_root, "ENGINES") |
105 | | - modules = _import_engine_modules(external_dir, project_root=project_root) |
106 | | - _register_discovered_engines(modules) |
| 136 | + _discover_external_plugins(external_dir, namespace_package="ENGINES") |
107 | 137 | except Exception: |
108 | | - pass |
| 138 | + logger.exception("Automatic engine discovery failed") |
| 139 | + _sync_engine_sdk_registry() |
0 commit comments