From 9973d187f1b13004ce21ff51f9fea94e67283b88 Mon Sep 17 00:00:00 2001 From: Tim Paine <3105306+timkpaine@users.noreply.github.com> Date: Sat, 21 Mar 2026 16:41:35 -0400 Subject: [PATCH] Move from hydra to lerna Signed-off-by: Tim Paine <3105306+timkpaine@users.noreply.github.com> --- ccflow/base.py | 13 ++++++-- ccflow/tests/test_base_registry.py | 6 +++- ccflow/tests/utils/test_hydra.py | 4 +-- ccflow/utils/hydra.py | 49 +++++++++++++++++++++++++----- pyproject.toml | 3 +- 5 files changed, 60 insertions(+), 15 deletions(-) diff --git a/ccflow/base.py b/ccflow/base.py index 2309577..9cfdcb9 100644 --- a/ccflow/base.py +++ b/ccflow/base.py @@ -588,7 +588,10 @@ def create_config_from_path( Returns: The instance of the model registry, with the configs loaded. """ - import hydra # Heavy import, only import if used. + try: + import lerna as hydra + except ImportError: + import hydra # Heavy import, only import if used. overrides = overrides or [] path = pathlib.Path(path).absolute() # Hydra requires absolute paths @@ -675,8 +678,12 @@ def load_config(self, cfg: DictConfig, registry: ModelRegistry, skip_exceptions: # This also allows for nested attributes on the model itself to # be constructed, even if they are not themselves of BaseModel type, # or if they are of a specific subclass of the parent. - from hydra.errors import InstantiationException - from hydra.utils import instantiate + try: + from lerna.errors import InstantiationException + from lerna.utils import instantiate + except ImportError: + from hydra.errors import InstantiationException + from hydra.utils import instantiate models_to_register = self._make_subregistries(cfg, [registry]) while True: diff --git a/ccflow/tests/test_base_registry.py b/ccflow/tests/test_base_registry.py index 46175e3..35e6f3c 100644 --- a/ccflow/tests/test_base_registry.py +++ b/ccflow/tests/test_base_registry.py @@ -6,7 +6,11 @@ from unittest import TestCase import pytest -from hydra.errors import InstantiationException + +try: + from lerna.errors import InstantiationException +except ImportError: + from hydra.errors import InstantiationException from omegaconf import OmegaConf from omegaconf.errors import InterpolationKeyError from pydantic import ConfigDict diff --git a/ccflow/tests/utils/test_hydra.py b/ccflow/tests/utils/test_hydra.py index 7c9886e..47f8732 100644 --- a/ccflow/tests/utils/test_hydra.py +++ b/ccflow/tests/utils/test_hydra.py @@ -338,7 +338,7 @@ def test_debug(basepath): assert "hydra/job_logging" in result.group_options assert len(result.group_options["hydra/job_logging"]) > 1 assert "config_user" in result.group_options - assert result.group_options["config_user"] == ["sample"] + assert "sample" in result.group_options["config_user"] # Arguable whether these should be here assert "conf_out_of_order" in result.group_options[""] @@ -347,7 +347,7 @@ def test_debug(basepath): assert merged assert "foo" in merged assert "config_user" in merged - assert merged["config_user"]["__options__"] == ["sample"] + assert "sample" in merged["config_user"]["__options__"] assert merged["config_user"]["__parent__"] == "conf" # Maybe this should be a path to a file assert merged["config_user"]["__selected__"] == "sample" assert "user_foo" in merged["config_user"] diff --git a/ccflow/utils/hydra.py b/ccflow/utils/hydra.py index f457c39..fa4805e 100644 --- a/ccflow/utils/hydra.py +++ b/ccflow/utils/hydra.py @@ -1,6 +1,7 @@ import argparse import inspect import os +import sys from dataclasses import dataclass from logging import getLogger from pathlib import Path @@ -8,7 +9,10 @@ from textwrap import dedent from typing import Any, Callable, Dict, List, Optional -from hydra._internal.defaults_list import DefaultsList +try: + from lerna._internal.defaults_list import DefaultsList +except ImportError: + from hydra._internal.defaults_list import DefaultsList from omegaconf import DictConfig, ListConfig, OmegaConf try: @@ -115,7 +119,10 @@ def _find_group_options(config_loader, path, config_name, overrides, results): Note that it will pick up config files that are not intended to be used as config group options, but that exist to provide common config options to other files in the group (i.e. to default) """ - from hydra.core.object_type import ObjectType + try: + from lerna.core.object_type import ObjectType + except ImportError: + from hydra.core.object_type import ObjectType groups = config_loader.get_group_options(path, ObjectType.GROUP, config_name, overrides) options = config_loader.get_group_options(path, ObjectType.CONFIG, config_name, overrides) @@ -184,9 +191,10 @@ def load_config( debug: (Experimental) Whether to enable debug mode. This will return more information about the configs on ConfigLoadResult. """ # Heavy import, only import if used - import os - - from hydra import compose, initialize_config_dir + try: + from lerna import compose, initialize_config_dir + except ImportError: + from hydra import compose, initialize_config_dir if return_hydra_config and debug: raise ValueError("Cannot return hydra config and debug=True at the same time. Please set return_hydra_config=False.") @@ -209,22 +217,47 @@ def load_config( result = ConfigLoadResult(root_config_dir=root_config_dir, root_config_name=root_config_name, cfg=cfg) if debug: import yaml - from hydra.core.global_hydra import GlobalHydra - from hydra.types import RunMode + + try: + from lerna.core.global_hydra import GlobalHydra + from lerna.types import RunMode + except ImportError: + from hydra.core.global_hydra import GlobalHydra + from hydra.types import RunMode # To track the source file for each config value, we need to monkey patch the yaml loader original_yaml_load = yaml.load + # Lerna may use a Rust YAML parser that bypasses yaml.load entirely. + # Temporarily disable it so our monkey patch can intercept all YAML loading. + _rust_patches = {} + try: + for _mod_name in ( + "lerna._internal.core_plugins.file_config_source", + "lerna._internal.core_plugins.importlib_resources_config_source", + ): + _mod = sys.modules.get(_mod_name) + if _mod and getattr(_mod, "_RUST_AVAILABLE", False): + _rust_patches[_mod] = True + _mod._RUST_AVAILABLE = False + except Exception: + pass + try: def yaml_load(*args, **kwargs): res = original_yaml_load(*args, **kwargs) - return _dict_add_source(res, args[0].name) + # hydra passes file objects (with .name) to yaml.load; + # lerna passes strings. Use "unknown" as fallback source. + source = getattr(args[0], "name", "unknown") if args else "unknown" + return _dict_add_source(res, source) yaml.load = yaml_load # We can't load the hydra config after monkey patching yaml loading, so skip that step result.cfg_sources = compose(config_name=root_config_name, overrides=overrides, return_hydra_config=False) finally: yaml.load = original_yaml_load + for _mod, _val in _rust_patches.items(): + _mod._RUST_AVAILABLE = _val config_loader = GlobalHydra.instance().config_loader() # Load defaults list using the standard hydra function diff --git a/pyproject.toml b/pyproject.toml index 7d1dd26..2286b58 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,9 +41,9 @@ dependencies = [ "cloudpickle", "dask", "deprecated", - "hydra-core", "IPython", "jinja2", + "lerna", "narwhals", "numpy<3", "orjson", @@ -87,6 +87,7 @@ develop = [ "cexprtk", "csp>=0.8.0,<1", "duckdb", + "hydra-core", "pandas", "panel", "panel_material_ui",