diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 63b1024e..d2b075a9 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -15,6 +15,12 @@ paths are considered internals and can change in minor and patch releases. v4.50.0 (unreleased) -------------------- +Added +^^^^^ +- ``FromConfigMixin.from_config`` with ``config_read_mode_fsspec_enabled=True`` + now supports fsspec config loading including the resolving of relative paths + (`#918 `__). + Changed ^^^^^^^ - Drop support for Python 3.9. The minimum supported Python version is now diff --git a/DOCUMENTATION.rst b/DOCUMENTATION.rst index f294ae2c..3aa97c93 100644 --- a/DOCUMENTATION.rst +++ b/DOCUMENTATION.rst @@ -744,10 +744,11 @@ If you import ``from jsonargparse import set_parsing_settings`` and then run functions and classes will also support loading from URLs: :meth:`parse_path <.ArgumentParser.parse_path>`, :meth:`get_defaults <.ArgumentParser.get_defaults>` (``default_config_files`` argument), -``action="config"``, :class:`.ActionJsonSchema`, :class:`.ActionJsonnet` and -:class:`.ActionParser`. This means that a tool that can receive a config file -via ``action="config"`` is able to get the content from a URL, thus something -like the following would work: +``action="config"``, :py:meth:`.FromConfigMixin.from_config`, +:class:`.ActionJsonSchema`, :class:`.ActionJsonnet` and :class:`.ActionParser`. +This means that a tool that can receive a config file via ``action="config"`` is +able to get the content from a URL, thus something like the following would +work: .. code-block:: bash diff --git a/jsonargparse/_from_config.py b/jsonargparse/_from_config.py index 6aafc45b..34791cc5 100644 --- a/jsonargparse/_from_config.py +++ b/jsonargparse/_from_config.py @@ -8,9 +8,10 @@ from ._core import ArgumentParser from ._loaders_dumpers import get_loader_exceptions, load_value from ._optionals import _get_config_read_mode +from ._paths import change_to_path_dir from ._required import clear_required, iter_required_keys from ._typehints import is_subclass_spec, resolve_class_path_by_name -from ._util import import_object +from ._util import import_object, load_config_path_context __all__ = ["FromConfigMixin"] @@ -30,7 +31,9 @@ class FromConfigMixin: defaults. 2. Adds a ``from_config`` ``@classmethod``, that instantiates the class - based on a config file or dict. + based on a config file or dict. If + ``config_read_mode_fsspec_enabled=True`` is set, then config paths + can be URLs. Attributes: __from_config_init_defaults__: Optional path to a config file for @@ -61,12 +64,17 @@ def from_config(cls: type[T], config: str | PathLike | dict) -> T: def _parse_class_kwargs_from_config(cls: type[T], config: str | PathLike | dict, **kwargs) -> tuple[dict, type[T]]: """Parse the init kwargs for ``cls`` from a config file or dict.""" parser = ArgumentParser(exit_on_error=False, **kwargs) + cfg_path = None if not isinstance(config, dict): from .typing import Path cfg_path = Path(config, mode=_get_config_read_mode()) - cfg_str = cfg_path.read_text() - with parser_context(load_value_mode=parser.parser_mode): + with ( + load_config_path_context(cfg_path), + change_to_path_dir(cfg_path), + parser_context(load_value_mode=parser.parser_mode), + ): + cfg_str = cfg_path.read_text() try: config = load_value(cfg_str, path=str(config)) except get_loader_exceptions() as ex: @@ -86,7 +94,8 @@ def _parse_class_kwargs_from_config(cls: type[T], config: str | PathLike | dict, parser.add_class_arguments(cls) for required in iter_required_keys(parser): clear_required(parser, required) - cfg = parser.parse_object(config, defaults=False) + with load_config_path_context(cfg_path), change_to_path_dir(cfg_path): + cfg = parser.parse_object(config, defaults=False) return parser.instantiate(cfg).as_dict(), cls diff --git a/jsonargparse_tests/test_from_config.py b/jsonargparse_tests/test_from_config.py index 737c0812..e087152d 100644 --- a/jsonargparse_tests/test_from_config.py +++ b/jsonargparse_tests/test_from_config.py @@ -2,9 +2,21 @@ import pytest -from jsonargparse import FromConfigMixin +from jsonargparse import FromConfigMixin, set_parsing_settings +from jsonargparse._optionals import fsspec_support from jsonargparse._paths import PathError -from jsonargparse_tests.conftest import json_or_yaml_dump, skip_if_no_pyyaml, skip_if_omegaconf_unavailable +from jsonargparse.typing import path_type +from jsonargparse_tests.conftest import ( + json_or_yaml_dump, + skip_if_fsspec_unavailable, + skip_if_no_pyyaml, + skip_if_omegaconf_unavailable, +) + +if fsspec_support: + import fsspec + +Path_fsr = path_type("fsr") # __init__ defaults override tests @@ -198,6 +210,29 @@ def test_from_config_method_path(tmp_cwd): assert instance.parent_param == "value_from_file" +@skip_if_fsspec_unavailable +def test_from_config_method_path_fsspec_relative_paths(): + class FromConfigMethodFsspecRelativePath(FromConfigMixin): + def __init__(self, file: Path_fsr): + self.file = file + + config_path = "memory://from_config/config.yaml" + file_path = "memory://from_config/file.txt" + with fsspec.open(config_path, "w") as f: + f.write(json_or_yaml_dump({"file": "file.txt"})) + with fsspec.open(file_path, "w") as f: + f.write("value_from_fsspec") + + set_parsing_settings(config_read_mode_fsspec_enabled=True) + try: + instance = FromConfigMethodFsspecRelativePath.from_config(config_path) + finally: + set_parsing_settings(config_read_mode_fsspec_enabled=False) + + assert instance.file.absolute == file_path + assert instance.file.read_text() == "value_from_fsspec" + + def test_from_config_method_path_does_not_exist(tmp_cwd): config_path = tmp_cwd / "config.yaml"