testcode

Author: jx2lee

tests/catalog/test_base.py

@@ -16,17 +16,18 @@
 #  under the License.
 # pylint:disable=redefined-outer-name
 
-
 from collections.abc import Generator
 from pathlib import PosixPath
 
 import pytest
+from pytest_mock import MockFixture
 
-from pyiceberg.catalog import Catalog, load_catalog
+from pyiceberg.catalog import Catalog, _get_env_config, load_catalog
 from pyiceberg.catalog.memory import InMemoryCatalog
 from pyiceberg.io import WAREHOUSE
 from pyiceberg.schema import Schema
 from pyiceberg.types import NestedField, StringType
+from pyiceberg.utils.config import Config
 
 
 @pytest.fixture
@@ -64,6 +65,20 @@ def test_load_catalog_has_type_and_impl() -> None:
     )
 
 
+def test_get_env_config_is_lazy_and_cached(mocker: MockFixture) -> None:
+    original_config = _get_env_config()
+    _get_env_config.cache_clear()
+    config = Config({"catalog": {"test": {"type": "in-memory"}}})
+    load_mock = mocker.patch("pyiceberg.catalog.Config.load", return_value=config)
+    assert _get_env_config() is config
+    assert _get_env_config() is config
+    load_mock.assert_called_once()
+
+    _get_env_config.cache_clear()
+    mocker.patch("pyiceberg.catalog.Config.load", return_value=original_config)
+    assert _get_env_config() is original_config
+
+
 def test_catalog_repr(catalog: InMemoryCatalog) -> None:
     s = repr(catalog)
     assert s == "test.in_memory.catalog (<class 'pyiceberg.catalog.memory.InMemoryCatalog'>)"

tests/catalog/test_rest.py

@@ -1887,18 +1887,18 @@ def test_catalog_from_environment_variables(catalog_config_mock: mock.Mock, rest
 
 
 @mock.patch.dict(os.environ, EXAMPLE_ENV)
-@mock.patch("pyiceberg.catalog._ENV_CONFIG.get_catalog_config")
-def test_catalog_from_environment_variables_override(catalog_config_mock: mock.Mock, rest_mock: Mocker) -> None:
+def test_catalog_from_environment_variables_override(rest_mock: Mocker) -> None:
     rest_mock.get(
         "https://other-service.io/api/v1/config",
         json={"defaults": {}, "overrides": {}},
         status_code=200,
     )
     env_config: RecursiveDict = Config._from_environment_variables({})
-
-    catalog_config_mock.return_value = cast(RecursiveDict, env_config.get("catalog")).get("production")
-    catalog = cast(RestCatalog, load_catalog("production", uri="https://other-service.io/api"))
-    assert catalog.uri == "https://other-service.io/api"
+    mock_env_config = mock.Mock()
+    mock_env_config.get_catalog_config.return_value = cast(RecursiveDict, env_config.get("catalog")).get("production")
+    with mock.patch("pyiceberg.catalog._get_env_config", return_value=mock_env_config):
+        catalog = cast(RestCatalog, load_catalog("production", uri="https://other-service.io/api"))
+        assert catalog.uri == "https://other-service.io/api"
 
 
 def test_catalog_from_parameters_empty_env(rest_mock: Mocker) -> None:

tests/cli/test_console.py

@@ -39,19 +39,19 @@
 def test_missing_uri(mocker: MockFixture, empty_home_dir_path: str) -> None:
     # mock to prevent parsing ~/.pyiceberg.yaml or {PYICEBERG_HOME}/.pyiceberg.yaml
     mocker.patch.dict(os.environ, values={"HOME": empty_home_dir_path, "PYICEBERG_HOME": empty_home_dir_path})
-    mocker.patch("pyiceberg.catalog._ENV_CONFIG", return_value=Config())
+    mocker.patch("pyiceberg.catalog._get_env_config", return_value=Config())
 
     runner = CliRunner()
     result = runner.invoke(run, ["list"])
 
     assert result.exit_code == 1
-    assert result.output == "Could not initialize catalog with the following properties: {}\n"
+    assert "URI missing, please provide using --uri" in result.output
 
 
 def test_hive_catalog_missing_uri_shows_helpful_error(mocker: MockFixture) -> None:
     mock_env_config = mocker.MagicMock()
     mock_env_config.get_catalog_config.return_value = {"type": "hive"}
-    mocker.patch("pyiceberg.catalog._ENV_CONFIG", mock_env_config)
+    mocker.patch("pyiceberg.catalog._get_env_config", return_value=mock_env_config)
 
     runner = CliRunner()
     result = runner.invoke(run, ["--catalog", "my_hive_catalog", "list"])

tests/conftest.py

@@ -122,7 +122,8 @@ def _isolate_pyiceberg_config() -> None:
     from pyiceberg.utils.config import Config
 
     with mock.patch.object(Config, "_from_configuration_files", return_value=None):
-        _catalog_module._ENV_CONFIG = Config()
+        _catalog_module._get_env_config.cache_clear()
+        _catalog_module._get_env_config()
 
 
 def pytest_addoption(parser: pytest.Parser) -> None:

tests/utils/test_config.py

@@ -28,18 +28,30 @@
 
 
 def test_config() -> None:
-    """To check if all the file lookups go well without any mocking"""
+    """Config() should be a pure empty container and perform no implicit IO."""
     assert Config()
+    assert Config().config == {}
+
+
+def test_config_does_not_load_implicitly() -> None:
+    with (
+        mock.patch.object(Config, "_from_configuration_files") as from_files_mock,
+        mock.patch.object(Config, "_from_environment_variables") as from_env_mock,
+    ):
+        Config()
+
+    from_files_mock.assert_not_called()
+    from_env_mock.assert_not_called()
 
 
 @mock.patch.dict(os.environ, EXAMPLE_ENV)
 def test_from_environment_variables() -> None:
-    assert Config().get_catalog_config("production") == {"uri": "https://service.io/api"}
+    assert Config.load().get_catalog_config("production") == {"uri": "https://service.io/api"}
 
 
 @mock.patch.dict(os.environ, EXAMPLE_ENV)
 def test_from_environment_variables_uppercase() -> None:
-    assert Config().get_catalog_config("PRODUCTION") == {"uri": "https://service.io/api"}
+    assert Config.load().get_catalog_config("PRODUCTION") == {"uri": "https://service.io/api"}
 
 
 @mock.patch.dict(
@@ -50,7 +62,7 @@ def test_from_environment_variables_uppercase() -> None:
     },
 )
 def test_fix_nested_objects_from_environment_variables() -> None:
-    assert Config().get_catalog_config("PRODUCTION") == {
+    assert Config.load().get_catalog_config("PRODUCTION") == {
         "s3.region": "eu-north-1",
         "s3.access-key-id": "username",
     }
@@ -59,7 +71,7 @@ def test_fix_nested_objects_from_environment_variables() -> None:
 @mock.patch.dict(os.environ, EXAMPLE_ENV)
 @mock.patch.dict(os.environ, {"PYICEBERG_CATALOG__DEVELOPMENT__URI": "https://dev.service.io/api"})
 def test_list_all_known_catalogs() -> None:
-    catalogs = Config().get_known_catalogs()
+    catalogs = Config.load().get_known_catalogs()
     assert "production" in catalogs
     assert "development" in catalogs
 
@@ -71,7 +83,7 @@ def test_from_configuration_files(tmp_path_factory: pytest.TempPathFactory) -> N
         file.write(yaml_str)
 
     os.environ["PYICEBERG_HOME"] = config_path
-    assert Config().get_catalog_config("production") == {"uri": "https://service.io/api"}
+    assert Config.load().get_catalog_config("production") == {"uri": "https://service.io/api"}
 
 
 def test_lowercase_dictionary_keys() -> None:
@@ -95,13 +107,13 @@ def test_from_configuration_files_get_typed_value(tmp_path_factory: pytest.TempP
 
     os.environ["PYICEBERG_HOME"] = config_path
     with pytest.raises(ValueError):
-        Config().get_bool("max-workers")
+        Config.load().get_bool("max-workers")
 
     with pytest.raises(ValueError):
-        Config().get_int("legacy-current-snapshot-id")
+        Config.load().get_int("legacy-current-snapshot-id")
 
-    assert Config().get_bool("legacy-current-snapshot-id")
-    assert Config().get_int("max-workers") == 4
+    assert Config.load().get_bool("legacy-current-snapshot-id")
+    assert Config.load().get_int("max-workers") == 4
 
 
 @pytest.mark.parametrize(
@@ -183,3 +195,14 @@ def create_config_file(path: str, uri: str | None) -> None:
     assert (
         result["catalog"]["default"]["uri"] if result else None  # type: ignore
     ) == expected_result, f"Unexpected configuration result. Expected: {expected_result}, Actual: {result}"
+
+
+def test_load_reads_file_and_environment_once(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setenv("PYICEBERG_CATALOG__PRODUCTION__URI", "https://env.service.io/api")
+    with mock.patch.object(
+        Config, "_from_configuration_files", return_value={"catalog": {"production": {"type": "rest"}}}
+    ) as files_mock:
+        config = Config.load()
+
+    files_mock.assert_called_once()
+    assert config.get_catalog_config("production") == {"type": "rest", "uri": "https://env.service.io/api"}