add get_str config support and get val from config

Author: Tom McCormick

pyiceberg/io/pyarrow.py

@@ -381,13 +381,15 @@ def to_input_file(self) -> PyArrowFile:
 
 class PyArrowFileIO(FileIO):
     fs_by_scheme: Callable[[str, Optional[str]], FileSystem]
+    config: Config
 
     def __init__(self, properties: Properties = EMPTY_DICT):
         self.fs_by_scheme: Callable[[str, Optional[str]], FileSystem] = lru_cache(self._initialize_fs)
+        self.config = Config()
         super().__init__(properties=properties)
 
     @staticmethod
-    def parse_location(location: str) -> Tuple[str, str, str]:
+    def parse_location(location: str, config: Config) -> Tuple[str, str, str]:
         """Return (scheme, netloc, path) for the given location.
 
         Uses environment variables DEFAULT_SCHEME and DEFAULT_NETLOC
@@ -396,8 +398,8 @@ def parse_location(location: str) -> Tuple[str, str, str]:
         uri = urlparse(location)
 
         # Load defaults from environment
-        default_scheme = os.getenv("DEFAULT_SCHEME", "file")
-        default_netloc = os.getenv("DEFAULT_NETLOC", "")
+        default_scheme = config.get_str("default-scheme") or "file"
+        default_netloc = config.get_str("default-netloc") or ""
 
         # Apply logic
         scheme = uri.scheme or default_scheme
@@ -599,7 +601,7 @@ def new_input(self, location: str) -> PyArrowFile:
         Returns:
             PyArrowFile: A PyArrowFile instance for the given location.
         """
-        scheme, netloc, path = self.parse_location(location)
+        scheme, netloc, path = self.parse_location(location, self.config)
         return PyArrowFile(
             fs=self.fs_by_scheme(scheme, netloc),
             location=location,
@@ -616,7 +618,7 @@ def new_output(self, location: str) -> PyArrowFile:
         Returns:
             PyArrowFile: A PyArrowFile instance for the given location.
         """
-        scheme, netloc, path = self.parse_location(location)
+        scheme, netloc, path = self.parse_location(location, self.config)
         return PyArrowFile(
             fs=self.fs_by_scheme(scheme, netloc),
             location=location,
@@ -637,7 +639,7 @@ def delete(self, location: Union[str, InputFile, OutputFile]) -> None:
                 an AWS error code 15.
         """
         str_location = location.location if isinstance(location, (InputFile, OutputFile)) else location
-        scheme, netloc, path = self.parse_location(str_location)
+        scheme, netloc, path = self.parse_location(str_location, self.config)
         fs = self.fs_by_scheme(scheme, netloc)
 
         try:

pyiceberg/utils/config.py

@@ -180,3 +180,8 @@ def get_bool(self, key: str) -> Optional[bool]:
             except ValueError as err:
                 raise ValueError(f"{key} should be a boolean or left unset. Current value: {val}") from err
         return None
+
+    def get_str(self, key: str) -> Optional[str]:
+        if (val := self.config.get(key)) is not None:
+            return val
+        return None

tests/io/test_pyarrow.py

@@ -106,6 +106,7 @@
     TimestamptzType,
     TimeType,
 )
+from pyiceberg.utils.config import Config
 from tests.catalog.test_base import InMemoryCatalog
 from tests.conftest import UNIFIED_AWS_SESSION_PROPERTIES
 
@@ -2024,7 +2025,7 @@ def test_writing_avro_file_adls(generated_manifest_entry_file: str, pyarrow_file
 
 def test_parse_location() -> None:
     def check_results(location: str, expected_schema: str, expected_netloc: str, expected_uri: str) -> None:
-        schema, netloc, uri = PyArrowFileIO.parse_location(location)
+        schema, netloc, uri = PyArrowFileIO.parse_location(location, Config())
         assert schema == expected_schema
         assert netloc == expected_netloc
         assert uri == expected_uri
@@ -2647,32 +2648,32 @@ def test_parse_location_environment_defaults() -> None:
     from pyiceberg.io.pyarrow import PyArrowFileIO
 
     # Test with default environment (no env vars set)
-    scheme, netloc, path = PyArrowFileIO.parse_location("/foo/bar")
+    scheme, netloc, path = PyArrowFileIO.parse_location("/foo/bar", Config())
     assert scheme == "file"
     assert netloc == ""
     assert path == "/foo/bar"
 
     try:
         # Test with environment variables set
-        os.environ["DEFAULT_SCHEME"] = "scheme"
-        os.environ["DEFAULT_NETLOC"] = "netloc:8000"
+        os.environ["PYICEBERG_DEFAULT_SCHEME"] = "scheme"
+        os.environ["PYICEBERG_DEFAULT_NETLOC"] = "netloc:8000"
 
-        scheme, netloc, path = PyArrowFileIO.parse_location("/foo/bar")
+        scheme, netloc, path = PyArrowFileIO.parse_location("/foo/bar", Config())
         assert scheme == "scheme"
         assert netloc == "netloc:8000"
         assert path == "netloc:8000/foo/bar"
 
         # Set environment variables
-        os.environ["DEFAULT_SCHEME"] = "hdfs"
-        os.environ["DEFAULT_NETLOC"] = "netloc:8000"
+        os.environ["PYICEBERG_DEFAULT_SCHEME"] = "hdfs"
+        os.environ["PYICEBERG_DEFAULT_NETLOC"] = "netloc:8000"
 
-        scheme, netloc, path = PyArrowFileIO.parse_location("/foo/bar")
+        scheme, netloc, path = PyArrowFileIO.parse_location("/foo/bar", Config())
         assert scheme == "hdfs"
         assert netloc == "netloc:8000"
         assert path == "/foo/bar"
     finally:
         # Clean up environment variables
-        if "DEFAULT_SCHEME" in os.environ:
-            del os.environ["DEFAULT_SCHEME"]
-        if "DEFAULT_NETLOC" in os.environ:
-            del os.environ["DEFAULT_NETLOC"]
+        if "PYICEBERG_DEFAULT_SCHEME" in os.environ:
+            del os.environ["PYICEBERG_DEFAULT_SCHEME"]
+        if "PYICEBERG_DEFAULT_NETLOC" in os.environ:
+            del os.environ["PYICEBERG_DEFAULT_NETLOC"]