use catalog env configs and update to use default scheme and netloc from properties

Author: Tom McCormick

pyiceberg/io/pyarrow.py

@@ -392,7 +392,7 @@ def __init__(self, properties: Properties = EMPTY_DICT):
         super().__init__(properties=properties)
 
     @staticmethod
-    def parse_location(location: str) -> Tuple[str, str, str]:
+    def parse_location(location: str, properties: Properties=EMPTY_DICT) -> Tuple[str, str, str]:
         """Return (scheme, netloc, path) for the given location.
 
         Uses environment variables DEFAULT_SCHEME and DEFAULT_NETLOC
@@ -401,8 +401,8 @@ def parse_location(location: str) -> Tuple[str, str, str]:
         uri = urlparse(location)
 
         # Load defaults from environment
-        default_scheme = os.getenv("DEFAULT_SCHEME", "file")
-        default_netloc = os.getenv("DEFAULT_NETLOC", "")
+        default_scheme = properties.get("DEFAULT_SCHEME", "file")
+        default_netloc = properties.get("DEFAULT_NETLOC", "")
 
         # Apply logic
         scheme = uri.scheme or default_scheme
@@ -629,7 +629,7 @@ def new_input(self, location: str) -> PyArrowFile:
         Returns:
             PyArrowFile: A PyArrowFile instance for the given location.
         """
-        scheme, netloc, path = self.parse_location(location)
+        scheme, netloc, path = self.parse_location(location, self.properties)
         return PyArrowFile(
             fs=self.fs_by_scheme(scheme, netloc),
             location=location,
@@ -646,7 +646,7 @@ def new_output(self, location: str) -> PyArrowFile:
         Returns:
             PyArrowFile: A PyArrowFile instance for the given location.
         """
-        scheme, netloc, path = self.parse_location(location)
+        scheme, netloc, path = self.parse_location(location, self.properties)
         return PyArrowFile(
             fs=self.fs_by_scheme(scheme, netloc),
             location=location,
@@ -667,7 +667,7 @@ def delete(self, location: Union[str, InputFile, OutputFile]) -> None:
                 an AWS error code 15.
         """
         str_location = location.location if isinstance(location, (InputFile, OutputFile)) else location
-        scheme, netloc, path = self.parse_location(str_location)
+        scheme, netloc, path = self.parse_location(str_location, self.properties)
         fs = self.fs_by_scheme(scheme, netloc)
 
         try:

tests/io/test_pyarrow.py

@@ -2798,27 +2798,21 @@ def test_parse_location_environment_defaults():
     assert netloc == ""
     assert path == "/foo/bar"
 
-    try:
-        # Test with environment variables set
-        os.environ["DEFAULT_SCHEME"] = "scheme"
-        os.environ["DEFAULT_NETLOC"] = "netloc:8000"
-
-        scheme, netloc, path = PyArrowFileIO.parse_location("/foo/bar")
-        assert scheme == "scheme"
-        assert netloc == "netloc:8000"
-        assert path == "netloc:8000/foo/bar"
-
-        # Set environment variables
-        os.environ["DEFAULT_SCHEME"] = "hdfs"
-        os.environ["DEFAULT_NETLOC"] = "netloc:8000"
-
-        scheme, netloc, path = PyArrowFileIO.parse_location("/foo/bar")
-        assert scheme == "hdfs"
-        assert netloc == "netloc:8000"
-        assert path == "/foo/bar"
-    finally:
-        # Clean up environment variables
-        if "DEFAULT_SCHEME" in os.environ:
-            del os.environ["DEFAULT_SCHEME"]
-        if "DEFAULT_NETLOC" in os.environ:
-            del os.environ["DEFAULT_NETLOC"]
+    # Test with properties set
+    properties = dict()
+    properties["DEFAULT_SCHEME"] = "scheme"
+    properties["DEFAULT_NETLOC"] = "netloc:8000"
+
+    scheme, netloc, path = PyArrowFileIO.parse_location("/foo/bar", properties=properties)
+    assert scheme == "scheme"
+    assert netloc == "netloc:8000"
+    assert path == "netloc:8000/foo/bar"
+
+    # Set properties
+    properties["DEFAULT_SCHEME"] = "hdfs"
+    properties["DEFAULT_NETLOC"] = "netloc:8000"
+
+    scheme, netloc, path = PyArrowFileIO.parse_location("/foo/bar", properties=properties)
+    assert scheme == "hdfs"
+    assert netloc == "netloc:8000"
+    assert path == "/foo/bar"