Merge pull request #12 from Schokuroff/fix_py_io

added param for choosing  pyiceberg.io.

Author: nicosuave

src/dlt_iceberg/destination_client.py

@@ -92,6 +92,10 @@ class IcebergRestConfiguration(DestinationClientConfiguration):
     s3_secret_access_key: Optional[str] = None
     s3_region: Optional[str] = None
 
+    py_io_impl: Optional[str] = None
+    # https://github.com/apache/iceberg-python/blob/main/pyiceberg/io/pyarrow.py
+    # https://github.com/apache/iceberg-python/blob/main/pyiceberg/io/fsspec.py
+
     # Retry configuration
     max_retries: int = 5
     retry_backoff_base: float = 2.0
@@ -115,6 +119,7 @@ class IcebergRestConfiguration(DestinationClientConfiguration):
     hard_delete_column: Optional[str] = "_dlt_deleted_at"
 
 
+
 class IcebergRestLoadJob(RunnableLoadJob):
     """
     Load job that processes a single parquet file.
@@ -668,6 +673,12 @@ def _get_catalog(self):
         if self.config.s3_region:
             catalog_config["s3.region"] = self.config.s3_region
 
+        if self.config.py_io_impl:
+            catalog_config["py-io-impl"] = self.config.py_io_impl
+        # For example:
+        # catalog_config["py-io-impl"] = "pyiceberg.io.fsspec.FsspecFileIO"
+        
+        
         logger.info(
             f"Creating catalog connection (type={catalog_type}, uri={self.config.catalog_uri})"
         )