|
194 | 194 | T = TypeVar("T") |
195 | 195 |
|
196 | 196 |
|
| 197 | +@lru_cache |
| 198 | +def _cached_resolve_s3_region(bucket: str) -> str: |
| 199 | + from pyarrow.fs import resolve_s3_region |
| 200 | + |
| 201 | + return resolve_s3_region(bucket=bucket) |
| 202 | + |
| 203 | + |
197 | 204 | class UnsupportedPyArrowTypeException(Exception): |
198 | 205 | """Cannot convert PyArrow type to corresponding Iceberg type.""" |
199 | 206 |
|
@@ -414,19 +421,19 @@ def _initialize_oss_fs(self) -> FileSystem: |
414 | 421 | return S3FileSystem(**client_kwargs) |
415 | 422 |
|
416 | 423 | def _initialize_s3_fs(self, netloc: Optional[str]) -> FileSystem: |
417 | | - from pyarrow.fs import S3FileSystem, resolve_s3_region |
| 424 | + from pyarrow.fs import S3FileSystem |
418 | 425 |
|
419 | 426 | # Resolve region from netloc(bucket), fallback to user-provided region |
420 | 427 | provided_region = get_first_property_value(self.properties, S3_REGION, AWS_REGION) |
421 | 428 |
|
422 | 429 | try: |
423 | | - bucket_region = resolve_s3_region(bucket=netloc) |
| 430 | + bucket_region = _cached_resolve_s3_region(bucket=netloc) |
424 | 431 | except (OSError, TypeError): |
425 | 432 | bucket_region = None |
426 | 433 | logger.warning(f"Unable to resolve region for bucket {netloc}, using default region {provided_region}") |
427 | 434 |
|
428 | 435 | bucket_region = bucket_region or provided_region |
429 | | - if bucket_region != provided_region: |
| 436 | + if provided_region is not None and bucket_region != provided_region: |
430 | 437 | logger.warning( |
431 | 438 | f"PyArrow FileIO overriding S3 bucket region for bucket {netloc}: " |
432 | 439 | f"provided region {provided_region}, actual region {bucket_region}" |
|
0 commit comments