Skip to content

Commit d0d5e84

Browse files
Fix lazy GCS chunk size caching and lineage fallback without gcp deps
1 parent b4f6615 commit d0d5e84

1 file changed

Lines changed: 8 additions & 9 deletions

File tree

sdks/python/apache_beam/io/gcp/gcsfilesystem.py

Lines changed: 8 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -50,19 +50,18 @@ def _get_gcsio(self):
5050
'GCSFileSystem requires apache-beam[gcp]. '
5151
'Install it with: pip install apache-beam[gcp]')
5252

53-
_CHUNK_SIZE = None
54-
55-
@property
56-
def CHUNK_SIZE(self):
57-
if self._CHUNK_SIZE is None:
58-
self._CHUNK_SIZE = self._get_gcsio().MAX_BATCH_OPERATION_SIZE
59-
return self._CHUNK_SIZE # Chuck size in batch operations
60-
6153
GCS_PREFIX = 'gs://'
6254

6355
def __init__(self, pipeline_options):
6456
super().__init__(pipeline_options)
6557
self._pipeline_options = pipeline_options
58+
self._chunk_size = None
59+
60+
@property
61+
def CHUNK_SIZE(self):
62+
if self._chunk_size is None:
63+
self._chunk_size = self._get_gcsio().MAX_BATCH_OPERATION_SIZE
64+
return self._chunk_size
6665

6766
@classmethod
6867
def scheme(cls):
@@ -387,7 +386,7 @@ def report_lineage(self, path, lineage):
387386
try:
388387
gcsio = self._get_gcsio()
389388
components = gcsio.parse_gcs_path(path, object_optional=True)
390-
except ValueError:
389+
except (ImportError, ValueError):
391390
# report lineage is fail-safe
392391
traceback.print_exc()
393392
return

0 commit comments

Comments
 (0)