Introduce is_real_gcs

Author: Yonghui-Lee

gcsfs/extended_gcsfs.py

@@ -196,7 +196,8 @@ async def _lookup_bucket_type(self, bucket):
             return self._storage_layout_cache[bucket]
         bucket_type = await self._get_bucket_type(bucket)
         # Dont cache UNKNOWN type by default.
-        # Allow caching it in tests (via config) to avoid slow repeated failing lookups on emulator.
+        # Caching can be enabled (via cache_unknown_buckets=True) to avoid repeated slow
+        # lookups on emulators or when users lack permissions for the Storage Control API.
         if bucket_type == BucketType.UNKNOWN and not self._cache_unknown_buckets:
             return bucket_type
         self._storage_layout_cache[bucket] = bucket_type

gcsfs/tests/conftest.py

@@ -25,6 +25,7 @@
     TEST_VERSIONED_BUCKET,
     TEST_ZONAL_BUCKET,
 )
+from gcsfs.tests.utils import is_real_gcs
 
 files = {
     "test/accounts.1.json": (
@@ -79,7 +80,7 @@
 def _avoid_adc_timeout(monkeypatch):
     """Avoid slow ADC lookups and Metadata Server requests in tests."""
     # Do not apply if tests are explicitly running against real GCS
-    if os.environ.get("STORAGE_EMULATOR_HOST") == "https://storage.googleapis.com":
+    if is_real_gcs():
         yield
         return
 
@@ -103,7 +104,7 @@ def stop_docker(container):
 @pytest.fixture(scope="session")
 def docker_gcs():
     if "STORAGE_EMULATOR_HOST" in os.environ:
-        if os.environ["STORAGE_EMULATOR_HOST"] != "https://storage.googleapis.com":
+        if not is_real_gcs():
             params["token"] = "anon"
         # assume using real API or otherwise have a server already set up
         yield os.getenv("STORAGE_EMULATOR_HOST")
@@ -140,9 +141,8 @@ def gcs_factory(docker_gcs):
     def factory(**kwargs):
         GCSFileSystem.clear_instance_cache()
         fs = fsspec.filesystem("gcs", **params, **kwargs)
-        # Enable caching of UNKNOWN bucket types if we are on emulator
-        is_emulator = params.get("endpoint_url") != "https://storage.googleapis.com"
-        if is_emulator and isinstance(fs, ExtendedGcsFileSystem):
+        # Enable caching of UNKNOWN bucket types if not using real GCS
+        if not is_real_gcs() and isinstance(fs, ExtendedGcsFileSystem):
             fs._cache_unknown_buckets = True
         return fs
 
@@ -308,9 +308,6 @@ def final_cleanup(gcs_factory, buckets_to_delete):
 def gcs_versioned(gcs_factory, buckets_to_delete):
     gcs = gcs_factory()
     gcs.version_aware = True
-    is_real_gcs = (
-        os.environ.get("STORAGE_EMULATOR_HOST") == "https://storage.googleapis.com"
-    )
     try:  # ensure we're empty.
         # The versioned bucket might be created by `is_versioning_enabled`
         # in test_core_versioned.py. We must register it for cleanup only if
@@ -324,7 +321,7 @@ def gcs_versioned(gcs_factory, buckets_to_delete):
                 buckets_to_delete.add(TEST_VERSIONED_BUCKET)
         except ImportError:
             pass  # test_core_versioned is not being run
-        if is_real_gcs:
+        if is_real_gcs():
             cleanup_versioned_bucket(gcs, TEST_VERSIONED_BUCKET)
         else:
             # For emulators, we delete and recreate the bucket for a clean state
@@ -339,7 +336,7 @@ def gcs_versioned(gcs_factory, buckets_to_delete):
     finally:
         # Ensure the bucket is empty after the test.
         try:
-            if is_real_gcs:
+            if is_real_gcs():
                 cleanup_versioned_bucket(gcs, TEST_VERSIONED_BUCKET)
         except Exception as e:
             logging.warning(
@@ -385,13 +382,9 @@ def cleanup_versioned_bucket(gcs, bucket_name, prefix=None):
 
 
 def _create_extended_gcsfs(gcs_factory, buckets_to_delete, populate_bucket, **kwargs):
-    is_real_gcs = (
-        os.environ.get("STORAGE_EMULATOR_HOST") == "https://storage.googleapis.com"
-    )
-
     extended_gcsfs = gcs_factory(**kwargs)
     # Only create/delete/populate the bucket if we are NOT using the real GCS endpoint.
-    if not is_real_gcs:
+    if not is_real_gcs():
         if not extended_gcsfs.exists(TEST_ZONAL_BUCKET):
             extended_gcsfs.mkdir(TEST_ZONAL_BUCKET)
             buckets_to_delete.add(TEST_ZONAL_BUCKET)
@@ -451,14 +444,14 @@ def gcs_hns(gcs_factory, buckets_to_delete):
 def zonal_write_mocks():
     """A fixture for mocking Zonal bucket write functionality."""
 
-    if os.environ.get("STORAGE_EMULATOR_HOST") == "https://storage.googleapis.com":
+    if is_real_gcs():
         yield None
         return
 
-    patch_target_get_bucket_type = (
+    patch_target_lookup_bucket_type = (
         "gcsfs.extended_gcsfs.ExtendedGcsFileSystem._lookup_bucket_type"
     )
-    patch_target_sync_get_bucket_type = (
+    patch_target_sync_lookup_bucket_type = (
         "gcsfs.extended_gcsfs.ExtendedGcsFileSystem._sync_lookup_bucket_type"
     )
     patch_target_init_aaow = "gcsfs.zb_hns_utils.init_aaow"
@@ -490,12 +483,12 @@ async def finalize_side_effect():
 
     with (
         mock.patch(
-            patch_target_get_bucket_type,
+            patch_target_lookup_bucket_type,
             new_callable=mock.AsyncMock,
             return_value=BucketType.ZONAL_HIERARCHICAL,
         ),
         mock.patch(
-            patch_target_sync_get_bucket_type,
+            patch_target_sync_lookup_bucket_type,
             return_value=BucketType.ZONAL_HIERARCHICAL,
         ),
         mock.patch(patch_target_gcsfs_info, mock_gcsfs_info),

gcsfs/tests/integration/test_async_gcsfs.py

@@ -20,6 +20,7 @@
 
 from gcsfs.extended_gcsfs import ExtendedGcsFileSystem
 from gcsfs.tests.settings import TEST_HNS_BUCKET
+from gcsfs.tests.utils import is_real_gcs
 
 REQUIRED_ENV_VAR = "GCSFS_EXPERIMENTAL_ZB_HNS_SUPPORT"
 
@@ -34,7 +35,7 @@
         reason=f"Skipping tests: {REQUIRED_ENV_VAR} env variable is not set",
     ),
     pytest.mark.skipif(
-        os.environ.get("STORAGE_EMULATOR_HOST") != "https://storage.googleapis.com",
+        not is_real_gcs(),
         reason="Skipping tests on emulator, requires real GCS.",
     ),
 ]

gcsfs/tests/integration/test_extended_hns.py

@@ -19,6 +19,7 @@
 
 from gcsfs.extended_gcsfs import BucketType, ExtendedGcsFileSystem
 from gcsfs.tests.settings import TEST_HNS_BUCKET, TEST_PROJECT
+from gcsfs.tests.utils import is_real_gcs
 
 should_run_hns = os.getenv("GCSFS_EXPERIMENTAL_ZB_HNS_SUPPORT", "false").lower() in (
     "true",
@@ -27,8 +28,7 @@
 
 # Skip these tests if not running against a real GCS backend or if experimentation flag is not set.
 pytestmark = pytest.mark.skipif(
-    os.environ.get("STORAGE_EMULATOR_HOST") != "https://storage.googleapis.com"
-    or not should_run_hns,
+    not is_real_gcs() or not should_run_hns,
     reason="This test class is for real GCS HNS buckets only and requires experimental flag.",
 )
 

gcsfs/tests/test_core_versioned.py

@@ -7,6 +7,7 @@
 
 from gcsfs import GCSFileSystem
 from gcsfs.tests.settings import TEST_VERSIONED_BUCKET
+from gcsfs.tests.utils import is_real_gcs
 
 a = TEST_VERSIONED_BUCKET + "/tmp/test/a"
 b = TEST_VERSIONED_BUCKET + "/tmp/test/b"
@@ -22,7 +23,7 @@ def is_versioning_enabled():
     """
     # Don't skip when using an emulator, as we create the versioned bucket ourselves.
     global _VERSIONED_BUCKET_CREATED_BY_TESTS
-    if os.environ.get("STORAGE_EMULATOR_HOST") != "https://storage.googleapis.com":
+    if not is_real_gcs():
         return True, ""
     try:
         gcs = GCSFileSystem(project=os.getenv("GCSFS_TEST_PROJECT", "project"))

gcsfs/tests/test_extended_gcsfs.py

@@ -32,7 +32,7 @@
 )
 from gcsfs.tests.conftest import csv_files, files, text_files
 from gcsfs.tests.settings import TEST_BUCKET, TEST_ZONAL_BUCKET
-from gcsfs.tests.utils import tempdir, tmpfile
+from gcsfs.tests.utils import is_real_gcs, tempdir, tmpfile
 from gcsfs.zb_hns_utils import MRDPool
 
 file = "test/accounts.1.json"
@@ -68,10 +68,7 @@ def gcs_bucket_mocks():
     @contextlib.contextmanager
     def _gcs_bucket_mocks_factory(file_data, bucket_type_val):
         """Creates mocks for a given file content and bucket type."""
-        is_real_gcs = (
-            os.environ.get("STORAGE_EMULATOR_HOST") == "https://storage.googleapis.com"
-        )
-        if is_real_gcs:
+        if is_real_gcs():
             yield None
             return
         patch_target_lookup_bucket_type = (
@@ -1125,18 +1122,17 @@ async def test_cp_file_not_implemented_error(
     short_uuid = str(uuid.uuid4())[:8]
     source_path = f"{source_bucket}/source_{short_uuid}"
     dest_path = f"{dest_bucket}/dest_{short_uuid}"
-    is_real_gcs = os.getenv("STORAGE_EMULATOR_HOST") == "https://storage.googleapis.com"
 
     # Source file needs to exist for last case when super method is called for standard buckets
-    if is_real_gcs:
+    if is_real_gcs():
         await async_gcs._pipe_file(source_path, b"test data", finalize_on_close=True)
 
     async def mock_is_zonal(bucket):
         return bucket == TEST_ZONAL_BUCKET
 
     is_zonal_patch_cm = (
         mock.patch.object(async_gcs, "_is_zonal_bucket", side_effect=mock_is_zonal)
-        if not is_real_gcs
+        if not is_real_gcs()
         else contextlib.nullcontext()
     )
 
@@ -1151,7 +1147,7 @@ async def mock_is_zonal(bucket):
             ):
                 await async_gcs._cp_file(source_path, dest_path)
         else:  # Standard -> Standard
-            if is_real_gcs:
+            if is_real_gcs():
                 await async_gcs._cp_file(source_path, dest_path)
                 assert await async_gcs._cat(dest_path) == b"test data"
             else:

gcsfs/tests/test_extended_gcsfs_unit.py

@@ -21,7 +21,7 @@
 from gcsfs.tests.conftest import csv_files, files
 from gcsfs.tests.settings import TEST_BUCKET, TEST_ZONAL_BUCKET
 from gcsfs.tests.test_extended_gcsfs import gcs_bucket_mocks  # noqa: F401
-from gcsfs.tests.utils import tmpfile
+from gcsfs.tests.utils import is_real_gcs, tmpfile
 
 file = "test/accounts.1.json"
 file_path = f"{TEST_ZONAL_BUCKET}/{file}"
@@ -46,7 +46,7 @@
         reason=f"Skipping tests: {REQUIRED_ENV_VAR} env variable is not set",
     ),
     pytest.mark.skipif(
-        os.environ.get("STORAGE_EMULATOR_HOST") == "https://storage.googleapis.com",
+        is_real_gcs(),
         reason="Contains Unit tests using mocks, does not require testing on real GCS.",
     ),
 ]

gcsfs/tests/test_extended_hns_gcsfs.py

@@ -18,6 +18,7 @@
 from gcsfs.extended_gcsfs import BucketType, ExtendedGcsFileSystem
 from gcsfs.retry import DEFAULT_RETRY_CONFIG, HttpError
 from gcsfs.tests.settings import TEST_HNS_BUCKET
+from gcsfs.tests.utils import is_real_gcs
 
 REQUIRED_ENV_VAR = "GCSFS_EXPERIMENTAL_ZB_HNS_SUPPORT"
 
@@ -754,7 +755,7 @@ def test_hns_mkdir_nested_fails_if_create_parents_false(
             mocks["super_mkdir"].assert_not_called()
 
     @pytest.mark.skipif(
-        os.environ.get("STORAGE_EMULATOR_HOST") == "https://storage.googleapis.com",
+        is_real_gcs(),
         reason="This test is only to check that create_folder is not called for non-HNS buckets.",
     )
     def test_mkdir_non_hns_bucket_falls_back(self, gcs_hns, gcs_hns_mocks):
@@ -1591,7 +1592,7 @@ def test_hns_rmdir_success(self, gcs_hns, gcs_hns_mocks):
             mocks["super_rmdir"].assert_not_called()
 
     @pytest.mark.skipif(
-        os.environ.get("STORAGE_EMULATOR_HOST") == "https://storage.googleapis.com",
+        is_real_gcs(),
         reason="This test is only to check that delete_folder is not called in case of non-HNS buckets."
         "In real GCS on non-HNS bucket there would be no empty directories to delete.",
     )

gcsfs/tests/test_zonal_file.py

@@ -9,7 +9,7 @@
 )
 
 from gcsfs.tests.settings import TEST_ZONAL_BUCKET
-from gcsfs.tests.utils import tempdir, tmpfile
+from gcsfs.tests.utils import is_real_gcs, tempdir, tmpfile
 from gcsfs.zonal_file import ZonalFile
 
 test_data = b"hello world"
@@ -354,7 +354,7 @@ def test_zonal_file_append_to_empty(extended_gcsfs, zonal_write_mocks, file_path
 
 
 @pytest.mark.skipif(
-    os.environ.get("STORAGE_EMULATOR_HOST") != "https://storage.googleapis.com",
+    not is_real_gcs(),
     reason="This test class is for real GCS only.",
 )
 class TestZonalFileRealGCS:

gcsfs/tests/utils.py

@@ -40,3 +40,8 @@ def tmpfile(extension="", dir=None):
             else:
                 with ignoring(OSError):
                     os.remove(filename)
+
+
+def is_real_gcs():
+    """Checks if tests are explicitly running against real GCS."""
+    return os.environ.get("STORAGE_EMULATOR_HOST") == "https://storage.googleapis.com"