Improve test speed and emulator compatibility (#843)

* Improve test speed with ADC timeout handling and isolated multi-threaded test file fixture

* Add cache_unknown_buckets config to allow caching unknown bucket types in tests

* add tests

* Introduce is_real_gcs

* remove _cache_unknown_buckets

Author: Yonghui-Lee

gcsfs/tests/conftest.py

@@ -25,6 +25,7 @@
     TEST_VERSIONED_BUCKET,
     TEST_ZONAL_BUCKET,
 )
+from gcsfs.tests.utils import is_real_gcs
 
 files = {
     "test/accounts.1.json": (
@@ -60,13 +61,6 @@
     "zonal/test/c": b"ab\n" + b"a" * (2**18) + b"\nab",
 }
 
-_MULTI_THREADED_TEST_DATA_SIZE = 5 * 1024 * 1024  # 5MB
-pattern = b"0123456789abcdef"
-text_files["multi_threaded_test_file"] = (
-    pattern * (_MULTI_THREADED_TEST_DATA_SIZE // len(pattern))
-    + pattern[: _MULTI_THREADED_TEST_DATA_SIZE % len(pattern)]
-)
-
 allfiles = dict(**files, **csv_files, **text_files)
 a = TEST_BUCKET + "/tmp/test/a"
 b = TEST_BUCKET + "/tmp/test/b"
@@ -82,6 +76,37 @@
 }
 
 
+@pytest.fixture(autouse=True)
+def _avoid_adc_timeout(monkeypatch):
+    """Avoid slow ADC lookups and Metadata Server requests in tests."""
+    # Do not apply if tests are explicitly running against real GCS
+    if is_real_gcs():
+        yield
+        return
+
+    # Disable GCE metadata check in google-auth and gcsfs
+    monkeypatch.setenv("NO_GCE_CHECK", "true")
+
+    # Set a dummy project to avoid project ID lookup timeouts if not set
+    if "GOOGLE_CLOUD_PROJECT" not in os.environ:
+        monkeypatch.setenv("GOOGLE_CLOUD_PROJECT", "dummy-project")
+
+    yield
+
+
+@pytest.fixture(autouse=True)
+def _mock_get_bucket_type_on_emulator():
+    """Mock _get_bucket_type to return UNKNOWN instantly on emulator."""
+    if not is_real_gcs():
+        with mock.patch(
+            "gcsfs.extended_gcsfs.ExtendedGcsFileSystem._get_bucket_type",
+            return_value=BucketType.UNKNOWN,
+        ):
+            yield
+    else:
+        yield
+
+
 def stop_docker(container):
     cmd = shlex.split('docker ps -a -q --filter "name=%s"' % container)
     cid = subprocess.check_output(cmd).strip().decode()
@@ -92,6 +117,8 @@ def stop_docker(container):
 @pytest.fixture(scope="session")
 def docker_gcs():
     if "STORAGE_EMULATOR_HOST" in os.environ:
+        if not is_real_gcs():
+            params["token"] = "anon"
         # assume using real API or otherwise have a server already set up
         yield os.getenv("STORAGE_EMULATOR_HOST")
         return
@@ -290,9 +317,6 @@ def final_cleanup(gcs_factory, buckets_to_delete):
 def gcs_versioned(gcs_factory, buckets_to_delete):
     gcs = gcs_factory()
     gcs.version_aware = True
-    is_real_gcs = (
-        os.environ.get("STORAGE_EMULATOR_HOST") == "https://storage.googleapis.com"
-    )
     try:  # ensure we're empty.
         # The versioned bucket might be created by `is_versioning_enabled`
         # in test_core_versioned.py. We must register it for cleanup only if
@@ -306,7 +330,7 @@ def gcs_versioned(gcs_factory, buckets_to_delete):
                 buckets_to_delete.add(TEST_VERSIONED_BUCKET)
         except ImportError:
             pass  # test_core_versioned is not being run
-        if is_real_gcs:
+        if is_real_gcs():
             cleanup_versioned_bucket(gcs, TEST_VERSIONED_BUCKET)
         else:
             # For emulators, we delete and recreate the bucket for a clean state
@@ -321,7 +345,7 @@ def gcs_versioned(gcs_factory, buckets_to_delete):
     finally:
         # Ensure the bucket is empty after the test.
         try:
-            if is_real_gcs:
+            if is_real_gcs():
                 cleanup_versioned_bucket(gcs, TEST_VERSIONED_BUCKET)
         except Exception as e:
             logging.warning(
@@ -367,13 +391,9 @@ def cleanup_versioned_bucket(gcs, bucket_name, prefix=None):
 
 
 def _create_extended_gcsfs(gcs_factory, buckets_to_delete, populate_bucket, **kwargs):
-    is_real_gcs = (
-        os.environ.get("STORAGE_EMULATOR_HOST") == "https://storage.googleapis.com"
-    )
-
     extended_gcsfs = gcs_factory(**kwargs)
     # Only create/delete/populate the bucket if we are NOT using the real GCS endpoint.
-    if not is_real_gcs:
+    if not is_real_gcs():
         if not extended_gcsfs.exists(TEST_ZONAL_BUCKET):
             extended_gcsfs.mkdir(TEST_ZONAL_BUCKET)
             buckets_to_delete.add(TEST_ZONAL_BUCKET)
@@ -433,7 +453,7 @@ def gcs_hns(gcs_factory, buckets_to_delete):
 def zonal_write_mocks():
     """A fixture for mocking Zonal bucket write functionality."""
 
-    if os.environ.get("STORAGE_EMULATOR_HOST") == "https://storage.googleapis.com":
+    if is_real_gcs():
         yield None
         return
 

gcsfs/tests/integration/test_async_gcsfs.py

@@ -20,6 +20,7 @@
 
 from gcsfs.extended_gcsfs import ExtendedGcsFileSystem
 from gcsfs.tests.settings import TEST_HNS_BUCKET
+from gcsfs.tests.utils import is_real_gcs
 
 REQUIRED_ENV_VAR = "GCSFS_EXPERIMENTAL_ZB_HNS_SUPPORT"
 
@@ -34,7 +35,7 @@
         reason=f"Skipping tests: {REQUIRED_ENV_VAR} env variable is not set",
     ),
     pytest.mark.skipif(
-        os.environ.get("STORAGE_EMULATOR_HOST") != "https://storage.googleapis.com",
+        not is_real_gcs(),
         reason="Skipping tests on emulator, requires real GCS.",
     ),
 ]

gcsfs/tests/integration/test_extended_hns.py

@@ -19,6 +19,7 @@
 
 from gcsfs.extended_gcsfs import BucketType, ExtendedGcsFileSystem
 from gcsfs.tests.settings import TEST_HNS_BUCKET, TEST_PROJECT
+from gcsfs.tests.utils import is_real_gcs
 
 should_run_hns = os.getenv("GCSFS_EXPERIMENTAL_ZB_HNS_SUPPORT", "false").lower() in (
     "true",
@@ -27,8 +28,7 @@
 
 # Skip these tests if not running against a real GCS backend or if experimentation flag is not set.
 pytestmark = pytest.mark.skipif(
-    os.environ.get("STORAGE_EMULATOR_HOST") != "https://storage.googleapis.com"
-    or not should_run_hns,
+    not is_real_gcs() or not should_run_hns,
     reason="This test class is for real GCS HNS buckets only and requires experimental flag.",
 )
 

gcsfs/tests/test_core.py

@@ -487,8 +487,7 @@ def test_rm_recursive(gcs):
 
 def test_rm_chunked_batch(gcs):
     files = [f"{TEST_BUCKET}/t{i}" for i in range(303)]
-    for fn in files:
-        gcs.touch(fn)
+    gcs.pipe({fn: b"" for fn in files})
 
     files_created = gcs.find(TEST_BUCKET)
     for fn in files:
@@ -511,8 +510,7 @@ def test_rm_wildcards_in_directory(gcs):
         f"{base_dir}/b1.dat",
         f"{base_dir}/subdir/nested.txt",
     ]
-    for f in files:
-        gcs.touch(f)
+    gcs.pipe({f: b"" for f in files})
 
     # 1. Test '?' wildcard (non-recursive)
     gcs.rm(f"{base_dir}/file?.txt")
@@ -2019,7 +2017,6 @@ def test_find_dircache(gcs):
         f"{TEST_BUCKET}/2014-01-01.csv",
         f"{TEST_BUCKET}/2014-01-02.csv",
         f"{TEST_BUCKET}/2014-01-03.csv",
-        f"{TEST_BUCKET}/multi_threaded_test_file",
         f"{TEST_BUCKET}/zonal",
     }
     assert set(gcs.ls(f"{TEST_BUCKET}/nested")) == {

gcsfs/tests/test_core_versioned.py

@@ -7,6 +7,7 @@
 
 from gcsfs import GCSFileSystem
 from gcsfs.tests.settings import TEST_VERSIONED_BUCKET
+from gcsfs.tests.utils import is_real_gcs
 
 a = TEST_VERSIONED_BUCKET + "/tmp/test/a"
 b = TEST_VERSIONED_BUCKET + "/tmp/test/b"
@@ -22,7 +23,7 @@ def is_versioning_enabled():
     """
     # Don't skip when using an emulator, as we create the versioned bucket ourselves.
     global _VERSIONED_BUCKET_CREATED_BY_TESTS
-    if os.environ.get("STORAGE_EMULATOR_HOST") != "https://storage.googleapis.com":
+    if not is_real_gcs():
         return True, ""
     try:
         gcs = GCSFileSystem(project=os.getenv("GCSFS_TEST_PROJECT", "project"))

gcsfs/tests/test_credentials.py

@@ -45,6 +45,43 @@ def test_connect_google_default_uses_request():
         assert isinstance(kwargs["request"], Request)
 
 
+def test_connect_cloud_success():
+    with patch("google.auth.compute_engine.Credentials") as mock_creds_class:
+        mock_creds = Mock()
+        mock_creds_class.return_value = mock_creds
+
+        cred = GoogleCredentials(
+            project="my-project", access="read_only", token="cloud", on_google=True
+        )
+
+        assert cred.credentials == mock_creds
+        assert mock_creds.refresh.called
+        assert cred.method == "cloud"
+
+
+def test_connect_cloud_failure():
+    import google.auth.exceptions
+
+    with patch("google.auth.compute_engine.Credentials") as mock_creds_class:
+        mock_creds = Mock()
+        mock_creds_class.return_value = mock_creds
+        mock_creds.refresh.side_effect = google.auth.exceptions.RefreshError(
+            "mock error"
+        )
+
+        with pytest.raises(ValueError, match="Invalid gcloud credentials"):
+            GoogleCredentials(
+                project="my-project", access="read_only", token="cloud", on_google=True
+            )
+
+
+def test_connect_cloud_not_on_google():
+    with pytest.raises(ValueError):
+        GoogleCredentials(
+            project="my-project", access="read_only", token="cloud", on_google=False
+        )
+
+
 @pytest.mark.parametrize("token", ["", "incorrect.token", "x" * 100])
 def test_credentials_from_raw_token(token):
     with patch.dict(os.environ, {"FETCH_RAW_TOKEN_EXPIRY": "false"}):