Run tests concurrently in CI (#845)

* Run tests concurrently in CI

* add _get_bucket_name

Author: Yonghui-Lee

.github/workflows/ci.yml

@@ -24,6 +24,8 @@ jobs:
       fail-fast: false
       matrix:
         python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"]
+    env:
+      STORAGE_EMULATOR_HOST: http://localhost:4443
 
     steps:
       - name: Checkout source
@@ -46,36 +48,44 @@ jobs:
       - name: install
         run: |
           pip install -e .
+
+      - name: Start GCS Emulator
+        run: |
+          docker run -d -p 4443:4443 --name gcsfs_test fsouza/fake-gcs-server:latest -scheme http -public-host 0.0.0.0:4443 -external-url http://localhost:4443 -backend memory
+          timeout 30s bash -c 'until curl -s http://localhost:4443/storage/v1/b; do sleep 1; done'
+
       - name: Run Standard Bucket Tests with extended feature support turned OFF
         run: |
-          export GOOGLE_APPLICATION_CREDENTIALS=$(pwd)/gcsfs/tests/fake-secret.json
+          export GOOGLE_APPLICATION_CREDENTIALS=$(pwd)/gcsfs/tests/fake-service-account-credentials.json
           export GCSFS_EXPERIMENTAL_ZB_HNS_SUPPORT="false"
           pytest -vv -s \
           --cov=gcsfs --cov-report=xml \
           --log-format="%(asctime)s %(levelname)s %(message)s" \
           --log-date-format="%H:%M:%S" \
+          -n auto \
           gcsfs/
       - name: Run Benchmark Infrastructure Unit Tests
         run: |
           pip install -r gcsfs/tests/perf/microbenchmarks/requirements.txt
           pytest gcsfs/tests/perf/microbenchmarks --run-benchmarks-infra
       - name: Run Standard Bucket Tests with default ON extended feature support
         run: |
-          export GOOGLE_APPLICATION_CREDENTIALS=$(pwd)/gcsfs/tests/fake-secret.json
+          export GOOGLE_APPLICATION_CREDENTIALS=$(pwd)/gcsfs/tests/fake-service-account-credentials.json
           pytest -vv -s \
             --cov=gcsfs --cov-append --cov-report=xml \
             --log-format="%(asctime)s %(levelname)s %(message)s" \
             --log-date-format="%H:%M:%S" \
+            -n auto \
             gcsfs/
       - name: Run Extended tests (Zonal & HNS Enabled)
         run: |
-          export GOOGLE_APPLICATION_CREDENTIALS=$(pwd)/gcsfs/tests/fake-secret.json
-          # TODO: Use dedicated test variables to decouple specialised test execution from the GCSFS_EXPERIMENTAL_ZB_HNS_SUPPORT feature flag.
+          export GOOGLE_APPLICATION_CREDENTIALS=$(pwd)/gcsfs/tests/fake-service-account-credentials.json
           export GCSFS_EXPERIMENTAL_ZB_HNS_SUPPORT="true"
           pytest -vv -s \
             --cov=gcsfs --cov-append --cov-report=xml \
             --log-format="%(asctime)s %(levelname)s %(message)s" \
             --log-date-format="%H:%M:%S" \
+            -n auto \
             gcsfs/tests/test_extended_gcsfs.py \
             gcsfs/tests/test_extended_hns_gcsfs.py \
             gcsfs/tests/test_extended_gcsfs_unit.py \

environment_gcsfs.yaml

@@ -22,6 +22,7 @@ dependencies:
   - pytest-asyncio
   - pytest-cov
   - pytest-subtests
+  - pytest-xdist
   - requests
   - ujson
   - pip:

gcsfs/tests/fake-secret.json

@@ -1,14 +1,23 @@
 import os
 
-TEST_BUCKET = os.getenv("GCSFS_TEST_BUCKET", "gcsfs_test")
-TEST_VERSIONED_BUCKET = os.getenv("GCSFS_TEST_VERSIONED_BUCKET", "gcsfs_test_versioned")
-TEST_HNS_BUCKET = os.getenv("GCSFS_HNS_TEST_BUCKET", "gcsfs_hns_test")
-TEST_ZONAL_BUCKET = os.getenv("GCSFS_ZONAL_TEST_BUCKET", "gcsfs_zonal_test")
+
+def _get_bucket_name(env_var: str, default_name: str) -> str:
+    worker_id = os.environ.get("PYTEST_XDIST_WORKER")
+    suffix = f"_{worker_id}" if worker_id else ""
+    return os.getenv(env_var, default_name) + suffix
+
+
+TEST_BUCKET = _get_bucket_name("GCSFS_TEST_BUCKET", "gcsfs_test")
+TEST_VERSIONED_BUCKET = _get_bucket_name(
+    "GCSFS_TEST_VERSIONED_BUCKET", "gcsfs_test_versioned"
+)
+TEST_HNS_BUCKET = _get_bucket_name("GCSFS_HNS_TEST_BUCKET", "gcsfs_hns_test")
+TEST_ZONAL_BUCKET = _get_bucket_name("GCSFS_ZONAL_TEST_BUCKET", "gcsfs_zonal_test")
 TEST_PROJECT = os.getenv("GCSFS_TEST_PROJECT", "project")
-TEST_REQUESTER_PAYS_BUCKET = os.getenv(
+TEST_REQUESTER_PAYS_BUCKET = _get_bucket_name(
     "GCSFS_TEST_REQ_PAYS_BUCKET", "gcsfs_test_req_pays"
 )
-TEST_HNS_REQUESTER_PAYS_BUCKET = os.getenv(
+TEST_HNS_REQUESTER_PAYS_BUCKET = _get_bucket_name(
     "GCSFS_HNS_TEST_REQ_PAYS_BUCKET", "gcsfs_hns_test_req_pays"
 )
 TEST_KMS_KEY = os.getenv(

gcsfs/tests/settings.py

@@ -2399,23 +2399,17 @@ def test_find_maxdepth(gcs):
 
 
 def test_sign(gcs, monkeypatch):
+    if not gcs.on_google:
+        pytest.skip("Emulator does not support signing")
+
     file = TEST_BUCKET + "/test.jpg"
     with gcs.open(file, "wb") as f:
         f.write(b"This is a test string")
     assert gcs.cat(file) == b"This is a test string"
 
-    # `sign` is creating a google Client on its own, it needs a realistically
-    # looking credentials file.
-    if not gcs.on_google:
-        monkeypatch.setenv(
-            "GOOGLE_APPLICATION_CREDENTIALS",
-            os.path.dirname(__file__) + "/fake-service-account-credentials.json",
-        )
-
     current_ts_utc = int(datetime.now(tz=timezone.utc).timestamp())
     result = gcs.sign(file)
 
-    # Check it here since emulator doesn't really validate those values
     params = parse_qs(urlparse(result).query)
     assert int(params["Expires"][0]) >= current_ts_utc + 100
 

gcsfs/tests/test_core.py

@@ -85,8 +85,10 @@ def test_connect_cloud_not_on_google():
 @pytest.mark.parametrize("token", ["", "incorrect.token", "x" * 100])
 def test_credentials_from_raw_token(token):
     with patch.dict(os.environ, {"FETCH_RAW_TOKEN_EXPIRY": "false"}):
+        fs = GCSFileSystem(project="myproject", token=token)
+        if not fs.on_google:
+            pytest.skip("Emulator does not validate tokens")
         with pytest.raises(HttpError, match="Invalid Credentials"):
-            fs = GCSFileSystem(project="myproject", token=token)
             fs.ls("/")
 
 

gcsfs/tests/test_credentials.py

@@ -1105,17 +1105,27 @@ def test_get_directory_from_zonal_bucket(extended_gcsfs):
 
 @pytest.mark.asyncio
 @pytest.mark.parametrize(
-    "source_bucket, dest_bucket, should_fail",
+    "source_bucket_type, dest_bucket_type, should_fail",
     [
-        (TEST_ZONAL_BUCKET, TEST_ZONAL_BUCKET, True),
-        (TEST_ZONAL_BUCKET, TEST_BUCKET, True),
-        (TEST_BUCKET, TEST_ZONAL_BUCKET, True),
-        (TEST_BUCKET, TEST_BUCKET, False),
+        (BucketType.ZONAL_HIERARCHICAL, BucketType.ZONAL_HIERARCHICAL, True),
+        (BucketType.ZONAL_HIERARCHICAL, BucketType.NON_HIERARCHICAL, True),
+        (BucketType.NON_HIERARCHICAL, BucketType.ZONAL_HIERARCHICAL, True),
+        (BucketType.NON_HIERARCHICAL, BucketType.NON_HIERARCHICAL, False),
     ],
 )
 async def test_cp_file_not_implemented_error(
-    async_gcs, source_bucket, dest_bucket, should_fail
+    async_gcs, source_bucket_type, dest_bucket_type, should_fail
 ):
+    source_bucket = (
+        TEST_ZONAL_BUCKET
+        if source_bucket_type == BucketType.ZONAL_HIERARCHICAL
+        else TEST_BUCKET
+    )
+    dest_bucket = (
+        TEST_ZONAL_BUCKET
+        if dest_bucket_type == BucketType.ZONAL_HIERARCHICAL
+        else TEST_BUCKET
+    )
     """
     Tests _cp_file behavior for combinations of Zonal and Standard buckets.
     """

gcsfs/tests/test_extended_gcsfs.py

@@ -45,6 +45,7 @@ dev = [
     "pytest-timeout",
     "pytest-cov",
     "pytest-subtests",
+    "pytest-xdist",
 ]
 
 [project.urls]