Mark lithops tests as flaky, run in separate CI job (#934)

* add empty release notes

* Mark lithops tests as flaky and run them in a separate CI job

Lithops tests intermittently fail with an ImportError due to lithops'
module shipping mechanism producing incomplete copies of virtualizarr
in subprocess workers. See https://github.com/zarr-developers/VirtualiZarr/issues/XXX

- Add --run-flaky pytest option (matching xarray's approach)
- Mark lithops tests with @pytest.mark.flaky so they're skipped by default
- Add a separate flaky-tests CI job with continue-on-error: true

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

Author: TomNicholas

.github/workflows/main.yml

@@ -59,6 +59,24 @@ jobs:
           fail_ci_if_error: false
           token: ${{ secrets.CODECOV_TOKEN }}
 
+  flaky:
+    name: flaky-tests
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    defaults:
+      run:
+        shell: bash -l {0}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: prefix-dev/setup-pixi@v0.9.3
+        with:
+          pixi-version: v0.59.0
+          environments: test-py311
+
+      - name: Running Flaky Tests
+        run: |
+          pixi run -e test-py311 run-flaky-tests
+
   check-docs:
     name: check-docs
     runs-on: ubuntu-latest

conftest.py

@@ -40,6 +40,11 @@ def pytest_addoption(parser):
         action="store_true",
         help="runs tests requiring docker and minio",
     )
+    parser.addoption(
+        "--run-flaky",
+        action="store_true",
+        help="runs flaky tests",
+    )
 
 
 def pytest_runtest_setup(item):
@@ -52,6 +57,8 @@ def pytest_runtest_setup(item):
         pytest.skip("set --run-minio-tests to run tests requiring docker and minio")
     if "slow" in item.keywords and not item.config.getoption("--run-slow-tests"):
         pytest.skip("set --run-slow-tests to run slow tests")
+    if "flaky" in item.keywords and not item.config.getoption("--run-flaky"):
+        pytest.skip("set --run-flaky to run flaky tests")
 
 
 def _xarray_subset():

pyproject.toml

@@ -206,6 +206,7 @@ run-tests-no-network = { cmd = "pytest -n auto --dist loadscope --verbose" }
 run-tests-cov = { cmd = "pytest -n auto --dist loadscope --run-network-tests --verbose --cov=virtualizarr --cov=term-missing" }
 run-tests-xml-cov = { cmd = "pytest -n auto --dist loadscope --run-network-tests --verbose --cov=virtualizarr --cov-report=xml" }
 run-tests-html-cov = { cmd = "pytest -n auto --dist loadscope --run-network-tests --verbose --cov=virtualizarr --cov-report=html" }
+run-flaky-tests = { cmd = "pytest -m flaky --run-flaky --run-network-tests --verbose" }
 
 # Define which features and groups to include in different pixi (similar to conda) environments)
 [tool.pixi.environments]
@@ -331,11 +332,7 @@ addopts = ["--doctest-modules", "--ignore=examples"]
 timeout = 300
 asyncio_default_fixture_loop_scope = "session"
 markers = [
-    # Although we may not use pytest.mark.flaky, some of our test modules import
-    # from xarray.tests, and xarray.tests.__init__.py references pytest.mark.flaky.
-    # Therefore, without the "flaky" marker below, during test execution, we see
-    # this warning: "PytestUnknownMarkWarning: Unknown pytest.mark.flaky"
-    "flaky: flaky tests",
+    "flaky: flaky tests (skipped by default, run with '--run-flaky')",
     "network: marks test requiring internet (select with '--run-network-tests')",
     "slow: marks test as slow (select with '--run-slow-tests')",
     "minio: marks test requiring docker and minio (select with '--run-minio-tests')",

virtualizarr/tests/test_parallel.py

@@ -6,6 +6,7 @@
 from virtualizarr.tests import requires_lithops
 
 
+@pytest.mark.flaky
 @requires_lithops
 def test_lithops_executor_with_multiple_args():
     with LithopsEagerFunctionExecutor() as exec:
@@ -14,6 +15,7 @@ def test_lithops_executor_with_multiple_args():
     assert tuple(results) == (5, 7, 9)
 
 
+@pytest.mark.flaky
 @requires_lithops
 def test_lithops_executor_with_partial():
     from functools import partial