Add "validation" pytest marker (#2633)

Author: VeckoTheGecko

.github/workflows/ci.yml

@@ -127,7 +127,7 @@ jobs:
           name: Unittest report ${{ matrix.os }}-${{ matrix.pixi-environment }}
           path: ${{ env.COVERAGE_REPORT }}
   flaky-unit-test:
-    name: "Flaky unit tests: ${{ matrix.os }} | pixi run -e ${{ matrix.pixi-environment }} tests -m 'flaky'"
+    name: "Flaky unit tests: ${{ matrix.os }} | pixi run -e ${{ matrix.pixi-environment }} tests-flaky"
     runs-on: ${{ matrix.os }}-latest
     needs: [cache-pixi-lock]
     permissions:
@@ -165,7 +165,7 @@ jobs:
       - name: Unit test
         id: unit-test
         run: |
-          pixi run -e ${{ matrix.pixi-environment }} tests -m 'flaky' -v -s --cov=parcels --cov-report=xml --html="${COVERAGE_REPORT}" --self-contained-html
+          pixi run -e ${{ matrix.pixi-environment }} tests-flaky -v -s --cov=parcels --cov-report=xml --html="${COVERAGE_REPORT}" --self-contained-html
       # explicitly save the cache so it gets updated, also do this even if it fails.
       - name: Save cached hypothesis directory
         id: save-hypothesis-cache

docs/development/maintainer.md

@@ -19,6 +19,7 @@
 
 ## Release checklist
 
+- Run the validation test suite (`pixi run tests-validation`)
 - Go to GitHub, draft new release. Enter name of version and "create new tag" if it doesn't already exist. Click "Generate Release Notes". Currate release notes as needed. Look at a previous version release to match the format (title, header, section organisation etc.)
 - Go to [conda-forge/parcels-feedstock](https://github.com/conda-forge/parcels-feedstock), create a new issue (select the "Bot Commands" issue from the menu) with title `@conda-forge-admin, please update version`. This will prompt a build, otherwise there can be a delay in the build.
   - Approve PR and merge on green

pixi.toml

@@ -79,7 +79,9 @@ pytest-html = "*"
 pytest-cov = "*"
 
 [feature.test.tasks]
-tests = { cmd = "pytest -m 'not flaky'", description = "Run the test suite." }
+tests = { cmd = "pytest", description = "Run the test suite." }
+tests-flaky = { cmd = "pytest -m 'flaky'", description = "Run flaky tests." }
+tests-validation = { cmd = "pytest -m 'validation'", description = "Run validation tests." }
 tests-notebooks = { cmd = "pytest --nbval-lax docs/user_guide/examples", description = "Run the user guide example notebooks as tests." }
 
 

pyproject.toml

@@ -58,8 +58,9 @@ testpaths = ["tests"]
 python_files = ["test_*.py", "example_*.py", "*tutorial*"]
 minversion = "7"
 markers = [ # can be skipped by doing `pytest -m "not slow"` etc.
-  "flaky: flaky tests",
   "slow: slow tests",
+  "validation: validation tests (skipped by default, run with `-m validation`)",
+  "flaky: flaky tests (skipped by default, run with `-m flaky`)",
   "v4alpha: failing tests that should work for v4alpha",
   "v4future: failing tests that should work for a future release of v4",
   "v4remove: failing tests that should probably be removed later",

tests/conftest.py

@@ -1,5 +1,19 @@
 import pytest
 
+SKIP_BY_DEFAULT = {"validation", "flaky"}
+
+
+def pytest_collection_modifyitems(config, items):
+    if not config.getoption("-m"):
+        for item in items:
+            print(item.keywords)
+            skip_by_default = list(SKIP_BY_DEFAULT & set(item.keywords))
+            if skip_by_default:
+                skip_marker = skip_by_default[0]  # get first marker in case of multiple
+                item.add_marker(
+                    pytest.mark.skip(reason=f"{skip_marker} tests skipped by default, use `-m {skip_marker}` to run")
+                )
+
 
 @pytest.fixture
 def tmp_parquet(tmp_path):

tests/validation/__init__.py

@@ -36,6 +36,8 @@
     AdvectionRK4_3D,
 )
 
+pytestmark = pytest.mark.validation
+
 # Uniform translation parameters
 T1_1_U0 = 0.001
 T1_1_V0 = 0.0005
@@ -67,7 +69,7 @@
 )
 @pytest.mark.parametrize("integrator", [AdvectionEE, AdvectionRK4], ids=["EE", "RK4"])
 def test_uniform_translation_exact(dataset_fn, integrator):
-    ds = dataset_fn(nx=20, U0=T1_1_U0, V0=T1_1_V0)
+    ds = dataset_fn(nx=20, u0=T1_1_U0, v0=T1_1_V0)
     fieldset = parcels.FieldSet.from_ugrid_conventions(ds, mesh="flat")
 
     pset = parcels.ParticleSet(fieldset, lon=[T1_1_LON0], lat=[T1_1_LAT0])