Add CI workflow for flaky runs (#2618)

Co-authored-by: Erik van Sebille <e.vansebille@uu.nl>

Author: VeckoTheGecko

.github/workflows/ci.yml

@@ -126,6 +126,66 @@ jobs:
         with:
           name: Unittest report ${{ matrix.os }}-${{ matrix.pixi-environment }}
           path: ${{ env.COVERAGE_REPORT }}
+  flaky-unit-test:
+    name: "Flaky unit tests: ${{ matrix.os }} | pixi run -e ${{ matrix.pixi-environment }} tests -m 'flaky'"
+    runs-on: ${{ matrix.os }}-latest
+    needs: [cache-pixi-lock]
+    permissions:
+      contents: read
+    env:
+      COVERAGE_REPORT: "${{ matrix.os }}_${{ matrix.pixi-environment }}_unit_test_report.html"
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu]
+        pixi-environment: [test]
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          persist-credentials: false
+      - name: Restore cached pixi lockfile
+        uses: Parcels-code/pixi-lock/restore@38495788b79a5ff26009aecc15daa9a8310b8832 # v0.1.0
+        with:
+          cache-key: ${{ needs.cache-pixi-lock.outputs.cache-key }}
+      - uses: prefix-dev/setup-pixi@fef5c9568ca6c4ff7707bf840ab0692ba3f08293 # v0.9.0
+        with:
+          pixi-version: ${{ needs.cache-pixi-lock.outputs.pixi-version }}
+          locked: false # TODO: Remove once v7 of the lock file is removed, or once we stop having external source dependencies https://github.com/Parcels-code/Parcels/pull/2550#issuecomment-4088660238
+          cache: true
+          cache-write: ${{ github.event_name == 'push' && github.ref_name == 'main' }}
+      # https://github.com/actions/cache/blob/main/tips-and-workarounds.md#update-a-cache
+      - name: Restore cached hypothesis directory
+        id: restore-hypothesis-cache
+        uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
+        with:
+          path: .hypothesis/
+          key: cache-hypothesis-${{ runner.os }}-${{ github.run_id }}
+          restore-keys: |
+            cache-hypothesis-${{ runner.os }}-
+      - name: Unit test
+        id: unit-test
+        run: |
+          pixi run -e ${{ matrix.pixi-environment }} tests -m 'flaky' -v -s --cov=parcels --cov-report=xml --html="${COVERAGE_REPORT}" --self-contained-html
+      # explicitly save the cache so it gets updated, also do this even if it fails.
+      - name: Save cached hypothesis directory
+        id: save-hypothesis-cache
+        if: always() && steps.unit-test.outcome != 'skipped'
+        uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
+        with:
+          path: .hypothesis/
+          key: cache-hypothesis-${{ runner.os }}-${{ github.run_id }}
+      - name: Codecov
+        uses: codecov/codecov-action@5a1091511ad55cbe89839c7260b706298ca349f7 # v5.5.1
+        env:
+          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} # zizmor: ignore[secrets-outside-env]
+        with:
+          flags: unit-tests
+      - name: Upload test results
+        if: ${{ always() }} # Always run this step, even if tests fail
+        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
+        with:
+          name: Flaky unittest report ${{ matrix.os }}-${{ matrix.pixi-environment }}
+          path: ${{ env.COVERAGE_REPORT }}
   integration-test:
     name: "Integration: ${{ matrix.os }} | pixi run -e ${{ matrix.pixi-environment }} tests-notebooks"
     runs-on: ${{ matrix.os }}-latest

pixi.toml

@@ -77,7 +77,7 @@ pytest-html = "*"
 pytest-cov = "*"
 
 [feature.test.tasks]
-tests = { cmd = "pytest", description = "Run the test suite." }
+tests = { cmd = "pytest -m 'not flaky'", description = "Run the test suite." }
 tests-notebooks = { cmd = "pytest --nbval-lax docs/user_guide/examples", description = "Run the user guide example notebooks as tests." }
 
 

tests/datasets/test_remote.py

@@ -11,6 +11,7 @@ def tmp_path_parcels_example_data(monkeypatch, tmp_path):
     return tmp_path
 
 
+@pytest.mark.flaky
 @pytest.mark.parametrize("url", [remote._ODIE.get_url(filename) for filename in remote._ODIE.registry.keys()])
 def test_pooch_registry_url_reponse(url):
     response = requests.head(url)