Add xfail_if_xyz_not_installed markers and use them instead of ignoring entire test files

Author: ValerianRey

.github/actions/run-tests/action.yml

@@ -2,10 +2,6 @@ name: Run Tests
 description: Run unit and doc tests with coverage
 
 inputs:
-  ignores:
-    description: 'Space-separated list of test files to ignore'
-    required: false
-    default: ''
   dtype:
     description: 'Torch dtype to use for tests'
     required: false
@@ -24,18 +20,4 @@ runs:
         PYTEST_TORCH_DTYPE: ${{ inputs.dtype }}
         PYTEST_TORCH_DEVICE: ${{ inputs.device }}
       run: |
-        # Build the pytest command
-        cmd="uv run pytest -W error tests/unit tests/doc"
-
-        # Add ignore flags for each file
-        if [ -n "${{ inputs.ignores }}" ]; then
-          for file in ${{ inputs.ignores }}; do
-            cmd="$cmd --ignore $file"
-          done
-        fi
-
-        # Add coverage options
-        cmd="$cmd --cov=src --cov-report=xml"
-
-        # Execute the command
-        eval $cmd
+        uv run pytest -W error tests/unit tests/doc --cov=src --cov-report=xml

.github/workflows/tests.yml

@@ -41,7 +41,7 @@ jobs:
           token: ${{ secrets.CODECOV_TOKEN }}
 
   tests-default-install:
-    name: Run (most) tests with default install
+    name: Run tests with default install
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repository
@@ -54,8 +54,6 @@ jobs:
           groups: test
 
       - uses: ./.github/actions/run-tests
-        with:
-          ignores: tests/unit/aggregation/test_cagrad.py tests/unit/aggregation/test_nash_mtl.py tests/doc/test_aggregation.py
 
       - *upload-codecov
 

tests/conftest.py

@@ -1,5 +1,6 @@
 import random as rand
 from contextlib import nullcontext
+from importlib.util import find_spec
 
 import torch
 from pytest import RaisesExc, fixture, mark
@@ -30,16 +31,35 @@ def pytest_addoption(parser):
 def pytest_configure(config):
     config.addinivalue_line("markers", "slow: mark test as slow to run")
     config.addinivalue_line("markers", "xfail_if_cuda: mark test as xfail if running on cuda")
+    config.addinivalue_line(
+        "markers", "xfail_if_cagrad_not_installed: mark test as xfail if CAGrad is not installed"
+    )
+    config.addinivalue_line(
+        "markers",
+        "xfail_if_nashmtl_not_installed: mark test as xfail if NashMTL is not installed",
+    )
 
 
 def pytest_collection_modifyitems(config, items):
     skip_slow = mark.skip(reason="Slow test. Use --runslow to run it.")
     xfail_cuda = mark.xfail(reason=f"Test expected to fail on {DEVICE}")
+
+    # Check if optional dependencies are installed
+    cagrad_installed = all(find_spec(name) is not None for name in ["cvxpy", "clarabel"])
+    nashmtl_installed = all(find_spec(name) is not None for name in ["cvxpy", "ecos"])
+
+    xfail_cagrad = mark.xfail(reason="CAGrad dependencies not installed")
+    xfail_nashmtl = mark.xfail(reason="NashMTL dependencies not installed")
+
     for item in items:
         if "slow" in item.keywords and not config.getoption("--runslow"):
             item.add_marker(skip_slow)
         if "xfail_if_cuda" in item.keywords and str(DEVICE).startswith("cuda"):
             item.add_marker(xfail_cuda)
+        if "xfail_if_cagrad_not_installed" in item.keywords and not cagrad_installed:
+            item.add_marker(xfail_cagrad)
+        if "xfail_if_nashmtl_not_installed" in item.keywords and not nashmtl_installed:
+            item.add_marker(xfail_nashmtl)
 
 
 def pytest_make_parametrize_id(config, val, argname):

tests/unit/aggregation/test_cagrad.py

@@ -10,6 +10,8 @@
 from ._asserts import assert_expected_structure, assert_non_conflicting, assert_non_differentiable
 from ._inputs import scaled_matrices, typical_matrices
 
+pytestmark = mark.xfail_if_cagrad_not_installed
+
 scaled_pairs = [(CAGrad(c=0.5), matrix) for matrix in scaled_matrices]
 typical_pairs = [(CAGrad(c=0.5), matrix) for matrix in typical_matrices]
 requires_grad_pairs = [(CAGrad(c=0.5), ones_(3, 5, requires_grad=True))]

tests/unit/aggregation/test_nash_mtl.py

@@ -8,6 +8,8 @@
 from ._asserts import assert_expected_structure, assert_non_differentiable
 from ._inputs import nash_mtl_matrices
 
+pytestmark = mark.xfail_if_nashmtl_not_installed
+
 
 def _make_aggregator(matrix: Tensor) -> NashMTL:
     return NashMTL(n_tasks=matrix.shape[0])

tests/unit/aggregation/test_values.py

@@ -87,8 +87,22 @@
 try:
     from torchjd.aggregation import CAGrad, CAGradWeighting
 
-    AGGREGATOR_PARAMETRIZATIONS.append((CAGrad(c=0.5), J_base, tensor([0.1835, 1.2041, 1.2041])))
-    WEIGHTING_PARAMETRIZATIONS.append((CAGradWeighting(c=0.5), G_base, tensor([0.7041, 0.5000])))
+    AGGREGATOR_PARAMETRIZATIONS.append(
+        param(
+            CAGrad(c=0.5),
+            J_base,
+            tensor([0.1835, 1.2041, 1.2041]),
+            marks=mark.xfail_if_cagrad_not_installed,
+        )
+    )
+    WEIGHTING_PARAMETRIZATIONS.append(
+        param(
+            CAGradWeighting(c=0.5),
+            G_base,
+            tensor([0.7041, 0.5000]),
+            marks=mark.xfail_if_cagrad_not_installed,
+        )
+    )
 except ImportError:
     pass
 
@@ -100,7 +114,10 @@
             NashMTL(n_tasks=2),
             J_base,
             tensor([0.0542, 0.7061, 0.7061]),
-            marks=mark.filterwarnings("ignore::UserWarning"),
+            marks=[
+                mark.filterwarnings("ignore::UserWarning"),
+                mark.xfail_if_nashmtl_not_installed,
+            ],
         )
     )