ci: parallelize regression tests per system using dynamic matrix

Author: harryswift01

.github/workflows/pr.yaml

@@ -20,10 +20,10 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6
+        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8
 
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
+        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405
         with:
           python-version: ${{ matrix.python-version }}
           cache: pip
@@ -36,12 +36,44 @@ jobs:
       - name: Pytest (unit) • ${{ matrix.os }}, ${{ matrix.python-version }}
         run: python -m pytest tests/unit
 
+  discover-systems:
+    name: Discover regression systems
+    runs-on: ubuntu-24.04
+    outputs:
+      systems: ${{ steps.set-systems.outputs.systems }}
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8
+
+      - name: Set up Python 3.14
+        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405
+        with:
+          python-version: "3.14"
+          cache: pip
+
+      - name: Install testing dependencies
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install -e .[testing]
+
+      - name: Discover systems
+        id: set-systems
+        run: |
+          SYSTEMS=$(python -m tests.regression.list_systems)
+          echo "systems=$SYSTEMS" >> $GITHUB_OUTPUT
+
   regression-quick:
-    name: Regression (fast)
-    needs: unit
+    name: Regression (fast) • ${{ matrix.system }}
+    needs: [unit, discover-systems]
     runs-on: ubuntu-24.04
     timeout-minutes: 35
 
+    strategy:
+      fail-fast: false
+      matrix:
+        system: ${{ fromJson(needs.discover-systems.outputs.systems) }}
+
     steps:
       - name: Checkout
         uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8
@@ -63,20 +95,21 @@ jobs:
           python -m pip install --upgrade pip
           python -m pip install -e .[testing]
 
-      - name: Run fast regression tests (parallel)
+      - name: Run fast regression tests (per system)
         run: |
           python -m pytest tests/regression \
             -m "not slow" \
             -n auto \
             --dist=loadscope \
+            -k "${{ matrix.system }}" \
             -vv \
             --durations=20
 
       - name: Upload artifacts (failure)
         if: failure()
         uses: actions/upload-artifact@v4
         with:
-          name: quick-regression-failure
+          name: quick-regression-failure-${{ matrix.system }}
           path: |
             .testdata/**
             /tmp/pytest-of-*/pytest-*/**
@@ -121,10 +154,10 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6
+        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8
 
       - name: Set up Python 3.14
-        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
+        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405
         with:
           python-version: "3.14"
           cache: pip

.github/workflows/weekly-regression.yaml

@@ -21,10 +21,21 @@ jobs:
       - name: Checkout
         uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8
 
+      - name: Set up Python 3.14
+        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405
+        with:
+          python-version: "3.14"
+          cache: pip
+
+      - name: Install testing dependencies
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install -e .[testing]
+
       - name: Generate system matrix
         id: set
         run: |
-          systems=$(ls tests/regression/configs | jq -R -s -c 'split("\n")[:-1]')
+          systems=$(python -m tests.regression.list_systems)
           echo "systems=$systems" >> $GITHUB_OUTPUT
 
   regression:
@@ -56,14 +67,16 @@ jobs:
 
       - name: Install testing dependencies
         run: |
-          pip install --upgrade pip
-          pip install -e .[testing]
+          python -m pip install --upgrade pip
+          python -m pip install -e .[testing]
 
-      - name: Run regression tests (slow)
+      - name: Run regression tests (slow, per system)
         run: |
           pytest tests/regression \
             -k "${{ matrix.system }}" \
             --run-slow \
+            -n auto \
+            --dist=loadscope \
             -vv \
             --durations=20
 

tests/regression/conftest.py

@@ -9,9 +9,6 @@ def pytest_addoption(parser: pytest.Parser) -> None:
 
     Adds options to control regression test execution, baseline updates,
     and debugging output.
-
-    Args:
-        parser (pytest.Parser): Pytest CLI parser.
     """
     parser.addoption(
         "--run-slow",
@@ -31,14 +28,17 @@ def pytest_addoption(parser: pytest.Parser) -> None:
         default=False,
         help="Print CodeEntropy stdout/stderr and paths for easier debugging.",
     )
+    parser.addoption(
+        "--system",
+        action="append",
+        default=None,
+        help="Run only tests for specified system(s). Can be passed multiple times.",
+    )
 
 
 def pytest_configure(config: pytest.Config) -> None:
     """
     Register custom pytest markers.
-
-    Args:
-        config (pytest.Config): Pytest configuration object.
     """
     config.addinivalue_line("markers", "regression: end-to-end regression tests")
     config.addinivalue_line("markers", "slow: long-running tests (20-30+ minutes)")
@@ -48,19 +48,37 @@ def pytest_collection_modifyitems(
     config: pytest.Config, items: list[pytest.Item]
 ) -> None:
     """
-    Modify collected test items to skip slow tests unless explicitly enabled.
-
-    Only tests marked with `@pytest.mark.slow` are skipped when the
-    `--run-slow` flag is not provided.
-
-    Args:
-        config (pytest.Config): Pytest configuration object.
-        items (list[pytest.Item]): Collected test items.
+    Modify collected test items to:
+    1. Skip slow tests unless --run-slow is provided
+    2. Filter tests by --system if specified
     """
-    if config.getoption("--run-slow"):
+    if not config.getoption("--run-slow"):
+        skip_slow = pytest.mark.skip(
+            reason="Skipped slow test (use --run-slow to run)."
+        )
+        for item in items:
+            if "slow" in item.keywords:
+                item.add_marker(skip_slow)
+
+    selected_systems = config.getoption("--system")
+    if not selected_systems:
         return
 
-    skip_slow = pytest.mark.skip(reason="Skipped slow test (use --run-slow to run).")
+    filtered_items = []
+
     for item in items:
-        if "slow" in item.keywords:
-            item.add_marker(skip_slow)
+        callspec = getattr(item, "callspec", None)
+
+        case = None
+        if callspec is not None:
+            case = callspec.params.get("case")
+
+        # Keep non-parametrized tests
+        if case is None:
+            filtered_items.append(item)
+            continue
+
+        if hasattr(case, "system") and case.system in selected_systems:
+            filtered_items.append(item)
+
+    items[:] = filtered_items

tests/regression/list_systems.py

@@ -0,0 +1,12 @@
+import json
+
+from tests.regression.cases import discover_cases
+
+
+def main():
+    systems = sorted({case.system for case in discover_cases()})
+    print(json.dumps(systems))
+
+
+if __name__ == "__main__":
+    main()