ci(test): enforce coverage-based test selection on PRs (#1468)

Author: sbryngelson

.github/workflows/common/test.sh

@@ -57,12 +57,14 @@ if [ -n "${job_shard:-}" ]; then
     shard_opts="--shard $job_shard"
 fi
 
-# Coverage-based test selection in SHADOW mode on PRs: prints what it WOULD select but the
-# full suite still runs (no --select-enforce). Changed files come from git detection
-# (self-healing deepen) since the SLURM job doesn't receive the paths-filter list.
+# Coverage-based test selection ENFORCED on PRs: runs only tests whose recorded coverage
+# overlaps the PR's changed files (conservative ladder; non-.fpp and uncovered-.fpp changes
+# fall back to run-all). Pushes to master run the full suite as a backstop. Changed files
+# come from git detection (self-healing deepen) since the SLURM job doesn't receive the
+# paths-filter list.
 select_opts=""
 if [ "${GITHUB_EVENT_NAME:-}" = "pull_request" ]; then
-    select_opts="--only-changes"
+    select_opts="--select-enforce"
 fi
 
 ./mfc.sh test -v --max-attempts 3 --no-build $select_opts -a -j $n_test_threads $rdma_opts $device_opts $build_opts $shard_opts -- -c $job_cluster

.github/workflows/test.yml

@@ -264,11 +264,13 @@ jobs:
       - name: Test
         if:   '!matrix.nvhpc'
         run:  |
-          # Coverage-based test selection in SHADOW mode on PRs: the selector
-          # prints what it WOULD run, but the full suite still runs (no
-          # --select-enforce). Enforcement is a separate, later change.
+          # Coverage-based test selection ENFORCED on PRs: runs only the tests whose
+          # recorded coverage overlaps the PR's changed files (conservative ladder in
+          # coverage.py — non-.fpp changes and .fpp files no test covers fall back to
+          # run-all). Pushes to master run the full suite (SELECT empty) as a backstop,
+          # and the NVHPC jobs below still run --test-all as a pre-merge full check.
           SELECT=()
-          [ "${{ github.event_name }}" = "pull_request" ] && SELECT=(--only-changes --changed-files "$CHANGED_FILES")
+          [ "${{ github.event_name }}" = "pull_request" ] && SELECT=(--select-enforce --changed-files "$CHANGED_FILES")
           /bin/bash mfc.sh test -v --max-attempts 3 -j $(nproc) "${SELECT[@]}" $TEST_ALL $TEST_PCT $PRECISION
         env:
           TEST_ALL: ${{ matrix.mpi == 'mpi' && '--test-all' || '' }}

toolchain/mfc/cli/commands.py

@@ -472,14 +472,14 @@
             dest="only_changes",
             action=ArgAction.STORE_TRUE,
             default=False,
-            help="Select only tests whose covered files overlap changed files (shadow mode unless --select-enforce).",
+            help="Shadow mode: compute and print the coverage-based test selection but still run the full suite. Use --select-enforce to actually prune.",
         ),
         Argument(
             name="select-enforce",
             dest="select_enforce",
             action=ArgAction.STORE_TRUE,
             default=False,
-            help="With --only-changes, actually skip unselected tests (otherwise shadow: print selection, run all).",
+            help="Run only the coverage-selected tests (implies selection; skips unselected tests). Without it, --only-changes is shadow-only.",
         ),
         Argument(name="changed-files", dest="changed_files", type=str, default=None, help="Changed-file list (newline-, space-, or comma-separated; from CI paths-filter). Overrides git detection."),
         Argument(name="changes-branch", dest="changes_branch", type=str, default="master", help="Branch to diff against for --only-changes."),

toolchain/mfc/test/test.py

@@ -178,7 +178,7 @@ def is_uuid(term):
         if not cases:
             raise MFCException(f"--shard {ARG('shard')} matched zero test cases. Total cases before sharding may be less than shard count.")
 
-    if ARG("only_changes"):
+    if ARG("only_changes") or ARG("select_enforce"):
         import datetime
 
         from .. import common