Use cargo-nextest for Rust tests

Signed-off-by: lucarlig <luca.carlig@ibm.com>

Author: lucarlig

.cargo/mutants.toml

@@ -0,0 +1,5 @@
+# SPDX-License-Identifier: Apache-2.0
+
+test_tool = "nextest"
+additional_cargo_test_args = ["--profile=mutants"]
+cap_lints = false

.config/nextest.toml

@@ -0,0 +1,11 @@
+# SPDX-License-Identifier: Apache-2.0
+
+nextest-version = "0.9.133"
+
+[profile.ci]
+fail-fast = false
+failure-output = "immediate-final"
+
+[profile.mutants]
+fail-fast = true
+failure-output = "final"

.github/workflows/ci-rust-python-package.yaml

@@ -7,6 +7,8 @@ on:
       - "Makefile"
       - "Cargo.toml"
       - "Cargo.lock"
+      - ".cargo/**"
+      - ".config/nextest.toml"
       - "deny.toml"
       - "crates/**"
       - "README.md"
@@ -23,6 +25,8 @@ on:
       - "Makefile"
       - "Cargo.toml"
       - "Cargo.lock"
+      - ".cargo/**"
+      - ".config/nextest.toml"
       - "deny.toml"
       - "crates/**"
       - "README.md"
@@ -115,6 +119,11 @@ jobs:
           rustc --version
           cargo --version
 
+      - name: Install cargo-nextest
+        run: |
+          cargo install cargo-nextest --version 0.9.133 --locked
+          cargo nextest --version
+
       - name: Install uv
         run: python -m pip install uv==0.9.30 maturin==1.12.6
 
@@ -125,11 +134,15 @@ jobs:
       - name: Plugin CI build verification
         if: matrix.os == 'ubuntu-latest'
         working-directory: plugins/rust/python-package/${{ matrix.plugin }}
+        env:
+          NEXTEST_PROFILE: ci
         run: make ci-build
 
       - name: Plugin CI verification
         if: matrix.os != 'ubuntu-latest'
         working-directory: plugins/rust/python-package/${{ matrix.plugin }}
+        env:
+          NEXTEST_PROFILE: ci
         run: make ci
 
   security-policy:
@@ -150,6 +163,50 @@ jobs:
       - name: Run cargo deny
         run: cargo deny check --config deny.toml
 
+  mutation-testing:
+    needs: validate-and-detect
+    if: github.event_name == 'pull_request' && needs.validate-and-detect.outputs.has_plugins == 'true'
+    strategy:
+      fail-fast: false
+      matrix:
+        cargo_package: ${{ fromJson(needs.validate-and-detect.outputs.cargo_packages) }}
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: bash
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
+        with:
+          fetch-depth: 0
+
+      - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405
+        with:
+          python-version: "3.12"
+
+      - name: Verify Rust toolchain
+        run: |
+          rustc --version
+          cargo --version
+
+      - name: Install Rust mutation testing tooling
+        run: |
+          cargo install cargo-nextest --version 0.9.133 --locked
+          cargo nextest --version
+          cargo install cargo-mutants --version 27.0.0 --locked
+          cargo mutants --version
+
+      - name: Create mutation diff
+        env:
+          BASE_SHA: ${{ github.event.pull_request.base.sha }}
+          HEAD_SHA: ${{ github.sha }}
+        run: git diff "${BASE_SHA}..${HEAD_SHA}" -- '*.rs' > cargo-mutants.diff
+
+      - name: Run cargo-mutants with nextest
+        env:
+          CARGO_PACKAGE: ${{ matrix.cargo_package }}
+          PYO3_PYTHON: python
+        run: cargo mutants -p "${CARGO_PACKAGE}" --in-diff cargo-mutants.diff
+
   coverage:
     needs: validate-and-detect
     if: needs.validate-and-detect.outputs.has_plugins == 'true'
@@ -168,13 +225,16 @@ jobs:
         run: |
           rustup component add llvm-tools-preview
           cargo install cargo-llvm-cov --version 0.8.4 --locked
+          cargo install cargo-nextest --version 0.9.133 --locked
+          cargo nextest --version
 
       - name: Install Python build tooling
         run: python -m pip install uv==0.9.30 maturin==1.12.6
 
       - name: Generate Rust coverage report
         env:
           CARGO_PACKAGES: ${{ needs.validate-and-detect.outputs.cargo_packages }}
+          NEXTEST_PROFILE: ci
           PLUGINS: ${{ needs.validate-and-detect.outputs.plugins }}
           PYO3_PYTHON: python
         run: |
@@ -186,6 +246,8 @@ jobs:
             cargo_args+=("-p" "${package}")
           done
           cargo llvm-cov clean --workspace
+          mkdir -p coverage
+          cargo llvm-cov nextest --no-report "${cargo_args[@]}" -P "${NEXTEST_PROFILE}"
           eval "$(cargo llvm-cov show-env --sh)"
           export CARGO_TARGET_DIR="${CARGO_LLVM_COV_TARGET_DIR}/llvm-cov-target"
           export CARGO_LLVM_COV_BUILD_DIR="${CARGO_TARGET_DIR}"
@@ -194,7 +256,6 @@ jobs:
           for plugin in "${plugins[@]}"; do
             (cd "plugins/rust/python-package/${plugin}" && make sync && uv run maturin develop)
           done
-          cargo test "${cargo_args[@]}"
           for plugin in "${plugins[@]}"; do
             (cd "plugins/rust/python-package/${plugin}" && make test-integration)
           done

.github/workflows/ci-scaffold-generator.yaml

@@ -27,9 +27,9 @@ jobs:
   test:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
 
-      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065
+      - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405
         with:
           python-version: "3.12"
 
@@ -52,6 +52,11 @@ jobs:
           rustc --version
           cargo --version
 
+      - name: Install cargo-nextest
+        run: |
+          cargo install cargo-nextest --version 0.9.133 --locked
+          cargo nextest --version
+
       - name: Generate default plugin (tool_pre_invoke)
         run: |
           python3 tools/scaffold_plugin.py --non-interactive \

Makefile

@@ -1,7 +1,7 @@
-.PHONY: help plugins-list plugins-validate plugin-test plugin-scaffold plugin-scaffold-help
+.PHONY: help plugins-list plugins-validate plugin-test plugin-mutants plugin-mutants-list plugin-scaffold plugin-scaffold-help
 
 help:
-	@printf "plugins-list\nplugins-validate\nplugin-test PLUGIN=<slug>\nplugin-scaffold\nplugin-scaffold-help\n"
+	@printf "plugins-list\nplugins-validate\nplugin-test PLUGIN=<slug>\nplugin-mutants PLUGIN=<slug>\nplugin-mutants-list PLUGIN=<slug>\nplugin-scaffold\nplugin-scaffold-help\n"
 
 plugins-list:
 	@python3 tools/plugin_catalog.py list .
@@ -14,6 +14,14 @@ plugin-test:
 	@test -n "$(PLUGIN)" || (echo "Set PLUGIN=<slug>" && exit 1)
 	@cd plugins/rust/python-package/$(PLUGIN) && make sync && make ci
 
+plugin-mutants:
+	@test -n "$(PLUGIN)" || (echo "Set PLUGIN=<slug>" && exit 1)
+	cargo mutants -p "$(PLUGIN)"
+
+plugin-mutants-list:
+	@test -n "$(PLUGIN)" || (echo "Set PLUGIN=<slug>" && exit 1)
+	cargo mutants --list -p "$(PLUGIN)"
+
 plugin-scaffold:
 	@python3 -m pip install --quiet jinja2 2>/dev/null || pip install --quiet jinja2 2>/dev/null || true
 	@python3 tools/scaffold_plugin.py

TESTING.md

@@ -30,10 +30,13 @@ Each plugin has its own Rust and Python test suite.
 ```bash
 cd plugins/rust/python-package/rate_limiter
 uv sync --dev
+cargo install cargo-nextest --version 0.9.133 --locked
 make install
 make test-all
 ```
 
+Set `NEXTEST_PROFILE=ci` to use the repository CI profile locally. The CI profile is defined in `.config/nextest.toml`; it disables fail-fast so all Rust test failures are reported in one run.
+
 Equivalent repo-level helper:
 
 ```bash
@@ -51,6 +54,7 @@ To run the same coverage check locally for all managed Rust plugins:
 ```bash
 rustup component add llvm-tools-preview
 cargo install cargo-llvm-cov --version 0.8.4 --locked
+cargo install cargo-nextest --version 0.9.133 --locked
 mkdir -p coverage
 CARGO_PACKAGES="$(python3 tools/plugin_catalog.py ci-selection-field . all '' '' cargo_packages)"
 PLUGINS="$(python3 tools/plugin_catalog.py ci-selection-field . all '' '' plugins)"
@@ -61,6 +65,7 @@ for package in "${cargo_packages[@]}"; do
   cargo_args+=("-p" "${package}")
 done
 cargo llvm-cov clean --workspace
+cargo llvm-cov nextest --no-report "${cargo_args[@]}" -P ci
 eval "$(cargo llvm-cov show-env --sh)"
 export CARGO_TARGET_DIR="${CARGO_LLVM_COV_TARGET_DIR}/llvm-cov-target"
 export CARGO_LLVM_COV_BUILD_DIR="${CARGO_TARGET_DIR}"
@@ -69,14 +74,29 @@ mkdir -p "${CARGO_TARGET_DIR}"
 for plugin in "${plugins[@]}"; do
   (cd "plugins/rust/python-package/${plugin}" && make sync && uv run maturin develop)
 done
-cargo test "${cargo_args[@]}"
 for plugin in "${plugins[@]}"; do
   (cd "plugins/rust/python-package/${plugin}" && make test-integration)
 done
 env -u CARGO_TARGET_DIR -u CARGO_LLVM_COV_BUILD_DIR -u CARGO_LLVM_COV_TARGET_DIR -u LLVM_PROFILE_FILE cargo llvm-cov report "${cargo_args[@]}" --cobertura --output-path coverage/cobertura.xml
 python3 tools/plugin_catalog.py coverage-check . coverage/cobertura.xml 90.00 "${PLUGINS}"
 ```
 
+Rust unit tests use `cargo nextest run`. Coverage uses `cargo llvm-cov nextest --no-report` for the Rust test phase, then runs pytest before generating the final report so PyO3 paths stay covered. CI uses the `ci` nextest profile, which disables fail-fast and prints failure output immediately and again at the end. Nextest does not run Rust doctests; this repo currently has no Rust doctest code blocks, so there is no separate doctest step.
+
+Criterion benchmarks are verified in CI with `cargo nextest run --benches`, which runs benchmarks in test mode rather than collecting noisy performance measurements on shared CI runners.
+
+## 4. Mutation Testing
+
+Mutation testing runs in PR CI on Ubuntu for Rust code touched by the pull request diff. It is also available locally through cargo-mutants and runs Rust tests with nextest.
+
+```bash
+cargo install cargo-mutants --version 27.0.0 --locked
+make plugin-mutants-list PLUGIN=retry_with_backoff
+make plugin-mutants PLUGIN=retry_with_backoff
+```
+
+`.cargo/mutants.toml` sets `test_tool = "nextest"`, selects the `mutants` nextest profile, and keeps `cap_lints = false` so Rust warnings are not downgraded during mutant builds. The `mutants` profile keeps fail-fast enabled because cargo-mutants only needs one failing test to mark a mutant as caught. CI installs `cargo-mutants` with `cargo install cargo-mutants --version 27.0.0 --locked` and runs `cargo mutants -p "${CARGO_PACKAGE}" --in-diff cargo-mutants.diff`.
+
 ## CI Behavior
 
 Repo contract tests run in their own CI workflow. The Rust plugin CI workflow uses the same plugin catalog to select affected plugin build, integration, and coverage jobs.

plugins/rust/python-package/encoded_exfil_detection/Makefile

@@ -5,6 +5,8 @@ help:
 PACKAGE_NAME := cpex-encoded-exfil-detection
 WHEEL_PREFIX := cpex_encoded_exfil_detection
 CARGO := cargo
+CARGO_PACKAGE := encoded_exfil_detection
+NEXTEST_PROFILE ?= default
 STUB_FILES := cpex_encoded_exfil_detection/__init__.pyi cpex_encoded_exfil_detection/encoded_exfil_detection_rust/__init__.pyi
 WHEEL_DIR := ../../../../target/wheels
 
@@ -38,7 +40,7 @@ sync:
 
 test-unit:
 	@echo "$(GREEN)Running encoded_exfil_detection Rust tests...$(NC)"
-	$(CARGO) test
+	$(CARGO) nextest run --profile $(NEXTEST_PROFILE) -p $(CARGO_PACKAGE)
 
 test: test-unit test-integration
 
@@ -85,16 +87,16 @@ uninstall:
 	@uv pip uninstall -y $(PACKAGE_NAME) 2>/dev/null || true
 
 # help: bench                 - Run Criterion benchmarks
-# help: bench-no-run          - Compile Criterion benchmarks without running them
+# help: bench-no-run          - Run Criterion benchmarks in nextest test mode
 .PHONY: bench bench-no-run
 
 bench:
 	@echo "$(GREEN)Running benchmarks...$(NC)"
 	$(CARGO) bench
 
 bench-no-run:
-	@echo "$(GREEN)Compiling benchmarks without running them...$(NC)"
-	$(CARGO) bench --no-run
+	@echo "$(GREEN)Running Criterion benchmarks in nextest test mode...$(NC)"
+	$(CARGO) nextest run --profile $(NEXTEST_PROFILE) -p $(CARGO_PACKAGE) --benches
 
 .PHONY: clean clean-all
 

plugins/rust/python-package/pii_filter/Makefile

@@ -5,6 +5,8 @@ help:
 PACKAGE_NAME := cpex-pii-filter
 WHEEL_PREFIX := cpex_pii_filter
 CARGO := cargo
+CARGO_PACKAGE := pii_filter
+NEXTEST_PROFILE ?= default
 STUB_FILES := cpex_pii_filter/__init__.pyi cpex_pii_filter/pii_filter_rust/__init__.pyi
 WHEEL_DIR := ../../../../target/wheels
 
@@ -39,13 +41,13 @@ sync:
 
 test-unit:
 	@echo "$(GREEN)Running pii_filter Rust tests...$(NC)"
-	$(CARGO) test
+	$(CARGO) nextest run --profile $(NEXTEST_PROFILE) -p $(CARGO_PACKAGE)
 
 test: test-unit test-integration
 
 test-verbose:
 	@echo "$(GREEN)Running pii_filter Rust tests (verbose)...$(NC)"
-	$(CARGO) test -- --nocapture
+	$(CARGO) nextest run --profile $(NEXTEST_PROFILE) -p $(CARGO_PACKAGE) --no-capture
 
 test-python:
 	$(MAKE) test-integration
@@ -90,7 +92,7 @@ uninstall:
 	@uv pip uninstall -y $(PACKAGE_NAME) 2>/dev/null || true
 
 # help: bench                 - Run Criterion benchmarks
-# help: bench-no-run          - Compile Criterion benchmarks without running them
+# help: bench-no-run          - Run Criterion benchmarks in nextest test mode
 # help: bench-compare         - Compare against saved baseline
 .PHONY: bench bench-no-run bench-baseline bench-compare
 
@@ -99,8 +101,8 @@ bench:
 	$(CARGO) bench
 
 bench-no-run:
-	@echo "$(GREEN)Compiling benchmarks without running them...$(NC)"
-	$(CARGO) bench --no-run
+	@echo "$(GREEN)Running Criterion benchmarks in nextest test mode...$(NC)"
+	$(CARGO) nextest run --profile $(NEXTEST_PROFILE) -p $(CARGO_PACKAGE) --benches
 
 bench-baseline:
 	$(CARGO) bench --bench pii_filter -- --save-baseline main

plugins/rust/python-package/rate_limiter/Makefile

@@ -13,6 +13,8 @@ help:
 PACKAGE_NAME := cpex-rate-limiter
 WHEEL_PREFIX := cpex_rate_limiter
 CARGO := cargo
+CARGO_PACKAGE := rate_limiter
+NEXTEST_PROFILE ?= default
 STUB_FILES := cpex_rate_limiter/__init__.pyi cpex_rate_limiter/rate_limiter_rust/__init__.pyi
 WHEEL_DIR := ../../../../target/wheels
 
@@ -53,13 +55,13 @@ sync:
 
 test-unit:
 	@echo "$(GREEN)Running rate_limiter Rust tests...$(NC)"
-	$(CARGO) test
+	$(CARGO) nextest run --profile $(NEXTEST_PROFILE) -p $(CARGO_PACKAGE)
 
 test: test-unit test-integration
 
 test-verbose:
 	@echo "$(GREEN)Running rate_limiter Rust tests (verbose)...$(NC)"
-	$(CARGO) test -- --nocapture
+	$(CARGO) nextest run --profile $(NEXTEST_PROFILE) -p $(CARGO_PACKAGE) --no-capture
 
 test-python:
 	$(MAKE) test-integration
@@ -110,7 +112,7 @@ uninstall:
 # 📊 BENCHMARKS
 # =============================================================================
 # help: bench                 - Run Criterion benchmarks
-# help: bench-no-run          - Compile Criterion benchmarks without running them
+# help: bench-no-run          - Run Criterion benchmarks in nextest test mode
 # help: bench-compare         - Compare against saved baseline
 .PHONY: bench bench-no-run bench-baseline bench-compare
 
@@ -119,8 +121,8 @@ bench:
 	$(CARGO) bench
 
 bench-no-run:
-	@echo "$(GREEN)Compiling benchmarks without running them...$(NC)"
-	$(CARGO) bench --no-run
+	@echo "$(GREEN)Running Criterion benchmarks in nextest test mode...$(NC)"
+	$(CARGO) nextest run --profile $(NEXTEST_PROFILE) -p $(CARGO_PACKAGE) --benches
 
 bench-baseline:
 	$(CARGO) bench --bench rate_limiter -- --save-baseline main

plugins/rust/python-package/rate_limiter/src/plugin.rs

@@ -523,6 +523,7 @@ mod tests {
 
     #[test]
     fn await_async_tuple_parses_successful_result() -> PyResult<()> {
+        Python::initialize();
         Python::attach(|py| -> PyResult<()> {
             let sys = py.import("sys")?;
             let asyncio = py.import("asyncio")?;