Skip to content

Commit 1edf031

Browse files
committed
Use cargo-nextest for Rust tests
Signed-off-by: lucarlig <luca.carlig@ibm.com>
1 parent 5e7d9f6 commit 1edf031

18 files changed

Lines changed: 806 additions & 66 deletions

File tree

.cargo/mutants.toml

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
# SPDX-License-Identifier: Apache-2.0
2+
3+
test_tool = "nextest"
4+
additional_cargo_test_args = ["--profile=mutants"]
5+
cap_lints = false

.config/nextest.toml

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
# SPDX-License-Identifier: Apache-2.0
2+
3+
nextest-version = "0.9.133"
4+
5+
[profile.ci]
6+
fail-fast = false
7+
failure-output = "immediate-final"
8+
9+
[profile.mutants]
10+
fail-fast = true
11+
failure-output = "final"

.github/workflows/ci-rust-python-package.yaml

Lines changed: 87 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,8 @@ on:
77
- "Makefile"
88
- "Cargo.toml"
99
- "Cargo.lock"
10+
- ".cargo/**"
11+
- ".config/nextest.toml"
1012
- "deny.toml"
1113
- "crates/**"
1214
- "README.md"
@@ -23,6 +25,8 @@ on:
2325
- "Makefile"
2426
- "Cargo.toml"
2527
- "Cargo.lock"
28+
- ".cargo/**"
29+
- ".config/nextest.toml"
2630
- "deny.toml"
2731
- "crates/**"
2832
- "README.md"
@@ -51,6 +55,9 @@ jobs:
5155
has_plugins: ${{ steps.detect.outputs.has_plugins }}
5256
plugin_count: ${{ steps.detect.outputs.plugin_count }}
5357
cargo_packages: ${{ steps.detect.outputs.cargo_packages }}
58+
mutation_cargo_packages: ${{ steps.detect.outputs.mutation_cargo_packages }}
59+
mutation_jobs: ${{ steps.detect.outputs.mutation_jobs }}
60+
has_mutation_cargo_packages: ${{ steps.detect.outputs.has_mutation_cargo_packages }}
5461
steps:
5562
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
5663
with:
@@ -74,21 +81,31 @@ jobs:
7481
selection="$(python3 tools/plugin_catalog.py ci-selection . diff "${{ github.event.before }}" "${{ github.sha }}")"
7582
fi
7683
77-
selection="$(printf '%s' "${selection}" | python3 -c 'import json, re, sys; payload = json.load(sys.stdin); slug_re = re.compile(r"^[a-z0-9_]+$"); plugins = payload.get("plugins"); cargo_packages = payload.get("cargo_packages"); has_plugins = payload.get("has_plugins"); plugin_count = payload.get("plugin_count"); assert isinstance(plugins, list) and all(isinstance(item, str) and slug_re.fullmatch(item) for item in plugins); assert isinstance(cargo_packages, list) and all(isinstance(item, str) and slug_re.fullmatch(item) for item in cargo_packages); assert isinstance(has_plugins, bool); assert isinstance(plugin_count, int) and plugin_count == len(plugins); print(json.dumps({"plugins": plugins, "has_plugins": has_plugins, "plugin_count": plugin_count, "cargo_packages": cargo_packages}))')"
84+
selection="$(printf '%s' "${selection}" | python3 -c 'import json, re, sys; payload = json.load(sys.stdin); slug_re = re.compile(r"^[a-z0-9_]+$"); plugins = payload.get("plugins"); cargo_packages = payload.get("cargo_packages"); mutation_cargo_packages = payload.get("mutation_cargo_packages"); mutation_jobs = payload.get("mutation_jobs"); has_plugins = payload.get("has_plugins"); plugin_count = payload.get("plugin_count"); assert isinstance(plugins, list) and all(isinstance(item, str) and slug_re.fullmatch(item) for item in plugins); assert isinstance(cargo_packages, list) and all(isinstance(item, str) and slug_re.fullmatch(item) for item in cargo_packages); assert isinstance(mutation_cargo_packages, list) and all(isinstance(item, str) and slug_re.fullmatch(item) for item in mutation_cargo_packages); assert isinstance(mutation_jobs, list); [(_ for _ in ()).throw(AssertionError()) for job in mutation_jobs if not (isinstance(job, dict) and isinstance(job.get("cargo_package"), str) and slug_re.fullmatch(job["cargo_package"]) and isinstance(job.get("in_diff"), bool) and isinstance(job.get("test_packages"), list) and all(isinstance(item, str) and slug_re.fullmatch(item) for item in job["test_packages"]))]; assert isinstance(has_plugins, bool); assert isinstance(plugin_count, int) and plugin_count == len(plugins); print(json.dumps({"plugins": plugins, "has_plugins": has_plugins, "plugin_count": plugin_count, "cargo_packages": cargo_packages, "mutation_cargo_packages": mutation_cargo_packages, "mutation_jobs": mutation_jobs}))')"
7885
plugins="$(printf '%s' "${selection}" | python3 -c 'import json, sys; print(json.dumps(json.load(sys.stdin)["plugins"]))')"
7986
has_plugins="$(printf '%s' "${selection}" | python3 -c 'import json, sys; print(str(json.load(sys.stdin)["has_plugins"]).lower())')"
8087
plugin_count="$(printf '%s' "${selection}" | python3 -c 'import json, sys; print(json.load(sys.stdin)["plugin_count"])')"
8188
cargo_packages="$(printf '%s' "${selection}" | python3 -c 'import json, sys; print(json.dumps(json.load(sys.stdin)["cargo_packages"]))')"
89+
mutation_cargo_packages="$(printf '%s' "${selection}" | python3 -c 'import json, sys; print(json.dumps(json.load(sys.stdin)["mutation_cargo_packages"]))')"
90+
mutation_jobs="$(printf '%s' "${selection}" | python3 -c 'import json, sys; print(json.dumps(json.load(sys.stdin)["mutation_jobs"]))')"
8291
if [[ "${has_plugins}" == "false" ]]; then
8392
has_plugins_output="false"
8493
else
8594
has_plugins_output="true"
8695
fi
96+
if [[ "${mutation_cargo_packages}" == "[]" ]]; then
97+
has_mutation_cargo_packages_output="false"
98+
else
99+
has_mutation_cargo_packages_output="true"
100+
fi
87101
{
88102
echo "plugins=${plugins}"
89103
echo "plugin_count=${plugin_count}"
90104
echo "cargo_packages=${cargo_packages}"
91105
echo "has_plugins=${has_plugins_output}"
106+
echo "mutation_cargo_packages=${mutation_cargo_packages}"
107+
echo "mutation_jobs=${mutation_jobs}"
108+
echo "has_mutation_cargo_packages=${has_mutation_cargo_packages_output}"
92109
} >> "$GITHUB_OUTPUT"
93110
94111
build-test:
@@ -115,6 +132,11 @@ jobs:
115132
rustc --version
116133
cargo --version
117134
135+
- name: Install cargo-nextest
136+
run: |
137+
cargo install cargo-nextest --version 0.9.133 --locked
138+
cargo nextest --version
139+
118140
- name: Install uv
119141
run: python -m pip install uv==0.9.30 maturin==1.12.6
120142

@@ -125,11 +147,15 @@ jobs:
125147
- name: Plugin CI build verification
126148
if: matrix.os == 'ubuntu-latest'
127149
working-directory: plugins/rust/python-package/${{ matrix.plugin }}
150+
env:
151+
NEXTEST_PROFILE: ci
128152
run: make ci-build
129153

130154
- name: Plugin CI verification
131155
if: matrix.os != 'ubuntu-latest'
132156
working-directory: plugins/rust/python-package/${{ matrix.plugin }}
157+
env:
158+
NEXTEST_PROFILE: ci
133159
run: make ci
134160

135161
security-policy:
@@ -150,6 +176,61 @@ jobs:
150176
- name: Run cargo deny
151177
run: cargo deny check --config deny.toml
152178

179+
mutation-testing:
180+
needs: validate-and-detect
181+
if: github.event_name == 'pull_request' && needs.validate-and-detect.outputs.has_mutation_cargo_packages == 'true'
182+
strategy:
183+
fail-fast: false
184+
matrix:
185+
mutation_job: ${{ fromJson(needs.validate-and-detect.outputs.mutation_jobs) }}
186+
runs-on: ubuntu-latest
187+
defaults:
188+
run:
189+
shell: bash
190+
steps:
191+
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
192+
with:
193+
fetch-depth: 0
194+
195+
- uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405
196+
with:
197+
python-version: "3.12"
198+
199+
- name: Verify Rust toolchain
200+
run: |
201+
rustc --version
202+
cargo --version
203+
204+
- name: Install Rust mutation testing tooling
205+
run: |
206+
cargo install cargo-nextest --version 0.9.133 --locked
207+
cargo nextest --version
208+
cargo install cargo-mutants --version 27.0.0 --locked
209+
cargo mutants --version
210+
211+
- name: Create mutation diff
212+
env:
213+
BASE_SHA: ${{ github.event.pull_request.base.sha }}
214+
HEAD_SHA: ${{ github.sha }}
215+
run: git diff "${BASE_SHA}..${HEAD_SHA}" -- '*.rs' > cargo-mutants.diff
216+
217+
- name: Run cargo-mutants with nextest
218+
env:
219+
CARGO_PACKAGE: ${{ matrix.mutation_job.cargo_package }}
220+
IN_DIFF: ${{ matrix.mutation_job.in_diff }}
221+
PYO3_PYTHON: python
222+
TEST_PACKAGES: ${{ toJson(matrix.mutation_job.test_packages) }}
223+
run: |
224+
mapfile -t test_packages < <(python3 -c 'import json, os; [print(package) for package in json.loads(os.environ["TEST_PACKAGES"])]')
225+
cargo_args=("-p" "${CARGO_PACKAGE}")
226+
if [[ "${IN_DIFF}" == "true" ]]; then
227+
cargo_args+=("--in-diff" "cargo-mutants.diff")
228+
fi
229+
for package in "${test_packages[@]}"; do
230+
cargo_args+=("--test-package" "${package}")
231+
done
232+
cargo mutants "${cargo_args[@]}"
233+
153234
coverage:
154235
needs: validate-and-detect
155236
if: needs.validate-and-detect.outputs.has_plugins == 'true'
@@ -168,13 +249,16 @@ jobs:
168249
run: |
169250
rustup component add llvm-tools-preview
170251
cargo install cargo-llvm-cov --version 0.8.4 --locked
252+
cargo install cargo-nextest --version 0.9.133 --locked
253+
cargo nextest --version
171254
172255
- name: Install Python build tooling
173256
run: python -m pip install uv==0.9.30 maturin==1.12.6
174257

175258
- name: Generate Rust coverage report
176259
env:
177260
CARGO_PACKAGES: ${{ needs.validate-and-detect.outputs.cargo_packages }}
261+
NEXTEST_PROFILE: ci
178262
PLUGINS: ${{ needs.validate-and-detect.outputs.plugins }}
179263
PYO3_PYTHON: python
180264
run: |
@@ -186,6 +270,8 @@ jobs:
186270
cargo_args+=("-p" "${package}")
187271
done
188272
cargo llvm-cov clean --workspace
273+
mkdir -p coverage
274+
cargo llvm-cov nextest --no-report "${cargo_args[@]}" -P "${NEXTEST_PROFILE}"
189275
eval "$(cargo llvm-cov show-env --sh)"
190276
export CARGO_TARGET_DIR="${CARGO_LLVM_COV_TARGET_DIR}/llvm-cov-target"
191277
export CARGO_LLVM_COV_BUILD_DIR="${CARGO_TARGET_DIR}"
@@ -194,7 +280,6 @@ jobs:
194280
for plugin in "${plugins[@]}"; do
195281
(cd "plugins/rust/python-package/${plugin}" && make sync && uv run maturin develop)
196282
done
197-
cargo test "${cargo_args[@]}"
198283
for plugin in "${plugins[@]}"; do
199284
(cd "plugins/rust/python-package/${plugin}" && make test-integration)
200285
done

.github/workflows/ci-scaffold-generator.yaml

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -27,9 +27,9 @@ jobs:
2727
test:
2828
runs-on: ubuntu-latest
2929
steps:
30-
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5
30+
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
3131

32-
- uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065
32+
- uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405
3333
with:
3434
python-version: "3.12"
3535

@@ -52,6 +52,11 @@ jobs:
5252
rustc --version
5353
cargo --version
5454
55+
- name: Install cargo-nextest
56+
run: |
57+
cargo install cargo-nextest --version 0.9.133 --locked
58+
cargo nextest --version
59+
5560
- name: Generate default plugin (tool_pre_invoke)
5661
run: |
5762
python3 tools/scaffold_plugin.py --non-interactive \

Makefile

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
1-
.PHONY: help plugins-list plugins-validate plugin-test plugin-scaffold plugin-scaffold-help
1+
.PHONY: help plugins-list plugins-validate plugin-test plugin-mutants plugin-mutants-list plugin-scaffold plugin-scaffold-help
22

33
help:
4-
@printf "plugins-list\nplugins-validate\nplugin-test PLUGIN=<slug>\nplugin-scaffold\nplugin-scaffold-help\n"
4+
@printf "plugins-list\nplugins-validate\nplugin-test PLUGIN=<slug>\nplugin-mutants PLUGIN=<slug>\nplugin-mutants-list PLUGIN=<slug>\nplugin-scaffold\nplugin-scaffold-help\n"
55

66
plugins-list:
77
@python3 tools/plugin_catalog.py list .
@@ -14,6 +14,14 @@ plugin-test:
1414
@test -n "$(PLUGIN)" || (echo "Set PLUGIN=<slug>" && exit 1)
1515
@cd plugins/rust/python-package/$(PLUGIN) && make sync && make ci
1616

17+
plugin-mutants:
18+
@test -n "$(PLUGIN)" || (echo "Set PLUGIN=<slug>" && exit 1)
19+
cargo mutants -p "$(PLUGIN)"
20+
21+
plugin-mutants-list:
22+
@test -n "$(PLUGIN)" || (echo "Set PLUGIN=<slug>" && exit 1)
23+
cargo mutants --list -p "$(PLUGIN)"
24+
1725
plugin-scaffold:
1826
@python3 -m pip install --quiet jinja2 2>/dev/null || pip install --quiet jinja2 2>/dev/null || true
1927
@python3 tools/scaffold_plugin.py

TESTING.md

Lines changed: 22 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,10 +30,13 @@ Each plugin has its own Rust and Python test suite.
3030
```bash
3131
cd plugins/rust/python-package/rate_limiter
3232
uv sync --dev
33+
cargo install cargo-nextest --version 0.9.133 --locked
3334
make install
3435
make test-all
3536
```
3637

38+
Set `NEXTEST_PROFILE=ci` to use the repository CI profile locally. The CI profile is defined in `.config/nextest.toml`; it disables fail-fast so all Rust test failures are reported in one run.
39+
3740
Equivalent repo-level helper:
3841

3942
```bash
@@ -51,6 +54,7 @@ To run the same coverage check locally for all managed Rust plugins:
5154
```bash
5255
rustup component add llvm-tools-preview
5356
cargo install cargo-llvm-cov --version 0.8.4 --locked
57+
cargo install cargo-nextest --version 0.9.133 --locked
5458
mkdir -p coverage
5559
CARGO_PACKAGES="$(python3 tools/plugin_catalog.py ci-selection-field . all '' '' cargo_packages)"
5660
PLUGINS="$(python3 tools/plugin_catalog.py ci-selection-field . all '' '' plugins)"
@@ -61,6 +65,7 @@ for package in "${cargo_packages[@]}"; do
6165
cargo_args+=("-p" "${package}")
6266
done
6367
cargo llvm-cov clean --workspace
68+
cargo llvm-cov nextest --no-report "${cargo_args[@]}" -P ci
6469
eval "$(cargo llvm-cov show-env --sh)"
6570
export CARGO_TARGET_DIR="${CARGO_LLVM_COV_TARGET_DIR}/llvm-cov-target"
6671
export CARGO_LLVM_COV_BUILD_DIR="${CARGO_TARGET_DIR}"
@@ -69,14 +74,30 @@ mkdir -p "${CARGO_TARGET_DIR}"
6974
for plugin in "${plugins[@]}"; do
7075
(cd "plugins/rust/python-package/${plugin}" && make sync && uv run maturin develop)
7176
done
72-
cargo test "${cargo_args[@]}"
7377
for plugin in "${plugins[@]}"; do
7478
(cd "plugins/rust/python-package/${plugin}" && make test-integration)
7579
done
7680
env -u CARGO_TARGET_DIR -u CARGO_LLVM_COV_BUILD_DIR -u CARGO_LLVM_COV_TARGET_DIR -u LLVM_PROFILE_FILE cargo llvm-cov report "${cargo_args[@]}" --cobertura --output-path coverage/cobertura.xml
7781
python3 tools/plugin_catalog.py coverage-check . coverage/cobertura.xml 90.00 "${PLUGINS}"
7882
```
7983

84+
Rust unit tests use `cargo nextest run`. Coverage uses `cargo llvm-cov nextest --no-report` for the Rust test phase, then runs pytest before generating the final report so PyO3 paths stay covered. CI uses the `ci` nextest profile, which disables fail-fast and prints failure output immediately and again at the end. Nextest does not run Rust doctests; this repo currently has no Rust doctest code blocks, so there is no separate doctest step.
85+
86+
Criterion benchmarks are verified in CI with `cargo nextest run --benches -E 'kind(bench)' --no-run`, which compiles benchmark test targets without rerunning normal unit tests or collecting noisy performance measurements on shared CI runners.
87+
88+
## 4. Mutation Testing
89+
90+
Mutation testing runs in PR CI on Ubuntu for Rust code touched by the pull request diff. It is also available locally through cargo-mutants and runs Rust tests with nextest.
91+
92+
```bash
93+
cargo install cargo-nextest --version 0.9.133 --locked
94+
cargo install cargo-mutants --version 27.0.0 --locked
95+
make plugin-mutants-list PLUGIN=retry_with_backoff
96+
make plugin-mutants PLUGIN=retry_with_backoff
97+
```
98+
99+
`.cargo/mutants.toml` sets `test_tool = "nextest"`, selects the `mutants` nextest profile, and keeps `cap_lints = false` so Rust warnings are not downgraded during mutant builds. The `mutants` profile keeps fail-fast enabled because cargo-mutants only needs one failing test to mark a mutant as caught. CI installs `cargo-mutants` with `cargo install cargo-mutants --version 27.0.0 --locked` and runs `cargo mutants "${cargo_args[@]}"`, using `--in-diff cargo-mutants.diff` for Rust source changes and full-package mutation for mutation-tooling config changes.
100+
80101
## CI Behavior
81102

82103
Repo contract tests run in their own CI workflow. The Rust plugin CI workflow uses the same plugin catalog to select affected plugin build, integration, and coverage jobs.

plugins/rust/python-package/encoded_exfil_detection/Makefile

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@ help:
55
PACKAGE_NAME := cpex-encoded-exfil-detection
66
WHEEL_PREFIX := cpex_encoded_exfil_detection
77
CARGO := cargo
8+
CARGO_PACKAGE := encoded_exfil_detection
9+
NEXTEST_PROFILE ?= default
810
STUB_FILES := cpex_encoded_exfil_detection/__init__.pyi cpex_encoded_exfil_detection/encoded_exfil_detection_rust/__init__.pyi
911
WHEEL_DIR := ../../../../target/wheels
1012

@@ -38,7 +40,7 @@ sync:
3840

3941
test-unit:
4042
@echo "$(GREEN)Running encoded_exfil_detection Rust tests...$(NC)"
41-
$(CARGO) test
43+
$(CARGO) nextest run --profile $(NEXTEST_PROFILE) -p $(CARGO_PACKAGE)
4244

4345
test: test-unit test-integration
4446

@@ -85,16 +87,16 @@ uninstall:
8587
@uv pip uninstall -y $(PACKAGE_NAME) 2>/dev/null || true
8688

8789
# help: bench - Run Criterion benchmarks
88-
# help: bench-no-run - Compile Criterion benchmarks without running them
90+
# help: bench-no-run - Compile Criterion benchmark targets with nextest
8991
.PHONY: bench bench-no-run
9092

9193
bench:
9294
@echo "$(GREEN)Running benchmarks...$(NC)"
9395
$(CARGO) bench
9496

9597
bench-no-run:
96-
@echo "$(GREEN)Compiling benchmarks without running them...$(NC)"
97-
$(CARGO) bench --no-run
98+
@echo "$(GREEN)Compiling Criterion benchmark targets with nextest...$(NC)"
99+
$(CARGO) nextest run --profile $(NEXTEST_PROFILE) -p $(CARGO_PACKAGE) --benches -E 'kind(bench)' --no-run
98100

99101
.PHONY: clean clean-all
100102

plugins/rust/python-package/pii_filter/Makefile

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@ help:
55
PACKAGE_NAME := cpex-pii-filter
66
WHEEL_PREFIX := cpex_pii_filter
77
CARGO := cargo
8+
CARGO_PACKAGE := pii_filter
9+
NEXTEST_PROFILE ?= default
810
STUB_FILES := cpex_pii_filter/__init__.pyi cpex_pii_filter/pii_filter_rust/__init__.pyi
911
WHEEL_DIR := ../../../../target/wheels
1012

@@ -39,13 +41,13 @@ sync:
3941

4042
test-unit:
4143
@echo "$(GREEN)Running pii_filter Rust tests...$(NC)"
42-
$(CARGO) test
44+
$(CARGO) nextest run --profile $(NEXTEST_PROFILE) -p $(CARGO_PACKAGE)
4345

4446
test: test-unit test-integration
4547

4648
test-verbose:
4749
@echo "$(GREEN)Running pii_filter Rust tests (verbose)...$(NC)"
48-
$(CARGO) test -- --nocapture
50+
$(CARGO) nextest run --profile $(NEXTEST_PROFILE) -p $(CARGO_PACKAGE) --no-capture
4951

5052
test-python:
5153
$(MAKE) test-integration
@@ -90,7 +92,7 @@ uninstall:
9092
@uv pip uninstall -y $(PACKAGE_NAME) 2>/dev/null || true
9193

9294
# help: bench - Run Criterion benchmarks
93-
# help: bench-no-run - Compile Criterion benchmarks without running them
95+
# help: bench-no-run - Compile Criterion benchmark targets with nextest
9496
# help: bench-compare - Compare against saved baseline
9597
.PHONY: bench bench-no-run bench-baseline bench-compare
9698

@@ -99,8 +101,8 @@ bench:
99101
$(CARGO) bench
100102

101103
bench-no-run:
102-
@echo "$(GREEN)Compiling benchmarks without running them...$(NC)"
103-
$(CARGO) bench --no-run
104+
@echo "$(GREEN)Compiling Criterion benchmark targets with nextest...$(NC)"
105+
$(CARGO) nextest run --profile $(NEXTEST_PROFILE) -p $(CARGO_PACKAGE) --benches -E 'kind(bench)' --no-run
104106

105107
bench-baseline:
106108
$(CARGO) bench --bench pii_filter -- --save-baseline main

0 commit comments

Comments
 (0)