Skip to content

Commit 590d5c7

Browse files
lucarliglucarlig
authored
fix: add missing Rust CI parity checks (#25)
* fix: add missing Rust CI parity checks Signed-off-by: lucarlig <luca.carlig@ibm.com> * fix: restore workspace cargo-deny config Signed-off-by: lucarlig <luca.carlig@ibm.com> * fix: run shared rust ci checks at workspace scope Signed-off-by: lucarlig <luca.carlig@ibm.com> * fix: scope shared rust ci to changed plugins Signed-off-by: lucarlig <luca.carlig@ibm.com> * fix: harden rust plugin ci parity checks Signed-off-by: lucarlig <luca.carlig@ibm.com> * fix: use valid cargo tool install syntax in ci Signed-off-by: lucarlig <luca.carlig@ibm.com> * fix: limit ci docs checks to library targets Signed-off-by: lucarlig <luca.carlig@ibm.com> * fix: silence pii filter rustdoc warning Signed-off-by: lucarlig <luca.carlig@ibm.com> * fix: stabilize pii filter stub generation Signed-off-by: lucarlig <luca.carlig@ibm.com> * fix: remove duplicate url reputation deny config Signed-off-by: lucarlig <luca.carlig@ibm.com> * fix: drop cargo deny from rust ci Signed-off-by: lucarlig <luca.carlig@ibm.com> * fix: use cargo deny for rust security policy Signed-off-by: lucarlig <luca.carlig@ibm.com> * fix: enforce per-plugin rust coverage floor Signed-off-by: lucarlig <luca.carlig@ibm.com> * fix: set rust coverage floor to 50 percent Signed-off-by: lucarlig <luca.carlig@ibm.com> * fix: address rust ci review findings Signed-off-by: lucarlig <luca.carlig@ibm.com> * test: exercise rust ci workflow dispatch Signed-off-by: lucarlig <luca.carlig@ibm.com> * fix: address rust ci review findings Signed-off-by: lucarlig <luca.carlig@ibm.com> --------- Signed-off-by: lucarlig <luca.carlig@ibm.com>
1 parent 51f2bb7 commit 590d5c7

16 files changed

Lines changed: 1059 additions & 128 deletions

File tree

‎.github/workflows/ci-rust-python-package.yaml‎

Lines changed: 107 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,8 @@ on:
66
paths:
77
- "Makefile"
88
- "Cargo.toml"
9+
- "Cargo.lock"
10+
- "deny.toml"
911
- "crates/**"
1012
- "README.md"
1113
- "DEVELOPING.md"
@@ -20,6 +22,8 @@ on:
2022
paths:
2123
- "Makefile"
2224
- "Cargo.toml"
25+
- "Cargo.lock"
26+
- "deny.toml"
2327
- "crates/**"
2428
- "README.md"
2529
- "DEVELOPING.md"
@@ -29,6 +33,7 @@ on:
2933
- "tools/**"
3034
- ".github/workflows/ci-rust-python-package.yaml"
3135
- ".github/workflows/release-rust-python-package.yaml"
36+
workflow_dispatch:
3237

3338
concurrency:
3439
group: ci-rust-python-package-${{ github.event.pull_request.head.repo.full_name || github.repository }}-${{ github.head_ref || github.ref_name }}
@@ -44,6 +49,8 @@ jobs:
4449
outputs:
4550
plugins: ${{ steps.detect.outputs.plugins }}
4651
has_plugins: ${{ steps.detect.outputs.has_plugins }}
52+
plugin_count: ${{ steps.detect.outputs.plugin_count }}
53+
cargo_packages: ${{ steps.detect.outputs.cargo_packages }}
4754
steps:
4855
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5
4956
with:
@@ -59,20 +66,30 @@ jobs:
5966
set -euo pipefail
6067
if [[ "${GITHUB_EVENT_NAME}" == "pull_request" ]]; then
6168
selection="$(python3 tools/plugin_catalog.py ci-selection . diff "${{ github.event.pull_request.base.sha }}" "${{ github.sha }}")"
69+
elif [[ "${GITHUB_EVENT_NAME}" == "workflow_dispatch" ]]; then
70+
selection="$(python3 tools/plugin_catalog.py ci-selection . all '' '')"
6271
elif [[ "${{ github.event.before }}" == "0000000000000000000000000000000000000000" ]]; then
6372
selection="$(python3 tools/plugin_catalog.py ci-selection . all '' '')"
6473
else
6574
selection="$(python3 tools/plugin_catalog.py ci-selection . diff "${{ github.event.before }}" "${{ github.sha }}")"
6675
fi
6776
77+
selection="$(printf '%s' "${selection}" | python3 -c 'import json, re, sys; payload = json.load(sys.stdin); slug_re = re.compile(r"^[a-z0-9_]+$"); plugins = payload.get("plugins"); cargo_packages = payload.get("cargo_packages"); has_plugins = payload.get("has_plugins"); plugin_count = payload.get("plugin_count"); assert isinstance(plugins, list) and all(isinstance(item, str) and slug_re.fullmatch(item) for item in plugins); assert isinstance(cargo_packages, list) and all(isinstance(item, str) and slug_re.fullmatch(item) for item in cargo_packages); assert isinstance(has_plugins, bool); assert isinstance(plugin_count, int) and plugin_count == len(plugins); print(json.dumps({"plugins": plugins, "has_plugins": has_plugins, "plugin_count": plugin_count, "cargo_packages": cargo_packages}))')"
6878
plugins="$(printf '%s' "${selection}" | python3 -c 'import json, sys; print(json.dumps(json.load(sys.stdin)["plugins"]))')"
6979
has_plugins="$(printf '%s' "${selection}" | python3 -c 'import json, sys; print(str(json.load(sys.stdin)["has_plugins"]).lower())')"
70-
echo "plugins=${plugins}" >> "$GITHUB_OUTPUT"
80+
plugin_count="$(printf '%s' "${selection}" | python3 -c 'import json, sys; print(json.load(sys.stdin)["plugin_count"])')"
81+
cargo_packages="$(printf '%s' "${selection}" | python3 -c 'import json, sys; print(json.dumps(json.load(sys.stdin)["cargo_packages"]))')"
7182
if [[ "${has_plugins}" == "false" ]]; then
72-
echo "has_plugins=false" >> "$GITHUB_OUTPUT"
83+
has_plugins_output="false"
7384
else
74-
echo "has_plugins=true" >> "$GITHUB_OUTPUT"
85+
has_plugins_output="true"
7586
fi
87+
{
88+
echo "plugins=${plugins}"
89+
echo "plugin_count=${plugin_count}"
90+
echo "cargo_packages=${cargo_packages}"
91+
echo "has_plugins=${has_plugins_output}"
92+
} >> "$GITHUB_OUTPUT"
7693
7794
build-test:
7895
needs: validate-and-detect
@@ -109,6 +126,93 @@ jobs:
109126
working-directory: plugins/rust/python-package/${{ matrix.plugin }}
110127
run: make ci
111128

129+
security-policy:
130+
needs: validate-and-detect
131+
if: needs.validate-and-detect.outputs.has_plugins == 'true'
132+
runs-on: ubuntu-latest
133+
steps:
134+
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5
135+
136+
- name: Verify Rust toolchain
137+
run: |
138+
rustc --version
139+
cargo --version
140+
141+
- name: Install cargo deny
142+
run: cargo install cargo-deny@0.19.0 --locked
143+
144+
- name: Run cargo deny
145+
run: cargo deny check --config deny.toml
146+
147+
coverage:
148+
needs: validate-and-detect
149+
if: needs.validate-and-detect.outputs.has_plugins == 'true'
150+
runs-on: ubuntu-latest
151+
defaults:
152+
run:
153+
shell: bash
154+
steps:
155+
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5
156+
157+
- uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065
158+
with:
159+
python-version: "3.12"
160+
161+
- name: Install Rust coverage tooling
162+
run: |
163+
rustup component add llvm-tools-preview
164+
cargo install cargo-llvm-cov --version 0.8.4 --locked
165+
166+
- name: Generate Rust coverage report
167+
env:
168+
CARGO_PACKAGES: ${{ needs.validate-and-detect.outputs.cargo_packages }}
169+
run: |
170+
mkdir -p coverage
171+
mapfile -t cargo_packages < <(python3 -c 'import json, os; [print(package) for package in json.loads(os.environ["CARGO_PACKAGES"])]')
172+
cargo_args=()
173+
for package in "${cargo_packages[@]}"; do
174+
cargo_args+=("-p" "${package}")
175+
done
176+
cargo llvm-cov "${cargo_args[@]}" --cobertura --output-path coverage/cobertura.xml
177+
178+
- name: Enforce per-plugin coverage floor
179+
env:
180+
PLUGINS: ${{ needs.validate-and-detect.outputs.plugins }}
181+
run: python3 tools/plugin_catalog.py coverage-check . coverage/cobertura.xml 50.00 "${PLUGINS}"
182+
183+
- name: Upload coverage to Codecov
184+
uses: codecov/codecov-action@b9fd7d16f6d7d1b5d2bec1a2887e65ceed900238
185+
with:
186+
files: ./coverage/cobertura.xml
187+
flags: rust-python-package-workspace
188+
name: rust-python-package-workspace-coverage
189+
190+
documentation:
191+
needs: validate-and-detect
192+
if: needs.validate-and-detect.outputs.has_plugins == 'true'
193+
runs-on: ubuntu-latest
194+
defaults:
195+
run:
196+
shell: bash
197+
steps:
198+
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5
199+
200+
- name: Verify Rust toolchain
201+
run: |
202+
rustc --version
203+
cargo --version
204+
205+
- name: Build Rust documentation
206+
env:
207+
CARGO_PACKAGES: ${{ needs.validate-and-detect.outputs.cargo_packages }}
208+
run: |
209+
mapfile -t cargo_packages < <(python3 -c 'import json, os; [print(package) for package in json.loads(os.environ["CARGO_PACKAGES"])]')
210+
cargo_args=()
211+
for package in "${cargo_packages[@]}"; do
212+
cargo_args+=("-p" "${package}")
213+
done
214+
cargo doc "${cargo_args[@]}" --lib --no-deps --document-private-items
215+
112216
release-validation:
113217
if: github.event_name == 'pull_request'
114218
needs: validate-and-detect

‎Cargo.lock‎

Lines changed: 8 additions & 8 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Cargo.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ log = "0.4"
2222
pyo3-async-runtimes = { version = "0.28", features = ["tokio-runtime"] }
2323
pyo3 = { version = "0.28.2", features = ["abi3-py311"] }
2424
pyo3-log = "0.13"
25-
pyo3-stub-gen = "0.20.0"
25+
pyo3-stub-gen = "0.22.1"
2626
regex = "1.12"
2727
serde_json = "1.0"
2828
thiserror = "2.0"

‎SECURITY.md‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ When reporting, please include:
2626

2727
### Supply Chain
2828

29-
- Rust dependencies are audited with [`cargo deny`](https://github.com/EmbarkStudios/cargo-deny). Each Rust+Python plugin includes a `deny.toml` configuration.
29+
- Rust dependencies are audited with [`cargo deny`](https://github.com/EmbarkStudios/cargo-deny). Rust+Python plugins share the workspace-level `deny.toml` configuration.
3030
- Python dependencies are managed with `uv` and locked via `uv.lock`.
3131
- GitHub Actions workflows use pinned action versions.
3232

@@ -55,7 +55,7 @@ When contributing a new plugin:
5555
- [ ] Validate all configuration at startup, not at request time.
5656
- [ ] Do not store secrets in source code or configuration files.
5757
- [ ] Use Pydantic models for configuration validation.
58-
- [ ] Add `deny.toml` for Rust dependency auditing.
58+
- [ ] Use the workspace-level `deny.toml` for Rust dependency auditing.
5959
- [ ] Document any fail-open behavior and its trade-offs.
6060
- [ ] Include security-relevant test cases (e.g., malformed input, boundary conditions).
6161
- [ ] Ensure error messages do not leak internal state or sensitive information.

‎deny.toml‎

Lines changed: 126 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,126 @@
1+
# Workspace-level cargo-deny config: applies to all crates in the workspace.
2+
# See https://embarkstudios.github.io/cargo-deny/
3+
4+
[advisories]
5+
# pyo3-stub-gen 0.22.1 still pulls rustpython-parser's unmaintained unic-* parser crates.
6+
# Keep this scoped to the known transitive advisories so new unmaintained dependencies fail CI.
7+
ignore = [
8+
"RUSTSEC-2025-0075", # unic-char-range via rustpython-parser
9+
"RUSTSEC-2025-0080", # unic-common via rustpython-parser
10+
"RUSTSEC-2025-0081", # unic-char-property via rustpython-parser
11+
"RUSTSEC-2025-0090", # unic-emoji-char via rustpython-parser
12+
"RUSTSEC-2025-0098", # unic-ucd-version via rustpython-parser
13+
"RUSTSEC-2025-0100", # unic-ucd-ident via rustpython-parser
14+
]
15+
16+
[licenses]
17+
# Listed licenses are allowed even if not encountered in the dependency tree.
18+
unused-allowed-license = "allow"
19+
confidence-threshold = 0.95
20+
allow = [
21+
"0BSD",
22+
"AFL-1.1",
23+
"AFL-1.2",
24+
"AFL-2.0",
25+
"AFL-2.1",
26+
"AFL-3.0",
27+
"Adobe-2006",
28+
"ANTLR-PD",
29+
"Apache-1.0",
30+
"Apache-1.1",
31+
"Apache-2.0",
32+
"Apache-2.0 WITH LLVM-exception",
33+
"Artistic-1.0",
34+
"Artistic-1.0-Perl",
35+
"Artistic-1.0-cl8",
36+
"Artistic-2.0",
37+
"Beerware",
38+
"BlueOak-1.0.0",
39+
"BSL-1.0",
40+
"BSD-1-Clause",
41+
"BSD-2-Clause",
42+
"BSD-2-Clause-FreeBSD",
43+
"BSD-2-Clause-NetBSD",
44+
"BSD-2-Clause-Patent",
45+
"BSD-2-Clause-Views",
46+
"BSD-3-Clause",
47+
"BSD-3-Clause-Attribution",
48+
"BSD-3-Clause-Clear",
49+
"BSD-4-Clause",
50+
"BSD-Protection",
51+
"BSD-Source-Code",
52+
"bzip2-1.0.6",
53+
"CC-BY-1.0",
54+
"CC-BY-2.0",
55+
"CC-BY-2.5",
56+
"CC-BY-2.5-AU",
57+
"CC-BY-3.0",
58+
"CC-BY-3.0-AT",
59+
"CC-BY-3.0-DE",
60+
"CC-BY-3.0-NL",
61+
"CC-BY-3.0-US",
62+
"CC-BY-4.0",
63+
"CC0-1.0",
64+
"CDDL-1.0",
65+
"CDDL-1.1",
66+
"ClArtistic",
67+
"CPL-1.0",
68+
"curl",
69+
"ECL-2.0",
70+
"eGenix",
71+
"EPL-1.0",
72+
"EPL-2.0",
73+
"ErlPL-1.1",
74+
"FSFAP",
75+
"FTL",
76+
"HPND",
77+
"ICU",
78+
"IJG",
79+
"Info-ZIP",
80+
"ISC",
81+
"JSON",
82+
"Libpng",
83+
"MIT",
84+
"MIT-0",
85+
"MIT-Modern-Variant",
86+
"MPL-1.0",
87+
"MPL-1.1",
88+
"MPL-2.0",
89+
"MS-PL",
90+
"Net-SNMP",
91+
"OLDAP-1.4",
92+
"OLDAP-2.8",
93+
"OFL-1.0",
94+
"OFL-1.1",
95+
"OpenSSL",
96+
"PHP-3.0",
97+
"PHP-3.01",
98+
"Plexus",
99+
"PostgreSQL",
100+
"PSF-2.0",
101+
"Python-2.0",
102+
"Python-2.0.1",
103+
"RSA-MD",
104+
"Ruby",
105+
"SISSL",
106+
"SISSL-1.2",
107+
"TCL",
108+
"Unicode-3.0",
109+
"Unicode-DFS-2015",
110+
"Unicode-DFS-2016",
111+
"Unicode-TOU",
112+
"Unlicense",
113+
"UPL-1.0",
114+
"W3C",
115+
"W3C-19980720",
116+
"W3C-20150513",
117+
"X11",
118+
"X11-distribute-modifications-variant",
119+
"Xnet",
120+
"Zend-2.0",
121+
"Zlib",
122+
"zlib-acknowledgement",
123+
"ZPL-1.1",
124+
"ZPL-2.0",
125+
"ZPL-2.1",
126+
]

‎plugins/rust/python-package/encoded_exfil_detection/deny.toml‎

Lines changed: 0 additions & 2 deletions
This file was deleted.

‎plugins/rust/python-package/pii_filter/cpex_pii_filter/pii_filter_rust/__init__.pyi‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -55,9 +55,9 @@ class PIIDetectorRust:
5555
* `max_text_bytes` (int): Maximum text payload size to inspect
5656
* `max_nested_depth` (int): Maximum nested container depth to inspect
5757
* `max_collection_items` (int): Maximum items to inspect per collection
58-
* `custom_patterns` (list[dict]): Additional regex-based PII patterns.
58+
* `custom_patterns` (`list[dict]`): Additional regex-based PII patterns.
5959
`mask_strategy` is optional and inherits `default_mask_strategy` when omitted or `None`.
60-
* `whitelist_patterns` (list[str]): Regex patterns to exclude from detection
60+
* `whitelist_patterns` (`list[str]`): Regex patterns to exclude from detection
6161
"""
6262
def detect(self, text: builtins.str) -> typing.Any:
6363
r"""

0 commit comments

Comments
 (0)