Fix secrets detector model dump counting #236
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI Rust Python Package Plugins | |
| on: | |
| push: | |
| branches: [main] | |
| paths: | |
| - "Makefile" | |
| - "Cargo.toml" | |
| - "Cargo.lock" | |
| - "deny.toml" | |
| - "crates/**" | |
| - "README.md" | |
| - "DEVELOPING.md" | |
| - "TESTING.md" | |
| - "plugins/rust/python-package/**" | |
| - "plugins/tests/**" | |
| - "tools/**" | |
| - ".github/workflows/ci-rust-python-package.yaml" | |
| - ".github/workflows/release-rust-python-package.yaml" | |
| pull_request: | |
| branches: [main] | |
| paths: | |
| - "Makefile" | |
| - "Cargo.toml" | |
| - "Cargo.lock" | |
| - "deny.toml" | |
| - "crates/**" | |
| - "README.md" | |
| - "DEVELOPING.md" | |
| - "TESTING.md" | |
| - "plugins/rust/python-package/**" | |
| - "plugins/tests/**" | |
| - "tools/**" | |
| - ".github/workflows/ci-rust-python-package.yaml" | |
| - ".github/workflows/release-rust-python-package.yaml" | |
| workflow_dispatch: | |
| concurrency: | |
| group: ci-rust-python-package-${{ github.event.pull_request.head.repo.full_name || github.repository }}-${{ github.head_ref || github.ref_name }} | |
| cancel-in-progress: true | |
| permissions: | |
| contents: read | |
| pull-requests: read | |
| jobs: | |
| validate-and-detect: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| plugins: ${{ steps.detect.outputs.plugins }} | |
| has_plugins: ${{ steps.detect.outputs.has_plugins }} | |
| plugin_count: ${{ steps.detect.outputs.plugin_count }} | |
| cargo_packages: ${{ steps.detect.outputs.cargo_packages }} | |
| steps: | |
| - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 | |
| with: | |
| fetch-depth: 0 | |
| - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 | |
| with: | |
| python-version: "3.12" | |
| - id: detect | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| if [[ "${GITHUB_EVENT_NAME}" == "pull_request" ]]; then | |
| selection="$(python3 tools/plugin_catalog.py ci-selection . diff "${{ github.event.pull_request.base.sha }}" "${{ github.sha }}")" | |
| elif [[ "${GITHUB_EVENT_NAME}" == "workflow_dispatch" ]]; then | |
| selection="$(python3 tools/plugin_catalog.py ci-selection . all '' '')" | |
| elif [[ "${{ github.event.before }}" == "0000000000000000000000000000000000000000" ]]; then | |
| selection="$(python3 tools/plugin_catalog.py ci-selection . all '' '')" | |
| else | |
| selection="$(python3 tools/plugin_catalog.py ci-selection . diff "${{ github.event.before }}" "${{ github.sha }}")" | |
| fi | |
| selection="$(printf '%s' "${selection}" | python3 -c 'import json, re, sys; payload = json.load(sys.stdin); slug_re = re.compile(r"^[a-z0-9_]+$"); plugins = payload.get("plugins"); cargo_packages = payload.get("cargo_packages"); has_plugins = payload.get("has_plugins"); plugin_count = payload.get("plugin_count"); assert isinstance(plugins, list) and all(isinstance(item, str) and slug_re.fullmatch(item) for item in plugins); assert isinstance(cargo_packages, list) and all(isinstance(item, str) and slug_re.fullmatch(item) for item in cargo_packages); assert isinstance(has_plugins, bool); assert isinstance(plugin_count, int) and plugin_count == len(plugins); print(json.dumps({"plugins": plugins, "has_plugins": has_plugins, "plugin_count": plugin_count, "cargo_packages": cargo_packages}))')" | |
| plugins="$(printf '%s' "${selection}" | python3 -c 'import json, sys; print(json.dumps(json.load(sys.stdin)["plugins"]))')" | |
| has_plugins="$(printf '%s' "${selection}" | python3 -c 'import json, sys; print(str(json.load(sys.stdin)["has_plugins"]).lower())')" | |
| plugin_count="$(printf '%s' "${selection}" | python3 -c 'import json, sys; print(json.load(sys.stdin)["plugin_count"])')" | |
| cargo_packages="$(printf '%s' "${selection}" | python3 -c 'import json, sys; print(json.dumps(json.load(sys.stdin)["cargo_packages"]))')" | |
| if [[ "${has_plugins}" == "false" ]]; then | |
| has_plugins_output="false" | |
| else | |
| has_plugins_output="true" | |
| fi | |
| { | |
| echo "plugins=${plugins}" | |
| echo "plugin_count=${plugin_count}" | |
| echo "cargo_packages=${cargo_packages}" | |
| echo "has_plugins=${has_plugins_output}" | |
| } >> "$GITHUB_OUTPUT" | |
| build-test: | |
| needs: validate-and-detect | |
| if: needs.validate-and-detect.outputs.has_plugins == 'true' | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: [ubuntu-latest, macos-latest, windows-latest] | |
| plugin: ${{ fromJson(needs.validate-and-detect.outputs.plugins) }} | |
| runs-on: ${{ matrix.os }} | |
| defaults: | |
| run: | |
| shell: bash | |
| steps: | |
| - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 | |
| - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 | |
| with: | |
| python-version: "3.12" | |
| - name: Verify Rust toolchain | |
| run: | | |
| rustc --version | |
| cargo --version | |
| - name: Install uv | |
| run: python -m pip install uv==0.9.30 maturin==1.12.6 | |
| - name: Sync plugin environment | |
| working-directory: plugins/rust/python-package/${{ matrix.plugin }} | |
| run: make sync | |
| - name: Plugin CI build verification | |
| if: matrix.os == 'ubuntu-latest' | |
| working-directory: plugins/rust/python-package/${{ matrix.plugin }} | |
| run: make ci-build | |
| - name: Plugin CI verification | |
| if: matrix.os != 'ubuntu-latest' | |
| working-directory: plugins/rust/python-package/${{ matrix.plugin }} | |
| run: make ci | |
| security-policy: | |
| needs: validate-and-detect | |
| if: needs.validate-and-detect.outputs.has_plugins == 'true' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 | |
| - name: Verify Rust toolchain | |
| run: | | |
| rustc --version | |
| cargo --version | |
| - name: Install cargo deny | |
| run: cargo install cargo-deny@0.19.0 --locked | |
| - name: Run cargo deny | |
| run: cargo deny check --config deny.toml | |
| coverage: | |
| needs: validate-and-detect | |
| if: needs.validate-and-detect.outputs.has_plugins == 'true' | |
| runs-on: ubuntu-latest | |
| defaults: | |
| run: | |
| shell: bash | |
| steps: | |
| - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 | |
| - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 | |
| with: | |
| python-version: "3.12" | |
| - name: Install Rust coverage tooling | |
| run: | | |
| rustup component add llvm-tools-preview | |
| cargo install cargo-llvm-cov --version 0.8.4 --locked | |
| - name: Install Python build tooling | |
| run: python -m pip install uv==0.9.30 maturin==1.12.6 | |
| - name: Generate Rust coverage report | |
| env: | |
| CARGO_PACKAGES: ${{ needs.validate-and-detect.outputs.cargo_packages }} | |
| PLUGINS: ${{ needs.validate-and-detect.outputs.plugins }} | |
| PYO3_PYTHON: python | |
| run: | | |
| mkdir -p coverage | |
| mapfile -t cargo_packages < <(python3 -c 'import json, os; [print(package) for package in json.loads(os.environ["CARGO_PACKAGES"])]') | |
| mapfile -t plugins < <(python3 -c 'import json, os; [print(plugin) for plugin in json.loads(os.environ["PLUGINS"])]') | |
| cargo_args=() | |
| for package in "${cargo_packages[@]}"; do | |
| cargo_args+=("-p" "${package}") | |
| done | |
| cargo llvm-cov clean --workspace | |
| eval "$(cargo llvm-cov show-env --sh)" | |
| export CARGO_TARGET_DIR="${CARGO_LLVM_COV_TARGET_DIR}/llvm-cov-target" | |
| export CARGO_LLVM_COV_BUILD_DIR="${CARGO_TARGET_DIR}" | |
| export LLVM_PROFILE_FILE="${CARGO_TARGET_DIR}/cpex-plugins-%p-%10m.profraw" | |
| mkdir -p "${CARGO_TARGET_DIR}" | |
| for plugin in "${plugins[@]}"; do | |
| (cd "plugins/rust/python-package/${plugin}" && make sync && uv run maturin develop) | |
| done | |
| cargo test "${cargo_args[@]}" | |
| for plugin in "${plugins[@]}"; do | |
| (cd "plugins/rust/python-package/${plugin}" && make test-integration) | |
| done | |
| env -u CARGO_TARGET_DIR -u CARGO_LLVM_COV_BUILD_DIR -u CARGO_LLVM_COV_TARGET_DIR -u LLVM_PROFILE_FILE cargo llvm-cov report "${cargo_args[@]}" --cobertura --output-path coverage/cobertura.xml | |
| - name: Enforce per-plugin coverage floor | |
| env: | |
| PLUGINS: ${{ needs.validate-and-detect.outputs.plugins }} | |
| run: python3 tools/plugin_catalog.py coverage-check . coverage/cobertura.xml 90.00 "${PLUGINS}" | |
| - name: Upload coverage to Codecov | |
| uses: codecov/codecov-action@b9fd7d16f6d7d1b5d2bec1a2887e65ceed900238 | |
| with: | |
| files: ./coverage/cobertura.xml | |
| flags: rust-python-package-workspace | |
| name: rust-python-package-workspace-coverage | |
| documentation: | |
| needs: validate-and-detect | |
| if: needs.validate-and-detect.outputs.has_plugins == 'true' | |
| runs-on: ubuntu-latest | |
| defaults: | |
| run: | |
| shell: bash | |
| steps: | |
| - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 | |
| - name: Verify Rust toolchain | |
| run: | | |
| rustc --version | |
| cargo --version | |
| - name: Build Rust documentation | |
| env: | |
| CARGO_PACKAGES: ${{ needs.validate-and-detect.outputs.cargo_packages }} | |
| run: | | |
| mapfile -t cargo_packages < <(python3 -c 'import json, os; [print(package) for package in json.loads(os.environ["CARGO_PACKAGES"])]') | |
| cargo_args=() | |
| for package in "${cargo_packages[@]}"; do | |
| cargo_args+=("-p" "${package}") | |
| done | |
| cargo doc "${cargo_args[@]}" --lib --no-deps --document-private-items | |
| release-validation: | |
| if: github.event_name == 'pull_request' | |
| needs: validate-and-detect | |
| permissions: | |
| contents: read | |
| pull-requests: read | |
| id-token: write | |
| uses: ./.github/workflows/release-rust-python-package.yaml | |
| with: | |
| tag: retry-with-backoff-v0.2.0 | |
| repository: testpypi | |
| publish_enabled: false |