ci: include coverage from running build scripts (#6007)

davidhewitt · ngoldbaum · web-flow · commit 6d0c1ee1923f · 2026-05-16T20:59:08.000Z
* ci: include coverage from running build scripts

* fix passing codecov token to report-coverage action

* completely skip coverage on windows-11-arm for now

* correct condition

* fixups, add coverage to cross-compilation jobs

* install uv in cross compilation job

* cleanups, try cargo-llvm-cov branch

* fix typo

* specify package to install for testing

* typo

* fixup cross compile step

* switch cross-compilation jobs to debug

* skip containers for cross compile tests

* fixups

* set linker for cross-compilation job

* correct secret

* fixup report names

* use released cargo-llvm-cov

* Apply suggestions from code review

Co-authored-by: Nathan Goldbaum &lt;nathan.goldbaum@gmail.com&gt;
Co-authored-by: David Hewitt &lt;mail@davidhewitt.dev&gt;

* try running in container

* fixup cross-compilation

* cross compilation host coverage only

* final tidy ups

---------

Co-authored-by: Nathan Goldbaum &lt;nathan.goldbaum@gmail.com&gt;
diff --git a/.github/actions/prepare-coverage/action.yml b/.github/actions/prepare-coverage/action.yml
@@ -0,0 +1,20 @@
+name: 'Prepare coverage'
+description: 'Installs cargo-llvm-cov and prepares the environment for coverage collection'
+inputs:
+  host_only:
+    description: 'Whether to prepare the environment for host-only coverage collection'
+    required: false
+    default: "false"
+runs:
+  using: "composite"
+  steps:
+  - name: Install cargo-llvm-cov
+    uses: taiki-e/install-action@cargo-llvm-cov
+
+  - name: Prepare coverage environment
+    shell: bash
+    run: |
+      cargo llvm-cov clean --workspace
+      uvx nox -s set-coverage-env -- ${{ inputs.host_only == 'true' && '--coverage-host-only' || '' }}
+    env:
+      CARGO_LLVM_COV_SETUP: "yes"
diff --git a/.github/actions/report-coverage/action.yml b/.github/actions/report-coverage/action.yml
@@ -0,0 +1,22 @@
+name: 'Report coverage'
+description: 'Generate coverage report using cargo-llvm-cov and upload it to Codecov'
+inputs:
+  name:
+    description: 'The name of the coverage report'
+    required: true
+  token:
+    description: 'Codecov upload token'
+    required: true
+runs:
+  using: "composite"
+  steps:
+  - name: Generate coverage report
+    shell: bash
+    run: uvx nox -s generate-coverage-report -- --codecov --output-path=coverage.json
+
+  - name: Upload coverage report
+    uses: codecov/codecov-action@v6
+    with:
+      files: coverage.json
+      name: ${{ inputs.name }}
+      token: ${{ inputs.token }}
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -1,6 +1,9 @@
 on:
   workflow_call:
     inputs:
+      sha:
+        required: true
+        type: string
       os:
         required: true
         type: string
@@ -34,14 +37,7 @@ jobs:
     steps:
       - uses: actions/checkout@v6.0.2
         with:
-          # For PRs, we need to run on the real PR head, not the resultant merge of the PR into the target branch.
-          #
-          # This is necessary for coverage reporting to make sense; we then get exactly the coverage change
-          # between the base branch and the real PR head.
-          #
-          # If it were run on the merge commit the problem is that the coverage potentially does not align
-          # with the commit diff, because the merge may affect line numbers.
-          ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
+          ref: ${{ inputs.sha }}
 
       # installs using setup-python do not work for arm macOS 3.9 and below
       - if: ${{ !(inputs.os == 'macos-latest' && contains(fromJSON('["3.8", "3.9"]'), inputs.python-version) && inputs.python-architecture == 'x64') }}
@@ -70,7 +66,7 @@ jobs:
           toolchain: ${{ inputs.rust }}
           targets: ${{ inputs.rust-target }}
           # rust-src needed to correctly format errors, see #1865
-          components: rust-src,llvm-tools-preview
+          components: rust-src
 
       # On windows 32 bit, we are running on an x64 host, so we need to specifically set the target
       # NB we don't do this for *all* jobs because it breaks coverage of proc macros to have an
@@ -119,13 +115,8 @@ jobs:
         if: ${{ endsWith(inputs.python-version, '-dev') || (steps.ffi-changes.outputs.changed == 'true' && inputs.rust == 'stable' && !startsWith(inputs.python-version, 'graalpy')) }}
         run: nox -s ffi-check
 
-      - name: Install cargo-llvm-cov
-        uses: taiki-e/install-action@cargo-llvm-cov
-
-      - name: Prepare coverage environment
-        run: |
-          cargo llvm-cov clean --workspace --profraw-only
-          nox -s set-coverage-env
+      - uses: ./.github/actions/prepare-coverage
+        if: ${{ inputs.os != 'windows-11-arm' }} # https://github.com/rust-lang/rust/issues/150123
 
       - name: Build docs
         run: nox -s docs
@@ -139,23 +130,9 @@ jobs:
         env:
           CARGO_TARGET_DIR: ${{ github.workspace }}/target
 
-      - name: Generate coverage report
-        # needs investigation why llvm-cov fails on windows-11-arm
-        continue-on-error: ${{ inputs.os == 'windows-11-arm' }}
-        run: cargo llvm-cov
-          --package=pyo3
-          --package=pyo3-build-config
-          --package=pyo3-macros-backend
-          --package=pyo3-macros
-          --package=pyo3-ffi
-          report --codecov --output-path coverage.json
-
-      - name: Upload coverage report
-        uses: codecov/codecov-action@v6
-        # needs investigation why llvm-cov fails on windows-11-arm
-        continue-on-error: ${{ inputs.os == 'windows-11-arm' }}
+      - uses: ./.github/actions/report-coverage
+        if: ${{ inputs.os != 'windows-11-arm' }} # https://github.com/rust-lang/rust/issues/150123
         with:
-          files: coverage.json
           name: ${{ inputs.os }}/${{ inputs.python-version }}/${{ inputs.rust }}
           token: ${{ secrets.CODECOV_TOKEN }}
 
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -12,6 +12,8 @@ concurrency:
 
 env:
   CARGO_TERM_COLOR: always
+  NOX_DEFAULT_VENV_BACKEND: uv
+  UV_PYTHON: 3.14
 
 jobs:
   fmt:
@@ -39,6 +41,15 @@ jobs:
     outputs:
       MSRV: ${{ steps.resolve-msrv.outputs.MSRV }}
       verbose: ${{ runner.debug == '1' }}
+      save-cache: ${{ github.ref == 'refs/heads/main' || contains(github.event.pull_request.labels.*.name, 'CI-save-pr-cache') }}
+      # For PRs, we need to run on the real PR head, not the resultant merge of the PR into the target branch.
+      #
+      # This is necessary for coverage reporting to make sense; we then get exactly the coverage change
+      # between the base branch and the real PR head.
+      #
+      # If it were run on the merge commit the problem is that the coverage potentially does not align
+      # with the commit diff, because the merge may affect line numbers.
+      coverage-sha: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
     steps:
       - uses: actions/checkout@v6.0.2
       - uses: actions/setup-python@v6
@@ -138,6 +149,7 @@ jobs:
     needs: [fmt, resolve]
     uses: ./.github/workflows/build.yml
     with:
+      sha: ${{ needs.resolve.outputs.coverage-sha }}
       os: ${{ matrix.platform.os }}
       python-version: ${{ matrix.python-version }}
       python-architecture: ${{ matrix.platform.python-architecture }}
@@ -231,6 +243,7 @@ jobs:
     needs: [fmt, resolve]
     uses: ./.github/workflows/build.yml
     with:
+      sha: ${{ needs.resolve.outputs.coverage-sha }}
       os: ${{ matrix.platform.os }}
       python-version: ${{ matrix.python-version }}
       python-architecture: ${{ matrix.platform.python-architecture }}
@@ -617,20 +630,26 @@ jobs:
       - run: python3 -m nox -s test
 
   test-version-limits:
-    needs: [fmt]
+    needs: [fmt, resolve]
     if: ${{ contains(github.event.pull_request.labels.*.name, 'CI-build-full') || github.event_name != 'pull_request' }}
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v6.0.2
-      - uses: actions/setup-python@v6
         with:
-          python-version: "3.14"
-      - uses: Swatinem/rust-cache@v2
+          ref: ${{ needs.resolve.outputs.coverage-sha }}
+      - uses: astral-sh/setup-uv@v7
         with:
-          save-if: ${{ github.ref == 'refs/heads/main' || contains(github.event.pull_request.labels.*.name, 'CI-save-pr-cache') }}
+          save-cache: ${{ needs.resolve.outputs.save-cache }}
       - uses: dtolnay/rust-toolchain@stable
-      - run: python3 -m pip install --upgrade pip && pip install nox[uv]
-      - run: python3 -m nox -s test-version-limits
+      - uses: Swatinem/rust-cache@v2
+        with:
+          save-if: ${{ needs.resolve.outputs.save-cache }}
+      - uses: ./.github/actions/prepare-coverage
+      - run: uvx nox -s test-version-limits
+      - uses: ./.github/actions/report-coverage
+        with:
+          name: ${{ github.job }}
+          token: ${{ secrets.CODECOV_TOKEN }}
 
   check-feature-powerset:
     needs: [fmt, resolve]
@@ -662,7 +681,7 @@ jobs:
       - run: python3 -m nox -s check-feature-powerset -- ${{ matrix.rust != 'stable' && 'minimal-versions' || '' }}
 
   test-cross-compilation:
-    needs: [fmt]
+    needs: [fmt, resolve]
     if: ${{ contains(github.event.pull_request.labels.*.name, 'CI-build-full') || github.event_name != 'pull_request' }}
     runs-on: ${{ matrix.os }}
     name: test-cross-compilation ${{ matrix.os }} -> ${{ matrix.target }}
@@ -686,45 +705,72 @@ jobs:
             target: "x86_64-pc-windows-gnu"
             # TODO: remove pyo3/generate-import-lib feature when maturin supports cross compiling to Windows without it
             flags: "-i python3.13 --features pyo3/generate-import-lib"
+            apt-packages: mingw-w64 llvm
           # windows x86_64 -> aarch64
           - os: "windows-latest"
             target: "aarch64-pc-windows-msvc"
             flags: "-i python3.13 --features pyo3/generate-import-lib"
     steps:
       - uses: actions/checkout@v6.0.2
+        with:
+          ref: ${{ needs.resolve.outputs.coverage-sha }}
+      - uses: astral-sh/setup-uv@v7
+        with:
+          save-cache: ${{ needs.resolve.outputs.save-cache }}
       - uses: actions/setup-python@v6
         with:
-          python-version: "3.14"
+          python-version: ${{ env.UV_PYTHON }}
       - uses: Swatinem/rust-cache@v2
         with:
           workspaces: examples/maturin-starter
           save-if: ${{ github.ref == 'refs/heads/main' || contains(github.event.pull_request.labels.*.name, 'CI-save-pr-cache') }}
           key: ${{ matrix.target }}
-      - name: Setup cross-compiler
-        if: ${{ matrix.target == 'x86_64-pc-windows-gnu' }}
-        run: sudo apt-get install -y mingw-w64 llvm
+      - name: Setup cross-compiler packages
+        if: ${{ matrix.apt-packages }}
+        run: sudo apt-get install -y ${{ matrix.apt-packages }}
+      - uses: ./.github/actions/prepare-coverage
+        with:
+         host_only: true
       - name: Compile version-specific library
         uses: PyO3/maturin-action@v1
         with:
           target: ${{ matrix.target }}
+          args: --profile=dev -m examples/maturin-starter/Cargo.toml ${{ matrix.flags }}
           manylinux: ${{ matrix.manylinux }}
-          args: --release -m examples/maturin-starter/Cargo.toml ${{ matrix.flags }}
+          docker-options: >-
+            -e LLVM_PROFILE_FILE
+            -e __CARGO_LLVM_COV_RUSTC_WRAPPER
+            -e __CARGO_LLVM_COV_RUSTC_WRAPPER_RUSTFLAGS
+            -e __CARGO_LLVM_COV_RUSTC_WRAPPER_CRATE_NAMES
+            -v /home/runner/.cargo/bin/cargo-llvm-cov:/home/runner/.cargo/bin/cargo-llvm-cov
       - name: Compile abi3 library
         uses: PyO3/maturin-action@v1
         with:
           target: ${{ matrix.target }}
+          args: --profile=dev -m examples/maturin-starter/Cargo.toml --features abi3 ${{ matrix.flags }}
           manylinux: ${{ matrix.manylinux }}
-          args: --release -m examples/maturin-starter/Cargo.toml --features abi3 ${{ matrix.flags }}
+          docker-options: >-
+            -e LLVM_PROFILE_FILE
+            -e __CARGO_LLVM_COV_RUSTC_WRAPPER
+            -e __CARGO_LLVM_COV_RUSTC_WRAPPER_RUSTFLAGS
+            -e __CARGO_LLVM_COV_RUSTC_WRAPPER_CRATE_NAMES
+            -v /home/runner/.cargo/bin/cargo-llvm-cov:/home/runner/.cargo/bin/cargo-llvm-cov
+      - uses: ./.github/actions/report-coverage
+        with:
+          name: ${{ github.job }}/${{ matrix.os }}/${{ matrix.target }}
+          token: ${{ secrets.CODECOV_TOKEN }}
 
   test-cross-compilation-windows:
-    needs: [fmt]
+    needs: [fmt, resolve]
     if: ${{ contains(github.event.pull_request.labels.*.name, 'CI-build-full') || github.event_name != 'pull_request' }}
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v6.0.2
-      - uses: actions/setup-python@v6
         with:
-          python-version: "3.14"
+          ref: ${{ needs.resolve.outputs.coverage-sha }}
+      - uses: astral-sh/setup-uv@v7
+        with:
+          save-cache: ${{ needs.resolve.outputs.save-cache }}
       - uses: dtolnay/rust-toolchain@stable
         with:
           targets: x86_64-pc-windows-gnu,x86_64-pc-windows-msvc
@@ -734,12 +780,17 @@ jobs:
         with:
           path: ~/.cache/cargo-xwin
           key: cargo-xwin-cache
+      - name: Setup cross-compiler
+        run: sudo apt-get install -y mingw-w64 llvm
+      - uses: ./.github/actions/prepare-coverage
+        with:
+          host_only: true
       - name: Test cross compile to Windows
-        run: |
-          set -ex
-          sudo apt-get install -y mingw-w64 llvm
-          pip install nox
-          nox -s test-cross-compilation-windows
+        run: uvx nox -s test-cross-compilation-windows
+      - uses: ./.github/actions/report-coverage
+        with:
+          name: ${{ github.job }}
+          token: ${{ secrets.CODECOV_TOKEN }}
 
   test-introspection:
     needs: [fmt]
diff --git a/noxfile.py b/noxfile.py
@@ -51,8 +51,16 @@
 FREE_THREADED_BUILD = bool(sysconfig.get_config_var("Py_GIL_DISABLED"))
 
 
-def _get_output(*args: str) -> str:
-    return subprocess.run(args, capture_output=True, text=True, check=True).stdout
+def _get_output(*args: str, env: Optional[Dict[str, str]] = None) -> str:
+    try:
+        return subprocess.run(
+            args, capture_output=True, text=True, check=True, stdin=None, env=env
+        ).stdout
+    except subprocess.CalledProcessError as e:
+        print(f"Command {args} failed with exit code {e.returncode}")
+        print(f"stdout:\n{e.stdout}")
+        print(f"stderr:\n{e.stderr}")
+        raise nox.command.CommandFailed() from e
 
 
 def _parse_supported_interpreter_version(
@@ -165,18 +173,14 @@ def coverage(session: nox.Session) -> None:
 def set_coverage_env(session: nox.Session) -> None:
     """For use in GitHub Actions to set coverage environment variables."""
     with open(os.environ["GITHUB_ENV"], "a") as env_file:
-        for k, v in _get_coverage_env().items():
+        for k, v in _get_coverage_env(*session.posargs).items():
             print(f"{k}={v}", file=env_file)
 
 
 @nox.session(name="generate-coverage-report", venv_backend="none")
 def generate_coverage_report(session: nox.Session) -> None:
-    cov_format = "codecov"
-    output_file = "coverage.json"
-
-    if "lcov" in session.posargs:
-        cov_format = "lcov"
-        output_file = "lcov.info"
+    # default to `--html` report if no additional arguments provided (convenient for local use)
+    posargs = ("--html",) if not session.posargs else tuple(session.posargs)
 
     _run_cargo(
         session,
@@ -186,10 +190,9 @@ def generate_coverage_report(session: nox.Session) -> None:
         "--package=pyo3-macros-backend",
         "--package=pyo3-macros",
         "--package=pyo3-ffi",
+        "--include-build-script",
         "report",
-        f"--{cov_format}",
-        "--output-path",
-        output_file,
+        *posargs,
     )
 
 
@@ -1615,10 +1618,16 @@ def _get_feature_sets() -> Tuple[Optional[str], ...]:
 _HOST_LINE_START = "host: "
 
 
-def _get_coverage_env() -> Dict[str, str]:
-    env = {}
-    output = _get_output("cargo", "llvm-cov", "show-env")
+def _get_coverage_env(*flags: str) -> Dict[str, str]:
+    llvm_cov_execution_env = os.environ.copy()
+    # prevent llvm-cov from hanging asking to install llvm-tools-preview
+    # (allow user to override this, if they wish, e.g. in CI)
+    llvm_cov_execution_env.setdefault("CARGO_LLVM_COV_SETUP", "no")
+    output = _get_output(
+        "cargo", "llvm-cov", "show-env", *flags, env=llvm_cov_execution_env
+    )
 
+    env = {}
     for line in output.strip().splitlines():
         (key, value) = line.split("=", maxsplit=1)
         # Strip single or double quotes from the variable value
