Add coverage automation (#1340)

mdboom · web-flow · commit 0d48eaa2d14d · 2025-12-16T13:42:09.000Z
* First pass at coverage automation

* Add code coverage information
diff --git a/.coveragerc b/.coveragerc
@@ -0,0 +1,9 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: LicenseRef-NVIDIA-SOFTWARE-LICENSE
+
+[report]
+show_missing = true
+
+[run]
+plugins = Cython.Coverage
+core = ctrace
diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml
@@ -0,0 +1,161 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+name: "CI: Coverage"
+
+on:
+  schedule:
+    - cron: '0 0 * * *' # This runs the workflow every day at 12:00 AM UTC
+  workflow_dispatch: {}
+
+defaults:
+  run:
+    shell: bash --noprofile --norc -xeuo pipefail {0}
+
+env:
+  PY_VER: "3.14"
+  CUDA_VER: "13.1.0"
+  LOCAL_CTK: "1"
+  GPU: "a100"
+  DRIVER: "latest"
+  ARCH: "x86_64"
+  HOST_PLATFORM: "linux-64"
+
+jobs:
+  coverage:
+    name: Coverage
+    runs-on: "linux-amd64-gpu-a100-latest-1"
+    permissions:
+      id-token: write
+      contents: write
+    # Our self-hosted runners require a container
+    # TODO: use a different (nvidia?) container
+    container:
+      options: -u root --security-opt seccomp=unconfined --shm-size 16g
+      image: ubuntu:22.04
+      env:
+        NVIDIA_VISIBLE_DEVICES: ${{ env.NVIDIA_VISIBLE_DEVICES }}
+    steps:
+      - name: Ensure GPU is working
+        run: nvidia-smi
+
+      # We have to install git before checking out the repo (so that we can
+      # deploy the docs at the end). This means we can't use install_unix_deps
+      # action so install the git package.
+      - name: Install git
+        run: |
+          apt-get update
+          apt-get install -y git
+
+      - name: Checkout ${{ github.event.repository.name }}
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8  # v5.0.0
+
+      - name: Install dependencies
+        uses: ./.github/actions/install_unix_deps
+        continue-on-error: false
+        with:
+          dependencies: "tree rsync libsqlite3-0 g++ jq wget libgl1 libegl1"
+          dependent_exes: "tree rsync libsqlite3-0 g++ jq wget libgl1 libegl1"
+
+      - name: Setup proxy cache
+        uses: nv-gha-runners/setup-proxy-cache@main
+        continue-on-error: true
+
+      - name: Set environment variables
+        env:
+          BUILD_CUDA_VER: ${{ env.CUDA_VER }}
+          CUDA_VER: ${{ env.CUDA_VER }}
+          HOST_PLATFORM: ${{ env.HOST_PLATFORM }}
+          LOCAL_CTK: ${{ env.LOCAL_CTK }}
+          PY_VER: ${{ env.PY_VER }}
+          SHA: ${{ github.sha }}
+        run: |
+          ./ci/tools/env-vars test
+          echo "CUDA_PYTHON_COVERAGE=1" >> $GITHUB_ENV
+
+      - name: Set up Python
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065  # v5.6.0
+        with:
+          python-version: ${{ env.PY_VER }}
+        env:
+          # we use self-hosted runners on which setup-python behaves weirdly...
+          AGENT_TOOLSDIRECTORY: "/opt/hostedtoolcache"
+
+      - name: Set up mini CTK
+        if: ${{ env.LOCAL_CTK == '1' }}
+        uses: ./.github/actions/fetch_ctk
+        continue-on-error: false
+        with:
+          host-platform: ${{ env.HOST_PLATFORM }}
+          cuda-version: ${{ env.CUDA_VER }}
+
+      - name: Create venv
+        run: |
+          python -m venv .venv
+
+      - name: Build cuda-pathfinder
+        run: |
+          .venv/bin/pip install -v ./cuda_pathfinder
+
+      - name: Build cuda-python-test-helpers
+        run: |
+          .venv/bin/pip install -v ./cuda_python_test_helpers
+
+      - name: Build cuda-bindings
+        run: |
+          cd cuda_bindings
+          ../.venv/bin/pip install -v . --group test
+
+      - name: Build cuda-core
+        run: |
+          cd cuda_core
+          ../.venv/bin/pip install -v . --group test
+
+      - name: Install coverage tools
+        run: |
+          .venv/bin/pip install coverage pytest-cov
+
+      - name: Set cuda package install root
+        run: |
+          echo "INSTALL_ROOT=$(.venv/bin/python -c 'import cuda; print(cuda.__path__[0])')/.." >> $GITHUB_ENV
+          echo "REPO_ROOT=$(pwd)" >> $GITHUB_ENV
+
+      - name: Run cuda.pathfinder tests
+        run: |
+          cd $INSTALL_ROOT
+          $REPO_ROOT/.venv/bin/pytest -v --cov=./cuda --cov-append --cov-config=$REPO_ROOT/.coveragerc $REPO_ROOT/cuda_pathfinder/tests
+
+      - name: Run cuda.bindings tests
+        run: |
+          cd $INSTALL_ROOT
+          $REPO_ROOT/.venv/bin/pytest -v --cov=./cuda --cov-append --cov-config=$REPO_ROOT/.coveragerc $REPO_ROOT/cuda_bindings/tests
+
+      - name: Run cuda.core tests
+        run: |
+          cd $INSTALL_ROOT
+          $REPO_ROOT/.venv/bin/pytest -v --cov=./cuda --cov-append --cov-config=$REPO_ROOT/.coveragerc $REPO_ROOT/cuda_core/tests
+
+      - name: Generate coverage report
+        run: |
+          cd $INSTALL_ROOT
+          $REPO_ROOT/.venv/bin/coverage html --rcfile=$REPO_ROOT/.coveragerc
+          $REPO_ROOT/.venv/bin/coverage xml --rcfile=$REPO_ROOT/.coveragerc -o htmlcov/coverage.xml
+          mkdir $REPO_ROOT/docs
+          mv htmlcov $REPO_ROOT/docs/coverage
+
+      - name: Archive code coverage results
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02  # v4.6.2
+        with:
+          name: coverage
+          path: docs/coverage/
+
+      - name: Deploy to gh-pages
+        uses: JamesIves/github-pages-deploy-action@4a3abc783e1a24aeb44c16e869ad83caf6b4cc23  # v4.7.4
+        with:
+          git-config-name: cuda-python-bot
+          git-config-email: cuda-python-bot@users.noreply.github.com
+          folder: docs/
+          target-folder: docs/
+          commit-message: "Deploy coverage: ${{ env.COMMIT_HASH }}"
+          clean: false
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -80,6 +80,7 @@ repos:
     hooks:
       - id: actionlint
         args: ["-shellcheck="]
+        exclude: ^\.github/workflows/coverage.yml$
 
   - repo: https://github.com/MarcoGorelli/cython-lint
     rev: "d9ff7ce99ef4f2ae8fba93079ca9d76c4651d4ac"  # frozen: v0.18.0
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -207,6 +207,12 @@ flowchart TD
 - **Parallel execution**: Matrix builds across Python versions and platforms
 - **Component isolation**: Each component (core, bindings, pathfinder, python) can be built/released independently
 
+## Code coverage
+
+Code coverage reports are produced nightly and posted to [GitHub Pages](https://nvidia.github.io/cuda-python/coverage).
+
+Known limitations: Code coverage is only run on Linux x86_64 with an a100 GPU.  We plan to add more platform and GPU coverage in the future.
+
 ---
 
 <a>1</a>: The `cuda-python` meta package shares the same license and the contributing guidelines as those of `cuda-bindings`.
diff --git a/cuda_bindings/setup.py b/cuda_bindings/setup.py
@@ -44,6 +44,8 @@
 PARSER_CACHING = os.environ.get("CUDA_PYTHON_PARSER_CACHING", False)
 PARSER_CACHING = bool(PARSER_CACHING)
 
+COMPILE_FOR_COVERAGE = bool(int(os.environ.get("CUDA_PYTHON_COVERAGE", "0")))
+
 # ----------------------------------------------------------------------
 # Parse user-provided CUDA headers
 
@@ -276,6 +278,10 @@ def generate_output(infile, local):
     # extra_compile_args += ["-D _LIBCPP_ENABLE_ASSERTIONS"] # Consider: if clang, use libc++ preprocessor macros.
     else:
         extra_compile_args += ["-O3"]
+if COMPILE_FOR_COVERAGE:
+    # CYTHON_TRACE_NOGIL indicates to trace nogil functions.  It is not
+    # related to free-threading builds.
+    extra_compile_args += ["-DCYTHON_TRACE_NOGIL=1", "-DCYTHON_USE_SYS_MONITORING=0"]
 
 # For Setup
 extensions = []
@@ -342,10 +348,15 @@ def cleanup_dst_files():
 
 
 def do_cythonize(extensions):
+    compiler_directives = dict(language_level=3, embedsignature=True, binding=True, freethreading_compatible=True)
+
+    if COMPILE_FOR_COVERAGE:
+        compiler_directives["linetrace"] = True
+
     return cythonize(
         extensions,
         nthreads=nthreads,
-        compiler_directives=dict(language_level=3, embedsignature=True, binding=True, freethreading_compatible=True),
+        compiler_directives=compiler_directives,
         **extra_cythonize_kwargs,
     )
 
diff --git a/cuda_core/build_hooks.py b/cuda_core/build_hooks.py
@@ -22,6 +22,8 @@
 build_sdist = _build_meta.build_sdist
 get_requires_for_build_sdist = _build_meta.get_requires_for_build_sdist
 
+COMPILE_FOR_COVERAGE = bool(int(os.environ.get("CUDA_PYTHON_COVERAGE", "0")))
+
 
 @functools.cache
 def _get_proper_cuda_bindings_major_version() -> str:
@@ -84,24 +86,34 @@ def get_cuda_paths():
         print("CUDA paths:", CUDA_PATH)
         return CUDA_PATH
 
+    extra_compile_args = []
+    if COMPILE_FOR_COVERAGE:
+        # CYTHON_TRACE_NOGIL indicates to trace nogil functions.  It is not
+        # related to free-threading builds.
+        extra_compile_args += ["-DCYTHON_TRACE_NOGIL=1", "-DCYTHON_USE_SYS_MONITORING=0"]
+
     ext_modules = tuple(
         Extension(
             f"cuda.core.experimental.{mod.replace(os.path.sep, '.')}",
             sources=[f"cuda/core/experimental/{mod}.pyx"],
             include_dirs=list(os.path.join(root, "include") for root in get_cuda_paths()),
             language="c++",
+            extra_compile_args=extra_compile_args,
         )
         for mod in module_names
     )
 
     nthreads = int(os.environ.get("CUDA_PYTHON_PARALLEL_LEVEL", os.cpu_count() // 2))
     compile_time_env = {"CUDA_CORE_BUILD_MAJOR": int(_get_proper_cuda_bindings_major_version())}
+    compiler_directives = {"embedsignature": True, "warn.deprecated.IF": False, "freethreading_compatible": True}
+    if COMPILE_FOR_COVERAGE:
+        compiler_directives["linetrace"] = True
     _extensions = cythonize(
         ext_modules,
         verbose=True,
         language_level=3,
         nthreads=nthreads,
-        compiler_directives={"embedsignature": True, "warn.deprecated.IF": False, "freethreading_compatible": True},
+        compiler_directives=compiler_directives,
         compile_time_env=compile_time_env,
     )
 
