test-wheel-linux.yml

# Copyright (c) 2024-2025, NVIDIA CORPORATION & AFFILIATES. ALL RIGHTS RESERVED.
#
# SPDX-License-Identifier: Apache-2.0

name: "CI: Test wheels"

on:
  workflow_call:
    inputs:
      build-type:
        type: string
        required: true
      host-platform:
        type: string
        required: true
      build-ctk-ver:
        type: string
        required: true
      matrix_filter:
        type: string
        default: "."

defaults:
  run:
    shell: bash --noprofile --norc -xeuo pipefail {0}

jobs:
  compute-matrix:
    runs-on: ubuntu-latest
    env:
      BUILD_TYPE: ${{ inputs.build-type }}
      ARCH: ${{ (inputs.host-platform == 'linux-64' && 'amd64') ||
                (inputs.host-platform == 'linux-aarch64' && 'arm64') }}
    outputs:
      MATRIX: ${{ steps.compute-matrix.outputs.MATRIX }}
    steps:
      - name: Validate Test Type
        run: |
          if [[ "$BUILD_TYPE" != "pull-request" ]] && [[ "$BUILD_TYPE" != "nightly" ]] && [[ "$BUILD_TYPE" != "branch" ]]; then
              echo "Invalid build type! Must be one of 'nightly', 'pull-request', or 'branch'."
              exit 1
          fi
      - name: Compute Python Test Matrix
        id: compute-matrix
        run: |
          # Set a default GPU based upon architecture.
          gpu="l4"
          if [[ "${ARCH}" == "arm64" ]]; then
            gpu="a100"
          fi
          # Add a special entry for the H100 runner on amd64.
          special_runner=""
          if [[ "${ARCH}" == "amd64" ]]; then
            special_runner="- { ARCH: ${ARCH}, PY_VER: '3.13', CUDA_VER: '12.9.0', LOCAL_CTK: '1', GPU: 'H100', DRIVER: 'latest' }"
          fi

          # Please keep the matrices sorted in ascending order by the following:
          #
          #     [PY_VER, CUDA_VER, LOCAL_CTK, GPU, DRIVER]
          #
          # Note that DRIVER: `earliest` does not work with CUDA 12.9.0 and LOCAL_CTK: 0 does not work with CUDA 12.0.1.
          #
          export MATRICES="
            pull-request:
              - { ARCH: ${ARCH}, PY_VER: '3.9',  CUDA_VER: '11.8.0', LOCAL_CTK: '0', GPU: ${gpu}, DRIVER: 'earliest' }
              - { ARCH: ${ARCH}, PY_VER: '3.9',  CUDA_VER: '12.0.1', LOCAL_CTK: '1', GPU: ${gpu}, DRIVER: 'latest' }
              - { ARCH: ${ARCH}, PY_VER: '3.9',  CUDA_VER: '12.9.0', LOCAL_CTK: '0', GPU: ${gpu}, DRIVER: 'latest' }
              - { ARCH: ${ARCH}, PY_VER: '3.10', CUDA_VER: '11.8.0', LOCAL_CTK: '1', GPU: ${gpu}, DRIVER: 'earliest' }
              - { ARCH: ${ARCH}, PY_VER: '3.10', CUDA_VER: '12.9.0', LOCAL_CTK: '1', GPU: ${gpu}, DRIVER: 'latest' }
              - { ARCH: ${ARCH}, PY_VER: '3.11', CUDA_VER: '11.8.0', LOCAL_CTK: '1', GPU: ${gpu}, DRIVER: 'latest' }
              - { ARCH: ${ARCH}, PY_VER: '3.11', CUDA_VER: '12.9.0', LOCAL_CTK: '0', GPU: ${gpu}, DRIVER: 'latest' }
              - { ARCH: ${ARCH}, PY_VER: '3.12', CUDA_VER: '12.0.1', LOCAL_CTK: '1', GPU: ${gpu}, DRIVER: 'latest' }
              - { ARCH: ${ARCH}, PY_VER: '3.12', CUDA_VER: '12.9.0', LOCAL_CTK: '1', GPU: ${gpu}, DRIVER: 'latest' }
              - { ARCH: ${ARCH}, PY_VER: '3.13', CUDA_VER: '11.8.0', LOCAL_CTK: '0', GPU: ${gpu}, DRIVER: 'latest' }
              - { ARCH: ${ARCH}, PY_VER: '3.13', CUDA_VER: '12.0.1', LOCAL_CTK: '1', GPU: ${gpu}, DRIVER: 'latest' }
              - { ARCH: ${ARCH}, PY_VER: '3.13', CUDA_VER: '12.9.0', LOCAL_CTK: '0', GPU: ${gpu}, DRIVER: 'latest' }
              ${special_runner}
            nightly:
              - { ARCH: ${ARCH}, PY_VER: '3.9',  CUDA_VER: '11.8.0', LOCAL_CTK: '0', GPU: ${gpu}, DRIVER: 'earliest' }
              - { ARCH: ${ARCH}, PY_VER: '3.9',  CUDA_VER: '11.8.0', LOCAL_CTK: '1', GPU: ${gpu}, DRIVER: 'latest' }
              - { ARCH: ${ARCH}, PY_VER: '3.9',  CUDA_VER: '12.0.1', LOCAL_CTK: '1', GPU: ${gpu}, DRIVER: 'latest' }
              - { ARCH: ${ARCH}, PY_VER: '3.9',  CUDA_VER: '12.9.0', LOCAL_CTK: '0', GPU: ${gpu}, DRIVER: 'latest' }
              - { ARCH: ${ARCH}, PY_VER: '3.9',  CUDA_VER: '12.9.0', LOCAL_CTK: '1', GPU: ${gpu}, DRIVER: 'latest' }
              - { ARCH: ${ARCH}, PY_VER: '3.10', CUDA_VER: '11.8.0', LOCAL_CTK: '0', GPU: ${gpu}, DRIVER: 'earliest' }
              - { ARCH: ${ARCH}, PY_VER: '3.10', CUDA_VER: '11.8.0', LOCAL_CTK: '1', GPU: ${gpu}, DRIVER: 'latest' }
              - { ARCH: ${ARCH}, PY_VER: '3.10', CUDA_VER: '12.0.1', LOCAL_CTK: '1', GPU: ${gpu}, DRIVER: 'latest' }
              - { ARCH: ${ARCH}, PY_VER: '3.10', CUDA_VER: '12.9.0', LOCAL_CTK: '0', GPU: ${gpu}, DRIVER: 'latest' }
              - { ARCH: ${ARCH}, PY_VER: '3.10', CUDA_VER: '12.9.0', LOCAL_CTK: '1', GPU: ${gpu}, DRIVER: 'latest' }
              - { ARCH: ${ARCH}, PY_VER: '3.11', CUDA_VER: '11.8.0', LOCAL_CTK: '0', GPU: ${gpu}, DRIVER: 'earliest' }
              - { ARCH: ${ARCH}, PY_VER: '3.11', CUDA_VER: '11.8.0', LOCAL_CTK: '1', GPU: ${gpu}, DRIVER: 'latest' }
              - { ARCH: ${ARCH}, PY_VER: '3.11', CUDA_VER: '12.0.1', LOCAL_CTK: '1', GPU: ${gpu}, DRIVER: 'latest' }
              - { ARCH: ${ARCH}, PY_VER: '3.11', CUDA_VER: '12.9.0', LOCAL_CTK: '0', GPU: ${gpu}, DRIVER: 'latest' }
              - { ARCH: ${ARCH}, PY_VER: '3.11', CUDA_VER: '12.9.0', LOCAL_CTK: '1', GPU: ${gpu}, DRIVER: 'latest' }
              - { ARCH: ${ARCH}, PY_VER: '3.12', CUDA_VER: '11.8.0', LOCAL_CTK: '0', GPU: ${gpu}, DRIVER: 'earliest' }
              - { ARCH: ${ARCH}, PY_VER: '3.12', CUDA_VER: '11.8.0', LOCAL_CTK: '1', GPU: ${gpu}, DRIVER: 'latest' }
              - { ARCH: ${ARCH}, PY_VER: '3.12', CUDA_VER: '12.0.1', LOCAL_CTK: '1', GPU: ${gpu}, DRIVER: 'latest' }
              - { ARCH: ${ARCH}, PY_VER: '3.12', CUDA_VER: '12.9.0', LOCAL_CTK: '0', GPU: ${gpu}, DRIVER: 'latest' }
              - { ARCH: ${ARCH}, PY_VER: '3.12', CUDA_VER: '12.9.0', LOCAL_CTK: '1', GPU: ${gpu}, DRIVER: 'latest' }
              - { ARCH: ${ARCH}, PY_VER: '3.13', CUDA_VER: '11.8.0', LOCAL_CTK: '0', GPU: ${gpu}, DRIVER: 'earliest' }
              - { ARCH: ${ARCH}, PY_VER: '3.13', CUDA_VER: '11.8.0', LOCAL_CTK: '1', GPU: ${gpu}, DRIVER: 'latest' }
              - { ARCH: ${ARCH}, PY_VER: '3.13', CUDA_VER: '12.0.1', LOCAL_CTK: '1', GPU: ${gpu}, DRIVER: 'latest' }
              - { ARCH: ${ARCH}, PY_VER: '3.13', CUDA_VER: '12.9.0', LOCAL_CTK: '0', GPU: ${gpu}, DRIVER: 'latest' }
              - { ARCH: ${ARCH}, PY_VER: '3.13', CUDA_VER: '12.9.0', LOCAL_CTK: '1', GPU: ${gpu}, DRIVER: 'latest' }
              ${special_runner}
          "

          # Use the nightly matrix for branch tests
          MATRIX_TYPE="${BUILD_TYPE}"
          if [[ "${MATRIX_TYPE}" == "branch" ]]; then
            MATRIX_TYPE="nightly"
          fi
          export MATRIX_TYPE
          TEST_MATRIX=$(yq -n 'env(MATRICES) | .[strenv(MATRIX_TYPE)]')
          export TEST_MATRIX

          MATRIX="$(
            yq -n -o json 'env(TEST_MATRIX)' | \
            jq -c '${{ inputs.matrix_filter }} | if (. | length) > 0 then {include: .} else "Error: Empty matrix\n" | halt_error(1) end'
          )"

          echo "MATRIX=${MATRIX}" | tee --append "${GITHUB_OUTPUT}"

  test:
    name: py${{ matrix.PY_VER }}, ${{ matrix.CUDA_VER }}, ${{ (matrix.LOCAL_CTK == '1' && 'local') || 'wheels' }}, GPU ${{ matrix.GPU }}
    needs: compute-matrix
    strategy:
      fail-fast: false
      matrix: ${{ fromJSON(needs.compute-matrix.outputs.MATRIX) }}
    runs-on: "linux-${{ matrix.ARCH }}-gpu-${{ matrix.GPU }}-${{ matrix.DRIVER }}-1"
    # The build stage could fail but we want the CI to keep moving.
    if: ${{ github.repository_owner == 'nvidia' && !cancelled() }}
    # Our self-hosted runners require a container
    # TODO: use a different (nvidia?) container
    container:
      options: -u root --security-opt seccomp=unconfined --shm-size 16g
      image: ubuntu:22.04
      env:
        NVIDIA_VISIBLE_DEVICES: ${{ env.NVIDIA_VISIBLE_DEVICES }}
    steps:
      - name: Ensure GPU is working
        run: nvidia-smi

      - name: Checkout ${{ github.event.repository.name }}
        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683  # v4.2.2
        with:
          fetch-depth: 0

      - name: Install dependencies
        uses: ./.github/actions/install_unix_deps
        continue-on-error: false
        with:
          # for artifact fetching
          dependencies: "jq wget"
          dependent_exes: "jq wget"

      - name: Set environment variables
        env:
          BUILD_CUDA_VER: ${{ inputs.build-ctk-ver }}
          CUDA_VER: ${{ matrix.CUDA_VER }}
          HOST_PLATFORM: ${{ inputs.host-platform }}
          LOCAL_CTK: ${{ matrix.LOCAL_CTK }}
          PY_VER: ${{ matrix.PY_VER }}
          SHA: ${{ github.sha }}
        run: ./ci/tools/env-vars test

      - name: Download cuda-python build artifacts
        if: ${{ env.SKIP_CUDA_BINDINGS_TEST == '0'}}
        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093  # v4.3.0
        with:
          name: cuda-python-wheel
          path: .

      - name: Download cuda.bindings build artifacts
        if: ${{ env.SKIP_CUDA_BINDINGS_TEST == '0'}}
        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093  # v4.3.0
        with:
          name: ${{ env.CUDA_BINDINGS_ARTIFACT_NAME }}
          path: ${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }}

      - name: Download cuda-python & cuda.bindings build artifacts from the prior branch
        if: ${{ env.SKIP_CUDA_BINDINGS_TEST == '1'}}
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          # See https://github.com/cli/cli/blob/trunk/docs/install_linux.md#debian-ubuntu-linux-raspberry-pi-os-apt.
          # gh is needed for artifact fetching.
          mkdir -p -m 755 /etc/apt/keyrings \
                && out=$(mktemp) && wget -nv -O$out https://cli.github.com/packages/githubcli-archive-keyring.gpg \
                && cat $out | tee /etc/apt/keyrings/githubcli-archive-keyring.gpg > /dev/null \
          && chmod go+r /etc/apt/keyrings/githubcli-archive-keyring.gpg \
          && echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | tee /etc/apt/sources.list.d/github-cli.list > /dev/null \
          && apt update \
          && apt install gh -y

          OLD_BRANCH=$(cat .github/BACKPORT_BRANCH)
          OLD_BASENAME="cuda-bindings-python${PYTHON_VERSION_FORMATTED}-cuda*-${{ inputs.host-platform }}*"
          LATEST_PRIOR_RUN_ID=$(gh run list -b ${OLD_BRANCH} -L 1 -w "build-and-test.yml" -s completed -R NVIDIA/cuda-python --json databaseId | jq '.[]| .databaseId')
          if [[ "$LATEST_PRIOR_RUN_ID" == "" ]]; then
            echo "LATEST_PRIOR_RUN_ID not found!"
            exit 1
          fi

          gh run download $LATEST_PRIOR_RUN_ID -p ${OLD_BASENAME} -R NVIDIA/cuda-python
          ls -al $OLD_BASENAME
          mkdir -p "${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }}"
          mv $OLD_BASENAME/*.whl "${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }}"/
          rmdir $OLD_BASENAME

          gh run download $LATEST_PRIOR_RUN_ID -p cuda-python-wheel -R NVIDIA/cuda-python
          ls -al cuda-python-wheel
          mv cuda-python-wheel/*.whl .
          rmdir cuda-python-wheel

      - name: Display structure of downloaded cuda-python artifacts
        run: |
          pwd
          ls -lahR .

      - name: Display structure of downloaded cuda.bindings artifacts
        run: |
          pwd
          ls -lahR $CUDA_BINDINGS_ARTIFACTS_DIR

      - name: Download cuda.bindings Cython tests
        if: ${{ env.SKIP_CYTHON_TEST == '0' }}
        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093  # v4.3.0
        with:
          name: ${{ env.CUDA_BINDINGS_ARTIFACT_NAME }}-tests
          path: ${{ env.CUDA_BINDINGS_CYTHON_TESTS_DIR }}

      - name: Display structure of downloaded cuda.bindings Cython tests
        if: ${{ env.SKIP_CYTHON_TEST == '0' }}
        run: |
          pwd
          ls -lahR $CUDA_BINDINGS_CYTHON_TESTS_DIR

      - name: Download cuda.core build artifacts
        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093  # v4.3.0
        with:
          name: ${{ env.CUDA_CORE_ARTIFACT_NAME }}
          path: ${{ env.CUDA_CORE_ARTIFACTS_DIR }}

      - name: Display structure of downloaded cuda.core build artifacts
        run: |
          pwd
          ls -lahR $CUDA_CORE_ARTIFACTS_DIR

      - name: Download cuda.core Cython tests
        if: ${{ env.SKIP_CYTHON_TEST == '0' }}
        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093  # v4.3.0
        with:
          name: ${{ env.CUDA_CORE_ARTIFACT_NAME }}-tests
          path: ${{ env.CUDA_CORE_CYTHON_TESTS_DIR }}

      - name: Display structure of downloaded cuda.core Cython tests
        if: ${{ env.SKIP_CYTHON_TEST == '0' }}
        run: |
          pwd
          ls -lahR $CUDA_CORE_CYTHON_TESTS_DIR

      - name: Set up Python ${{ matrix.PY_VER }}
        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065  # v5.6.0
        with:
          python-version: ${{ matrix.PY_VER }}
        env:
          # we use self-hosted runners on which setup-python behaves weirdly...
          AGENT_TOOLSDIRECTORY: "/opt/hostedtoolcache"

      - name: Set up mini CTK
        if: ${{ matrix.LOCAL_CTK == '1' }}
        uses: ./.github/actions/fetch_ctk
        continue-on-error: false
        with:
          host-platform: ${{ inputs.host-platform }}
          cuda-version: ${{ matrix.CUDA_VER }}

      - name: Set up latest cuda_sanitizer_api
        if: ${{ env.SETUP_SANITIZER == '1' }}
        uses: ./.github/actions/fetch_ctk
        continue-on-error: false
        with:
          host-platform: ${{ inputs.host-platform }}
          cuda-version: ${{ env.LATEST_CUDA_VERSION }}
          cuda-components: "cuda_sanitizer_api"

      - name: Set up compute-sanitizer
        run: setup-sanitizer

      - name: Run cuda.bindings tests
        if: ${{ env.SKIP_CUDA_BINDINGS_TEST == '0' }}
        env:
          CUDA_VER: ${{ matrix.CUDA_VER }}
          LOCAL_CTK: ${{ matrix.LOCAL_CTK }}
        run: run-tests bindings

      - name: Run cuda.core tests
        env:
          CUDA_VER: ${{ matrix.CUDA_VER }}
          LOCAL_CTK: ${{ matrix.LOCAL_CTK }}
        run: run-tests core

      - name: Ensure cuda-python installable
        run: |
          if [[ "${{ matrix.LOCAL_CTK }}" == 1 ]]; then
            pip install cuda_python*.whl
          else
            pip install $(ls cuda_python*.whl)[all]
          fi