test-wheel-windows.yml

# SPDX-FileCopyrightText: Copyright (c) 2024-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
#
# SPDX-License-Identifier: Apache-2.0

name: "CI: Test wheels"

on:
  workflow_call:
    inputs:
      build-type:
        type: string
        required: true
      host-platform:
        type: string
        required: true
      build-ctk-ver:
        type: string
        required: true
      matrix_filter:
        type: string
        default: "."

jobs:
  compute-matrix:
    runs-on: ubuntu-latest
    defaults:
      run:
        shell: bash --noprofile --norc -xeuo pipefail {0}
    env:
      BUILD_TYPE: ${{ inputs.build-type }}
      ARCH: ${{ (inputs.host-platform == 'win-64' && 'amd64') }}
    outputs:
      MATRIX: ${{ steps.compute-matrix.outputs.MATRIX }}
    steps:
      - name: Checkout ${{ github.event.repository.name }}
        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8  # v5.0.0
        with:
          fetch-depth: 1

      - name: Validate Test Type
        run: |
          if [[ "$BUILD_TYPE" != "pull-request" ]] && [[ "$BUILD_TYPE" != "nightly" ]] && [[ "$BUILD_TYPE" != "branch" ]]; then
              echo "Invalid build type! Must be one of 'nightly', 'pull-request', or 'branch'."
              exit 1
          fi
      - name: Compute Python Test Matrix
        id: compute-matrix
        run: |
          # Use the nightly matrix for branch tests
          MATRIX_TYPE="${BUILD_TYPE}"
          if [[ "${MATRIX_TYPE}" == "branch" ]]; then
            MATRIX_TYPE="nightly"
          fi

          # Read base matrix from JSON file for the specific architecture
          TEST_MATRIX=$(jq --arg arch "$ARCH" --arg matrix_type "$MATRIX_TYPE" '
            .windows[$matrix_type] |
            map(select(.ARCH == $arch))
          ' ci/test-matrix.json)

          MATRIX="$(
            jq -c '${{ inputs.matrix_filter }} | if (. | length) > 0 then {include: .} else "Error: Empty matrix\n" | halt_error(1) end' <<< "$TEST_MATRIX"
          )"

          echo "MATRIX=${MATRIX}" | tee --append "${GITHUB_OUTPUT}"

  test:
    # TODO: switch to this once the self-hosted runners are ready
    # name: py${{ matrix.PY_VER }}, ${{ matrix.CUDA_VER }}, ${{ (matrix.LOCAL_CTK == '1' && 'local') || 'wheels' }}, GPU ${{ matrix.GPU }}
    name: py${{ matrix.PY_VER }}, ${{ matrix.CUDA_VER }}, ${{ (matrix.LOCAL_CTK == '1' && 'local') || 'wheels' }}
    # The build stage could fail but we want the CI to keep moving.
    needs: compute-matrix
    strategy:
      fail-fast: false
      matrix: ${{ fromJSON(needs.compute-matrix.outputs.MATRIX) }}
    if: ${{ github.repository_owner == 'nvidia' && !cancelled() }}
    # TODO: switch to self-hosted runners once they are ready
    # runs-on: "windows-${{ matrix.ARCH }}-gpu-${{ matrix.GPU }}-${{ matrix.DRIVER }}-1"
    runs-on: 'cuda-python-windows-gpu-github'
    steps:
      - name: Checkout ${{ github.event.repository.name }}
        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683  # v4.2.2
        with:
          fetch-depth: 0

      # TODO: use setup-proxy-cache once we have self-hosted Windows runners
      # - name: Setup proxy cache
      #   uses: nv-gha-runners/setup-proxy-cache@main
      #   continue-on-error: true

      - name: Update driver
        run: |
          .github/workflows/install_gpu_driver.ps1

      - name: Ensure GPU is working
        run: nvidia-smi

      # TODO: remove this block once self-hosted runners are ready
      - name: Install Git for Windows
        # the GPU runner image does not have Git Bash pre-installed...
        env:
          # doesn't seem there's an easy way to avoid hard-coding it?
          GFW_EXE_URL: https://github.com/git-for-windows/git/releases/download/v2.49.0.windows.1/PortableGit-2.49.0-64-bit.7z.exe
        run: |
          Invoke-WebRequest -Uri "$env:GFW_EXE_URL" -OutFile "PortableGit.7z.exe"
          # Self-extracting, see https://gitforwindows.org/zip-archives-extracting-the-released-archives.html
          Start-Process .\PortableGit.7z.exe -Wait -Verbose -ArgumentList '-y -gm2'
          ls -l PortableGit
          echo "$((Get-Location).Path)\\PortableGit\\bin" >> $env:GITHUB_PATH
          $env:Path += ";$((Get-Location).Path)\\PortableGit\\bin"
          bash --version

      - name: Set environment variables
        env:
          BUILD_CUDA_VER: ${{ inputs.build-ctk-ver }}
          CUDA_VER: ${{ matrix.CUDA_VER }}
          HOST_PLATFORM: ${{ inputs.host-platform }}
          LOCAL_CTK: ${{ matrix.LOCAL_CTK }}
          PY_VER: ${{ matrix.PY_VER }}
          SHA: ${{ github.sha }}
        shell: bash --noprofile --norc -xeuo pipefail {0}
        run: ./ci/tools/env-vars test

      - name: Download cuda-python build artifacts
        if: ${{ env.SKIP_CUDA_BINDINGS_TEST == '0'}}
        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093  # v4.3.0
        with:
          name: cuda-python-wheel
          path: .

      - name: Download cuda.bindings build artifacts
        if: ${{ env.SKIP_CUDA_BINDINGS_TEST == '0'}}
        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093  # v4.3.0
        with:
          name: ${{ env.CUDA_BINDINGS_ARTIFACT_NAME }}
          path: ${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }}

      # TODO: remove this block once self-hosted runners are ready
      - name: Install gh cli
        # the GPU runner image does not have gh pre-installed...
        env:
          # doesn't seem there's an easy way to avoid hard-coding it?
          GH_MSI_URL: https://github.com/cli/cli/releases/download/v2.67.0/gh_2.67.0_windows_amd64.msi
        run: |
          Invoke-WebRequest -Uri "$env:GH_MSI_URL" -OutFile "gh_installer.msi"
          Start-Process msiexec.exe -Wait -Verbose -ArgumentList '/i "gh_installer.msi" /qn'
          $GH_POSSIBLE_PATHS = "C:\\Program Files\\GitHub CLI", "C:\\Program Files (x86)\\GitHub CLI"
          foreach ($p in $GH_POSSIBLE_PATHS) {
              echo "$p" >> $env:GITHUB_PATH
              $env:Path += ";$p"
          }
          gh --version

      - name: Download cuda-pathfinder build artifacts from main branch
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          $MAIN_BRANCH = "main"
          $PATHFINDER_BASENAME = "cuda-pathfinder-wheel"
          $runData = gh run list -b $MAIN_BRANCH -L 1 -w "CI" -s completed -R NVIDIA/cuda-python --json databaseId | ConvertFrom-Json
          if (-not $runData -or $runData.Length -eq 0 -or -not $runData[0].databaseId -or [string]::IsNullOrEmpty($runData[0].databaseId)) {
              Write-Host "LATEST_MAIN_RUN_ID not found!"
              exit 1
          }
          $LATEST_MAIN_RUN_ID = $runData[0].databaseId

          gh run download $LATEST_MAIN_RUN_ID -p $PATHFINDER_BASENAME -R NVIDIA/cuda-python
          Get-ChildItem -Path $PATHFINDER_BASENAME
          New-Item -Path "./cuda_pathfinder" -ItemType Directory -Force
          Move-Item -Path "$PATHFINDER_BASENAME/*.whl" -Destination "./cuda_pathfinder/"
          Remove-Item -Path $PATHFINDER_BASENAME -Force

      - name: Install zstd
        # the GPU runner image does not have zstd pre-installed... and it's needed by actions/cache
        if: ${{ matrix.LOCAL_CTK == '1' }}
        env:
          # doesn't seem there's an easy way to avoid hard-coding it?
          ZSTD_URL: https://github.com/facebook/zstd/releases/download/v1.5.7/zstd-v1.5.7-win64.zip
          ZSTD_DIR: zstd-v1.5.7-win64
        run: |
          Invoke-WebRequest -Uri "$env:ZSTD_URL" -OutFile "zstd-win64.zip"
          Expand-Archive -Path "zstd-win64.zip" -DestinationPath .
          ls -l $env:ZSTD_DIR
          echo "$((Get-Location).Path)\\$env:ZSTD_DIR" >> $env:GITHUB_PATH
          $env:Path += ";$((Get-Location).Path)\\$env:ZSTD_DIR"
          zstd --version

      - name: Download cuda-python & cuda.bindings build artifacts from the prior branch
        if: ${{ env.SKIP_CUDA_BINDINGS_TEST == '1'}}
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          $OLD_BRANCH = Get-Content .github/BACKPORT_BRANCH
          $OLD_BASENAME = "cuda-bindings-python${env:PYTHON_VERSION_FORMATTED}-cuda*-${{ inputs.host-platform }}*"
          $runData = gh run list -b $OLD_BRANCH -L 1 -w "build-and-test.yml" -s completed -R NVIDIA/cuda-python --json databaseId | ConvertFrom-Json
          if (-not $runData -or $runData.Length -eq 0 -or -not $runData[0].databaseId -or [string]::IsNullOrEmpty($runData[0].databaseId)) {
              Write-Host "LATEST_PRIOR_RUN_ID not found!"
              exit 1
          }
          $LATEST_PRIOR_RUN_ID = $runData[0].databaseId

          gh run download $LATEST_PRIOR_RUN_ID -p $OLD_BASENAME -R NVIDIA/cuda-python
          Get-ChildItem -Path $OLD_BASENAME
          New-Item -Path "${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }}" -ItemType Directory -Force
          Move-Item -Path "$OLD_BASENAME/*.whl" -Destination "${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }}"
          Remove-Item -Path $OLD_BASENAME -Force

          gh run download $LATEST_PRIOR_RUN_ID -p cuda-python-wheel -R NVIDIA/cuda-python
          Get-ChildItem -Path cuda-python-wheel
          Move-Item -Path "cuda-python-wheel/*.whl" -Destination .
          Remove-Item -Path cuda-python-wheel -Force

      - name: Display structure of downloaded cuda-python artifacts
        run: |
          Get-Location
          Get-ChildItem -Recurse -Force | Select-Object Mode, LastWriteTime, Length, FullName

      - name: Display structure of downloaded cuda.bindings artifacts
        run: |
          Get-Location
          Get-ChildItem -Recurse -Force $env:CUDA_BINDINGS_ARTIFACTS_DIR | Select-Object Mode, LastWriteTime, Length, FullName

      - name: Download cuda.bindings Cython tests
        if: ${{ env.SKIP_CYTHON_TEST == '0' }}
        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093  # v4.3.0
        with:
          name: ${{ env.CUDA_BINDINGS_ARTIFACT_NAME }}-tests
          path: ${{ env.CUDA_BINDINGS_CYTHON_TESTS_DIR }}

      - name: Display structure of downloaded cuda.bindings Cython tests
        if: ${{ env.SKIP_CYTHON_TEST == '0' }}
        run: |
          Get-Location
          Get-ChildItem -Recurse -Force $env:CUDA_BINDINGS_CYTHON_TESTS_DIR | Select-Object Mode, LastWriteTime, Length, FullName



      - name: Set up Python ${{ matrix.PY_VER }}
        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065  # v5.6.0
        with:
          python-version: ${{ matrix.PY_VER }}

      - name: Set up mini CTK
        if: ${{ matrix.LOCAL_CTK == '1' }}
        uses: ./.github/actions/fetch_ctk
        continue-on-error: false
        with:
          host-platform: ${{ inputs.host-platform }}
          cuda-version: ${{ matrix.CUDA_VER }}

      - name: Run cuda.bindings tests
        if: ${{ env.SKIP_CUDA_BINDINGS_TEST == '0' }}
        env:
          CUDA_VER: ${{ matrix.CUDA_VER }}
          LOCAL_CTK: ${{ matrix.LOCAL_CTK }}
        shell: bash --noprofile --norc -xeuo pipefail {0}
        run: run-tests bindings



      - name: Ensure cuda-python installable
        run: |
          if ('${{ matrix.LOCAL_CTK }}' -eq '1') {
            pip install (Get-ChildItem -Filter cuda_python*.whl).FullName
          } else {
            pip install "$((Get-ChildItem -Filter cuda_python*.whl).FullName)[all]"
          }