run_ock_external_tests.yml

# Simple workflow for running external tests
name: Run external tests
on:
  workflow_call:
    inputs:
      ock:
        required: false
        type: boolean
        description: 'Enable ock artefact building'
        default: true
      test_tornado:
        required: false
        type: boolean
        description: 'Enable testing tornado'
        default: true
      test_remote_hal:
        required: false
        type: boolean
        description: 'Enable testing remote_hal'
        default: true
      test_sanitizers:
        required: false
        type: boolean
        description: 'Enable testing sanitizers'
        default: true
      test_sycl_cts:
        required: false
        type: boolean
        description: 'Enable testing sycl-cts'
        default: true
      test_sycl_e2e:
        required: false
        type: boolean
        description: 'Enable testing sycl-e2e'
        default: true
      test_opencl_cts:
        required: false
        type: boolean
        description: 'Enable testing opencl-cts'
        default: true
      target_list:
        required: false
        description: 'List of enabled targets'
        type: string
      llvm_version:
        required: false
        type: string
        description: 'llvm major version (e.g 20,21,main) - to be used for llvm specific fails'
        default: 20
      llvm_branch:
        required: false
        type: string
        description: 'The actual llvm branch to check out e.g. release/20.x'
        default: 'release/20.x'
      download_ock_artefact:
        required: false
        type: string
        description: 'download ock artefact rather than building, of form <target>=id;<target2=id2>.'
        default: ''
      download_dpcpp_artefact:
        required: false
        type: string
        description: 'download dpcpp from previous artefact run e.g. host_x86_64_linux=12915462445;host_aarch64_linux=12915462445'
        default: ''
      download_sycl_cts_artefact:
        required: false
        type: string
        description: 'download sycl cts from previous artefact run e.g. host_x86_64_linux=12915462445;host_aarch64_linux=12915462445'
        default: ''
      llvm_source:
        required: false
        type: string
        description: 'method of sourcing llvm (install, cache or run-id)'
        default: 'cache'
      save_cache:
        required: false
        type: boolean
        description: 'save any caches required during building of any artefacts (usually on main only)'
        default: false
      dpcpp_repo_pr_no:
        required: false
        type: string
        description: 'dpc++ repo ref: enter PR number (else default branch is used)'
        default: ""
     
permissions:
  packages: read

jobs:

  # Calculate some useful variables that can be used through the workflow
  # Currently this can be used to exclude all but certain targets in matrices
  workflow_vars:
    runs-on: ubuntu-22.04
    outputs:
      matrix_only_linux_x86_64_aarch64_riscv64: ${{ steps.vars.outputs.matrix_only_linux_x86_64_aarch64_riscv64 }}
      matrix_only_linux_x86_64_aarch64:         ${{ steps.vars.outputs.matrix_only_linux_x86_64_aarch64 }}
      matrix_only_linux_x86_64:                 ${{ steps.vars.outputs.matrix_only_linux_x86_64 }}
      matrix_only_linux_aarch64:                ${{ steps.vars.outputs.matrix_only_linux_aarch64 }}
      matrix_only_linux_riscv64:                ${{ steps.vars.outputs.matrix_only_linux_riscv64 }}
    steps:
    - id: vars
      # TODO: If we expand on this, come up with a more programmatical way of doing only certain targets.
      # These variables are for excluding certain targets from the total list, which is why just including
      # certain targets involves a long list of excludes
      run: |
        echo matrix_only_linux_x86_64_aarch64_riscv64="[ \
           {\"target\": \"host_arm_linux\"}, \
           {\"target\": \"refsi_riscv64_linux\"}, \
           {\"target\": \"host_i686_linux\"}, \
           {\"target\": \"host_x86_64_windows\"}]" >> $GITHUB_OUTPUT
        echo matrix_only_linux_x86_64_aarch64="[ \
           {\"target\": \"host_arm_linux\"}, \
           {\"target\": \"host_riscv64_linux\"}, \
           {\"target\": \"refsi_riscv64_linux\"}, \
           {\"target\": \"host_i686_linux\"}, \
           {\"target\": \"host_x86_64_windows\"}]" >> $GITHUB_OUTPUT
        echo matrix_only_linux_x86_64="[ \
           {\"target\": \"host_aarch64_linux\"}, \
           {\"target\": \"host_riscv64_linux\"}, \
           {\"target\": \"host_arm_linux\"}, \
           {\"target\": \"refsi_riscv64_linux\"}, \
           {\"target\": \"host_i686_linux\"}, \
           {\"target\": \"host_x86_64_windows\"}]" >> $GITHUB_OUTPUT
        echo matrix_only_linux_aarch64="[ \
           {\"target\": \"host_x86_64_linux\"}, \
           {\"target\": \"host_riscv64_linux\"}, \
           {\"target\": \"host_arm_linux\"}, \
           {\"target\": \"refsi_riscv64_linux\"}, \
           {\"target\": \"host_i686_linux\"}, \
           {\"target\": \"host_x86_64_windows\"}]" >> $GITHUB_OUTPUT
        echo matrix_only_linux_riscv64="[ \
           {\"target\": \"host_x86_64_linux\"}, \
           {\"target\": \"host_aarch64_linux\"}, \
           {\"target\": \"host_arm_linux\"}, \
           {\"target\": \"refsi_riscv64_linux\"}, \
           {\"target\": \"host_i686_linux\"}, \
           {\"target\": \"host_x86_64_windows\"}]" >> $GITHUB_OUTPUT
        cat $GITHUB_OUTPUT    

  create_ock_artefacts_windows:
    needs: [workflow_vars]
    # Currently there is only one windows target so we don't bother with the
    # matrix, just check if it's enabled
    runs-on: windows-2025
    if: inputs.ock && contains(inputs.target_list, 'host_x86_64_windows')
    steps:
      - name: Checkout repo
        uses: actions/checkout@v6
      # Todo: move to a common place e.g. setup_build
      - name: Setup Windows llvm base
        uses: llvm/actions/setup-windows@a1ea791b03c8e61f53a0e66f2f73db283aa0f01e # main branch
        with:
          arch: amd64
      - name: build ock artefact
        uses: ./.github/actions/do_build_ock_artefact
        with:
          target: host_x86_64_windows
          llvm_version: ${{ inputs.llvm_version }}
          llvm_source: ${{ inputs.llvm_source }}


  create_ock_artefacts_ubuntu:
    needs: [workflow_vars]
    strategy:
      fail-fast: false  # let all matrix jobs complete
      matrix:
        target: ${{ fromJson(inputs.target_list) }}
        exclude: ${{ fromJson(needs.workflow_vars.outputs.matrix_only_linux_x86_64_aarch64_riscv64) }}

    # risc-v needs ubuntu 24.04 so we get the latest qemu as well as how we
    # build llvm. Otherwise we choose ubuntu-22.04 (use a container for both for consistency).
    runs-on: ${{ contains(matrix.target, 'host_aarch64') && 'ubuntu-22.04-arm' || (contains(matrix.target, 'host_riscv64') && 'ubuntu-24.04' || 'ubuntu-22.04') }}
    container:
      image: ${{  contains(matrix.target, 'host_riscv')   && 'ghcr.io/uxlfoundation/ock_ubuntu_24.04-x86-64:latest'
             || ( contains(matrix.target, 'host_aarch64') && 'ghcr.io/uxlfoundation/ock_ubuntu_22.04-aarch64:latest'
             ||                                              'ghcr.io/uxlfoundation/ock_ubuntu_22.04-x86-64:latest' ) }}
      volumes:
        - ${{github.workspace}}:${{github.workspace}}
    if: inputs.ock && contains(inputs.target_list, 'linux')
    steps:
      - name: Checkout repo
        uses: actions/checkout@v6
      - uses: ./.github/actions/setup_gh
        with:
          os: ${{ contains( matrix.target, 'windows') && 'windows' || 'ubuntu' }}
          token: ${{ secrets.GITHUB_TOKEN }}
      - name: build ock artefact
        uses: ./.github/actions/do_build_ock_artefact
        with:
          target: ${{ matrix.target }}
          llvm_version: ${{ inputs.llvm_version }}
          download_ock_artefact: ${{ inputs.download_ock_artefact }}
          llvm_source: ${{ inputs.llvm_source }}

  build_icd:
    if: inputs.ock && ( inputs.test_tornado || inputs.test_opencl_cts || inputs.test_sycl_cts || inputs.test_sycl_e2e) && contains(inputs.target_list, 'linux')
    needs: [workflow_vars]
    strategy:
      fail-fast: false  # let all matrix jobs complete
      matrix:
        target: ${{ fromJson(inputs.target_list) }}
        exclude: ${{ fromJson(needs.workflow_vars.outputs.matrix_only_linux_x86_64_aarch64_riscv64) }}

    runs-on: ${{ contains(matrix.target, 'host_aarch64') && 'ubuntu-22.04-arm' || (contains(matrix.target, 'host_riscv64') && 'ubuntu-24.04' || 'ubuntu-22.04') }}
    container:
      image: ${{  contains(matrix.target, 'host_riscv')   && 'ghcr.io/uxlfoundation/ock_ubuntu_24.04-x86-64:latest'
             || ( contains(matrix.target, 'host_aarch64') && 'ghcr.io/uxlfoundation/ock_ubuntu_22.04-aarch64:latest'
             ||                                              'ghcr.io/uxlfoundation/ock_ubuntu_22.04-x86-64:latest' ) }}
      volumes:
        - ${{github.workspace}}:${{github.workspace}}
    steps:
      - name: clone ock platform
        uses: actions/checkout@v6
        with:
          sparse-checkout: |
            platform
            .github
      - name : build and upload icd ${{matrix.target}}
        uses: ./.github/actions/do_build_icd
        with:
          target: ${{matrix.target}}

  build_run_sanitizers:
    if: inputs.test_sanitizers
    needs: [ workflow_vars ]
    strategy:
      fail-fast: false  # let all matrix jobs complete
      matrix:
        san_type: [ 'tsan', 'aubsan' ]
        # sanitizer jobs are long-running in their original format and close to the
        # default job timeout value. Splitting into part1/part2 trims c.45 mins from
        # total matrix run time across all jobs.
        part_id: [ 'part1', 'part2' ]
    runs-on: 'ubuntu-22.04'
    container:
      image: 'ghcr.io/uxlfoundation/ock_ubuntu_22.04-x86-64:latest'
      volumes:
        - ${{github.workspace}}:${{github.workspace}}
    steps:
      - name: Checkout repo
        uses: actions/checkout@v6
      - name: setup-ubuntu
        uses: ./.github/actions/setup_build
        with:
          llvm_version: ${{ inputs.llvm_version }}
          llvm_source: ${{ inputs.llvm_source}}
          github_token: ${{ secrets.GITHUB_TOKEN }}
      - name : build run sanitizers
        uses: ./.github/actions/do_build_run_sanitizers
        with:
          san_type: ${{ matrix.san_type }}
          part_id: ${{ matrix.part_id }}
          llvm_branch: ${{ inputs.llvm_branch }}

  build_run_remote_hal:
    if: inputs.test_remote_hal
    needs: [ workflow_vars ]
    runs-on: 'ubuntu-24.04'
    container:
      image: 'ghcr.io/uxlfoundation/ock_ubuntu_24.04-x86-64:latest'
      volumes:
        - ${{github.workspace}}:${{github.workspace}}
    steps:
      - name: Checkout repo
        uses: actions/checkout@v6
      - name: setup-ubuntu
        uses: ./.github/actions/setup_build
        with:
          llvm_version: ${{ inputs.llvm_version }}
          llvm_source: ${{ inputs.llvm_source}}
          github_token: ${{ secrets.GITHUB_TOKEN }}
      - name : build run remote_hal
        uses: ./.github/actions/do_build_run_remote_hal

  # Currently only builds and runs on x86_64 linux
  build_run_tornado:
    if: inputs.test_tornado
    needs: [ workflow_vars, build_icd, create_ock_artefacts_ubuntu ]
    strategy:
      fail-fast: false  # let all matrix jobs complete
      matrix:
        target: ${{ fromJson(inputs.target_list) }}
        exclude: ${{ fromJson(needs.workflow_vars.outputs.matrix_only_linux_x86_64_aarch64) }}

    runs-on: ${{ matrix.target == 'host_x86_64_linux' && 'ubuntu-22.04' || 'ubuntu-22.04-arm' }}
    steps:
      - name: clone ock platform
        uses: actions/checkout@v6
        with:
          sparse-checkout: |
            platform
            .github
      # TODO: Consider separating out tornado build and run in the future
      - name : build and upload tornado
        uses: ./.github/actions/do_build_tornado
        with:
          target: ${{ matrix.target }}
      - name : run tornado
        uses: ./.github/actions/run_tornado
        with:
          target: ${{ matrix.target }}

  build_run_opencl_cts_x86_64_aarch64:
    if: inputs.test_opencl_cts && (contains(inputs.target_list, 'host_x86_64_linux') || contains(inputs.target_list, 'host_aarch64_linux'))
    needs: [ workflow_vars, build_icd, create_ock_artefacts_ubuntu ]
    strategy:
      fail-fast: false  # let all matrix jobs complete
      matrix:
        target: ${{ fromJson(inputs.target_list) }}
        exclude: ${{ fromJson(needs.workflow_vars.outputs.matrix_only_linux_x86_64_aarch64) }}

    runs-on: ${{ contains(matrix.target, 'host_aarch64') && 'ubuntu-22.04-arm' || 'ubuntu-22.04' }}
    container:
      image: ${{  contains(matrix.target, 'host_riscv')   && 'ghcr.io/uxlfoundation/ock_ubuntu_24.04-x86-64:latest'
             || ( contains(matrix.target, 'host_aarch64') && 'ghcr.io/uxlfoundation/ock_ubuntu_22.04-aarch64:latest'
             ||                                              'ghcr.io/uxlfoundation/ock_ubuntu_22.04-x86-64:latest' ) }}
      volumes:
        - ${{github.workspace}}:${{github.workspace}}
    steps:
      - name: clone ock
        uses: actions/checkout@v6
        with:
          # scripts: for run_cities.py
          # source: for CTS filter.csv files
          # platform: for toolchain files
          sparse-checkout: |
            scripts
            source
            platform
            .github
      # TODO: Consider separating out opencl_cts build and run in the future
      - name : build and upload opencl_cts
        uses: ./.github/actions/do_build_opencl_cts
        with:
          target: ${{ matrix.target }}
      - name : run opencl_cts
        uses: ./.github/actions/run_opencl_cts
        with:
          target: ${{ matrix.target }}
          llvm_version: ${{ inputs.llvm_version }}

  build_opencl_cts_riscv64:
    if: inputs.test_opencl_cts && (contains(inputs.target_list, 'host_riscv64_linux'))
    needs: [ workflow_vars, build_icd, create_ock_artefacts_ubuntu ]
    runs-on: 'ubuntu-24.04'
    container:
      image: 'ghcr.io/uxlfoundation/ock_ubuntu_24.04-x86-64:latest'
      volumes:
        - ${{github.workspace}}:${{github.workspace}}
    steps:
      - name: clone ock
        uses: actions/checkout@v6
        with:
          # scripts: for run_cities.py
          # source: for CTS filter.csv files
          # platform: for toolchain files
          # .github for actions
          sparse-checkout: |
            scripts
            source
            platform
            .github
      - name : build and upload opencl_cts
        uses: ./.github/actions/do_build_opencl_cts
        with:
          target: host_riscv64_linux

  run_opencl_cts_riscv64:
    if: inputs.test_opencl_cts && (contains(inputs.target_list, 'host_riscv64_linux'))
    needs: [ workflow_vars, build_icd, create_ock_artefacts_ubuntu, build_opencl_cts_riscv64 ]
    runs-on: 'ubuntu-24.04'
    strategy:
      fail-fast: false  # let all matrix jobs complete
      matrix:
        split_index: [ 0, 1 ]
    container:
      image: 'ghcr.io/uxlfoundation/ock_ubuntu_24.04-x86-64:latest'
      volumes:
        - ${{github.workspace}}:${{github.workspace}}
    steps:
      - name: clone ock
        uses: actions/checkout@v6
        with:
          # scripts: for run_cities.py
          # source: for CTS filter.csv files
          # platform: for toolchain files
          # .github for actions          
          sparse-checkout: |
            scripts
            source
            platform
            .github
      - name : run opencl_cts
        uses: ./.github/actions/run_opencl_cts
        with:
          target: host_riscv64_linux
          llvm_version: ${{ inputs.llvm_version }}
          split_index: ${{ matrix.split_index }}

  build_dpcpp_native_x86_64:
    needs: [workflow_vars]
    runs-on: 'ubuntu-24.04'
    container:
      image: 'ghcr.io/uxlfoundation/ock_ubuntu_22.04-x86-64:latest'
      volumes:
        - ${{github.workspace}}:${{github.workspace}}

    if: (inputs.test_sycl_cts || inputs.test_sycl_e2e) && ( contains(inputs.target_list, 'host_x86_64_linux') || contains(inputs.target_list, 'host_riscv64_linux') )
    steps:
      - name: Checkout repo
        uses: actions/checkout@v6
      - name: set up gh
        uses: ./.github/actions/setup_gh
        with:
          os: 'ubuntu'
          token: ${{ secrets.GITHUB_TOKEN }}
      - name: build dpc++ artefact
        uses: ./.github/actions/do_build_dpcpp
        with:
          target: host_x86_64_linux
          download_dpcpp_artefact: ${{ inputs.download_dpcpp_artefact }}
          save_cache: ${{ inputs.save_cache }}
          dpcpp_repo_pr_no: ${{ inputs.dpcpp_repo_pr_no }}

  build_dpcpp_native_aarch64:
    needs: [workflow_vars]
    runs-on: 'ubuntu-24.04-arm'
    container:
      image: 'ghcr.io/uxlfoundation/ock_ubuntu_22.04-aarch64:latest'
      volumes:
        - ${{github.workspace}}:${{github.workspace}}

    if: (inputs.test_sycl_cts || inputs.test_sycl_e2e) && contains(inputs.target_list, 'host_aarch64_linux')
    steps:
      - name: Checkout repo
        uses: actions/checkout@v6
      - name: set up gh
        uses: ./.github/actions/setup_gh
        with:
          os: 'ubuntu'
          token: ${{ secrets.GITHUB_TOKEN }}
      - name: build dpc++ artefact
        uses: ./.github/actions/do_build_dpcpp
        with:
          target: host_aarch64_linux
          download_dpcpp_artefact: ${{ inputs.download_dpcpp_artefact }}
          save_cache: ${{ inputs.save_cache }}
          dpcpp_repo_pr_no: ${{ inputs.dpcpp_repo_pr_no }}

  build_sycl_cts_x86_64:
    needs: [workflow_vars, build_icd, build_dpcpp_native_x86_64]
    strategy:
      fail-fast: false  # let all matrix jobs complete
      matrix:
        subset: [ 'A', 'B', 'C' ]
    runs-on: 'ubuntu-24.04'
    container:
      image: 'ghcr.io/uxlfoundation/ock_ubuntu_22.04-x86-64:latest'
      volumes:
        - ${{github.workspace}}:${{github.workspace}}

    if: inputs.test_sycl_cts && inputs.ock && contains(inputs.target_list, 'host_x86_64_linux')
    steps:
      - name: Checkout repo
        uses: actions/checkout@v6
      - name: set up gh
        uses: ./.github/actions/setup_gh
        with:
          os: 'ubuntu'
          token: ${{ secrets.GITHUB_TOKEN }}
      - name: build sycl cts artefact
        uses: ./.github/actions/do_build_sycl_cts
        with:
          target: host_x86_64_linux
          subset: ${{ matrix.subset }}
          download_sycl_cts_artefact: ${{ inputs.download_sycl_cts_artefact }}

  build_sycl_cts_x86_64_combine:
    needs: [build_sycl_cts_x86_64]
    runs-on: 'ubuntu-24.04'

    if: inputs.test_sycl_cts && inputs.ock && contains(inputs.target_list, 'host_x86_64_linux')
    steps:
      - name: Checkout repo
        uses: actions/checkout@v6
      # download sycl cts build subset artefacts, combine, upload
      - name: combine SYCL-CTS build artifacts
        uses: ./.github/actions/combine_sycl_cts_artifacts
        with:
          target: host_x86_64_linux
          path: tests_dir

  build_sycl_cts_aarch64:
    needs: [workflow_vars, build_icd, build_dpcpp_native_aarch64]
    strategy:
      fail-fast: false  # let all matrix jobs complete
      matrix:
        subset: [ 'A', 'B', 'C' ]
    runs-on: 'ubuntu-22.04-arm'
    container:
      image: 'ghcr.io/uxlfoundation/ock_ubuntu_22.04-aarch64:latest'
      volumes:
        - ${{github.workspace}}:${{github.workspace}}

    if: inputs.test_sycl_cts && inputs.ock && contains(inputs.target_list, 'host_aarch64_linux')
    steps:
      - name: Checkout repo
        uses: actions/checkout@v6
      - name: set up gh
        uses: ./.github/actions/setup_gh
        with:
          os: 'ubuntu'
          token: ${{ secrets.GITHUB_TOKEN }}
      - name: build sycl cts artefact
        uses: ./.github/actions/do_build_sycl_cts
        with:
          target: host_aarch64_linux
          subset: ${{ matrix.subset }}
          download_sycl_cts_artefact: ${{ inputs.download_sycl_cts_artefact }}

  build_sycl_cts_aarch64_combine:
    needs: [build_sycl_cts_aarch64]
    runs-on: 'ubuntu-24.04'

    if: inputs.test_sycl_cts && inputs.ock && contains(inputs.target_list, 'host_aarch64_linux')
    steps:
      - name: Checkout repo
        uses: actions/checkout@v6
      # download sycl cts build subset artefacts, combine, upload
      - name: combine SYCL-CTS build artifacts
        uses: ./.github/actions/combine_sycl_cts_artifacts
        with:
          target: host_aarch64_linux
          path: tests_dir

  run_sycl_cts_x86_64:
    needs: [workflow_vars, create_ock_artefacts_ubuntu, build_dpcpp_native_x86_64, build_sycl_cts_x86_64_combine]
    runs-on: 'ubuntu-24.04'
    container:
      image: 'ghcr.io/uxlfoundation/ock_ubuntu_22.04-x86-64:latest'
      volumes:
        - ${{github.workspace}}:${{github.workspace}}

    if: inputs.test_sycl_cts && inputs.ock && contains(inputs.target_list, 'host_x86_64_linux')
    steps:
      - name: Checkout repo
        uses: actions/checkout@v6
      - name: run sycl cts
        uses: ./.github/actions/run_sycl_cts
        with:
          target: host_x86_64_linux
          llvm_version: ${{ inputs.llvm_version }}

  run_sycl_cts_aarch64:
    needs: [workflow_vars, create_ock_artefacts_ubuntu, build_dpcpp_native_aarch64, build_sycl_cts_aarch64_combine]
    runs-on: 'ubuntu-24.04-arm'
    container:
      image: 'ghcr.io/uxlfoundation/ock_ubuntu_22.04-aarch64:latest'
      volumes:
        - ${{github.workspace}}:${{github.workspace}}

    if: inputs.test_sycl_cts && inputs.ock && contains(inputs.target_list, 'host_aarch64_linux')
    steps:
      - name: Checkout repo
        uses: actions/checkout@v6
      - name: run sycl cts
        uses: ./.github/actions/run_sycl_cts
        with:
          target: host_aarch64_linux
          llvm_version: ${{ inputs.llvm_version }}

  run_sycl_e2e_x86_64:
    needs: [workflow_vars, build_dpcpp_native_x86_64, build_icd, create_ock_artefacts_ubuntu]
    runs-on: 'ubuntu-24.04'
    container:
      image: 'ghcr.io/uxlfoundation/ock_ubuntu_22.04-x86-64:latest'
      volumes:
        - ${{github.workspace}}:${{github.workspace}}

    if: inputs.test_sycl_e2e && inputs.ock && contains(inputs.target_list, 'host_x86_64_linux')
    steps:
      - name: Checkout repo
        uses: actions/checkout@v6
      - name: build e2e
        uses: ./.github/actions/do_build_run_sycl_e2e
        with:
          target: host_x86_64_linux

  run_sycl_e2e_aarch64:
    needs: [workflow_vars, build_dpcpp_native_aarch64]
    runs-on: 'ubuntu-24.04-arm'
    container:
      image: 'ghcr.io/uxlfoundation/ock_ubuntu_22.04-aarch64:latest'
      volumes:
        - ${{github.workspace}}:${{github.workspace}}

    if: inputs.test_sycl_e2e && inputs.ock && contains(inputs.target_list, 'host_aarch64_linux')
    steps:
      - name: Checkout repo
        uses: actions/checkout@v6
      - name: build e2e
        uses: ./.github/actions/do_build_run_sycl_e2e
        with:
          target: host_aarch64_linux