[CI] Move SPIRV-backend testing to post-commit (#19669)

Development happens mostly upstream, so there is no need to have that in
pre-commit, post-commit should be enough.

Before this PR, when running E2E tests in "in-tree" mode (as opposite to
a standalone e2e tests cmake invocation), some utilities (e.g.
`llvm-ar`) were only searched for in `PATH` that didn't contain freshly
built toolchain. Our pre-commit used a container image with last
successful "Nightly" toolchain installed, so those were picked up from
there. Post-commit uses GNU Compiler instead of Clang, so these
utilities aren't available in `PATH`.

This PR modifies`lit.cfg.py` such that we search for those utilities in
the newly built toolchain first (still not modifying `PATH` which seems
to be the approach taken in LIT). That also ensures that `opencl-aot`
feature is available when running E2E tests in `build-only` mode in
Post-commit's build job.

Author: aelovikov-intel

.github/workflows/sycl-linux-precommit.yml

@@ -60,7 +60,6 @@ jobs:
 
       toolchain_artifact: sycl_linux_default
       e2e_binaries_artifact: e2e_bin
-      e2e_binaries_spirv_backend_artifact: e2e_bin_spirv_backend
       e2e_binaries_preview_artifact: e2e_bin_preview
 
   # If a PR changes CUDA adapter, run the build on Ubuntu 22.04 as well.
@@ -129,12 +128,6 @@ jobs:
             runner: '["Linux", "bmg"]'
             image_options: -u 1001 --device=/dev/dri -v /dev/dri/by-path:/dev/dri/by-path --privileged --cap-add SYS_ADMIN
             target_devices: level_zero_v1:gpu;level_zero_v2:gpu
-          - name: SPIR-V Backend / Intel Battlemage Graphics
-            runner: '["Linux", "bmg"]'
-            image_options: -u 1001 --device=/dev/dri -v /dev/dri/by-path:/dev/dri/by-path --privileged --cap-add SYS_ADMIN
-            target_devices: level_zero:gpu;opencl:gpu;opencl:cpu
-            extra_lit_opts: --param spirv-backend=True
-            e2e_binaries_artifact: e2e_bin_spirv_backend
           - name: Preview Mode
             runner: '["Linux", "gen12"]'
             image_options: -u 1001 --device=/dev/dri -v /dev/dri/by-path:/dev/dri/by-path --privileged --cap-add SYS_ADMIN

.github/workflows/sycl-post-commit.yml

@@ -43,6 +43,7 @@ jobs:
       build_configure_extra_args: --no-assertions --hip --cuda --native_cpu -DSYCL_ENABLE_STACK_PRINTING=ON -DSYCL_LIB_WITH_DEBUG_SYMBOL=ON
 
       toolchain_artifact: sycl_linux_default
+      e2e_binaries_spirv_backend_artifact: e2e_bin_spirv_backend
 
   e2e-lin:
     needs: [detect_changes, build-lin]
@@ -73,6 +74,13 @@ jobs:
             env: '{"LIT_FILTER":"PerformanceTests/"}'
             extra_lit_opts: -a -j 1 --param enable-perf-tests=True
             target_devices: all
+          - name: SPIR-V Backend / Intel Battlemage Graphics
+            runner: '["Linux", "bmg"]'
+            image_options: -u 1001 --device=/dev/dri -v /dev/dri/by-path:/dev/dri/by-path --privileged --cap-add SYS_ADMIN
+            target_devices: level_zero:gpu;opencl:gpu;opencl:cpu
+            extra_lit_opts: --param spirv-backend=True
+            e2e_binaries_artifact: e2e_bin_spirv_backend
+            e2e_testing_mode: 'run-only'
     uses: ./.github/workflows/sycl-linux-run-tests.yml
     with:
       name: ${{ matrix.name }}
@@ -89,6 +97,9 @@ jobs:
       toolchain_artifact_filename: ${{ needs.build-lin.outputs.toolchain_artifact_filename }}
       toolchain_decompress_command: ${{ needs.build-lin.outputs.toolchain_decompress_command }}
 
+      e2e_binaries_artifact: ${{ matrix.e2e_binaries_artifact }}
+      e2e_testing_mode: ${{ matrix.e2e_testing_mode || 'full' }}
+
       # Do not install drivers on AMD and CUDA runners.
       install_igc_driver: >-
         ${{ github.event_name == 'pull_request' }} &&

sycl/test-e2e/Graph/Explicit/interop-level-zero-get-native-mem.cpp

@@ -2,7 +2,7 @@
 // UNSUPPORTED: ze_debug
 // UNSUPPORTED-INTENDED: Leaks detection is done at UR level and doesn't account
 // for native L0 API calls.
-// UNSUPPORTED: linux && gpu-intel-dg2 && run-mode && !igc-dev
+// UNSUPPORTED: linux && (gpu-intel-dg2 || arch-intel_gpu_bmg_g21) && run-mode && !igc-dev
 // UNSUPPORTED-TRACKER: https://github.com/intel/llvm/issues/18273
 // RUN: %{build} %level_zero_options -o %t.out
 // RUN: %{run} %t.out

sycl/test-e2e/Graph/RecordReplay/interop-level-zero-get-native-mem.cpp

@@ -2,7 +2,7 @@
 // UNSUPPORTED: ze_debug
 // UNSUPPORTED-INTENDED: Leaks detection is done at UR level and doesn't account
 // for native L0 API calls.
-// UNSUPPORTED: linux && gpu-intel-dg2 && run-mode && !igc-dev
+// UNSUPPORTED: linux && (gpu-intel-dg2 || arch-intel_gpu_bmg_g21) && run-mode && !igc-dev
 // UNSUPPORTED-TRACKER: https://github.com/intel/llvm/issues/18273
 // RUN: %{build} %level_zero_options -o %t.out
 // RUN: %{run} %t.out

sycl/test-e2e/lit.cfg.py

@@ -650,12 +650,6 @@ def open_check_file(file_name):
 if not sycl_ls:
     lit_config.fatal("can't find `sycl-ls`")
 
-syclbin_dump = FindTool("syclbin-dump").resolve(
-    llvm_config, os.pathsep.join([config.dpcpp_bin_dir, config.llvm_tools_dir])
-)
-if not syclbin_dump:
-    lit_config.fatal("can't find `syclbin-dump`")
-
 if (
     len(config.sycl_build_targets) == 1
     and next(iter(config.sycl_build_targets)) == "target-all"
@@ -824,6 +818,8 @@ def remove_level_zero_suffix(devices):
 feature_tools = [
     ToolSubst("llvm-spirv", unresolved="ignore"),
     ToolSubst("llvm-link", unresolved="ignore"),
+    ToolSubst("opencl-aot", unresolved="ignore"),
+    ToolSubst("ocloc", unresolved="ignore"),
 ]
 
 tools = [
@@ -833,8 +829,14 @@ def remove_level_zero_suffix(devices):
     ToolSubst(
         r"\| \bnot\b", command=FindTool("not"), verbatim=True, unresolved="ignore"
     ),
-    ToolSubst("sycl-ls", command=sycl_ls, unresolved="ignore"),
-    ToolSubst("syclbin-dump", command=syclbin_dump, unresolved="ignore"),
+    ToolSubst("sycl-ls", command=sycl_ls, unresolved="fatal"),
+    ToolSubst("syclbin-dump", unresolved="fatal"),
+    ToolSubst("llvm-ar", unresolved="fatal"),
+    ToolSubst("clang-offload-bundler", unresolved="fatal"),
+    ToolSubst("clang-offload-wrapper", unresolved="fatal"),
+    ToolSubst("sycl-post-link", unresolved="fatal"),
+    ToolSubst("file-table-tform", unresolved="fatal"),
+    ToolSubst("llvm-foreach", unresolved="fatal"),
 ] + feature_tools
 
 # Try and find each of these tools in the DPC++ bin directory, in the llvm tools directory
@@ -855,19 +857,6 @@ def remove_level_zero_suffix(devices):
 if shutil.which("cmc") is not None:
     config.available_features.add("cm-compiler")
 
-# Device AOT compilation tools aren't part of the SYCL project,
-# so they need to be pre-installed on the machine
-aot_tools = ["ocloc", "opencl-aot"]
-
-for aot_tool in aot_tools:
-    if shutil.which(aot_tool) is not None:
-        lit_config.note("Found pre-installed AOT device compiler " + aot_tool)
-        config.available_features.add(aot_tool)
-    else:
-        lit_config.warning(
-            "Couldn't find pre-installed AOT device compiler " + aot_tool
-        )
-
 # Clear build targets when not in build-only, to populate according to devices
 if config.test_mode != "build-only":
     config.sycl_build_targets = set()