Partially reapply "Move conformance tests over to lit" (#18382) (#18392)

This reverts commit 44ed9b74e290bbd3a00575af974194c7b3d07e35, but
excludes the problematic changes to jobs.

Author: RossBrunton

CMakeLists.txt

@@ -65,8 +65,7 @@ set(UR_SYCL_LIBRARY_DIR "" CACHE PATH
 set(UR_CONFORMANCE_TARGET_TRIPLES "" CACHE STRING
     "List of sycl targets to build CTS device binaries for")
 set(UR_CONFORMANCE_AMD_ARCH "" CACHE STRING "AMD device target ID to build CTS binaries for")
-option(UR_CONFORMANCE_ENABLE_MATCH_FILES "Enable CTS match files" ON)
-option(UR_CONFORMANCE_TEST_LOADER "Also test the loader in the conformance tests" OFF)
+set(UR_CONFORMANCE_SELECTOR "" CACHE STRING "If nonempty, the device selector for conformance tests")
 option(UR_USE_DEBUG_POSTFIX "Enable debug postfix 'd' for libraries" OFF)
 set(UR_ADAPTER_LEVEL_ZERO_SOURCE_DIR "" CACHE PATH
     "Path to external 'level_zero' adapter source dir")

README.md

@@ -127,8 +127,7 @@ List of options provided by CMake:
 | UR_ENABLE_SANITIZER | Enable device sanitizer layer | ON/OFF | ON |
 | UR_CONFORMANCE_TARGET_TRIPLES | SYCL triples to build CTS device binaries for | Comma-separated list | spir64 |
 | UR_CONFORMANCE_AMD_ARCH | AMD device target ID to build CTS binaries for | string | `""` |
-| UR_CONFORMANCE_ENABLE_MATCH_FILES | Enable CTS match files | ON/OFF | ON |
-| UR_CONFORMANCE_TEST_LOADER | Additionally build and run "loader" tests for the CTS | ON/OFF | OFF |
+| UR_CONFORMANCE_SELECTOR | `ONEAPI_DEVICE_SELECTOR` for conformance testing | string | All enabled adapters |
 | UR_BUILD_ADAPTER_L0     | Build the Level-Zero adapter            | ON/OFF     | OFF     |
 | UR_BUILD_ADAPTER_OPENCL | Build the OpenCL adapter                | ON/OFF     | OFF     |
 | UR_BUILD_ADAPTER_CUDA   | Build the CUDA adapter                  | ON/OFF     | OFF     |

scripts/core/CONTRIB.rst

@@ -272,14 +272,21 @@ Conformance tests *must* not make assumptions about the adapter under test.
 Tests fixtures or cases *must* query for support of optional features and skip
 testing if unsupported by the adapter.
 
-All tests in the Unified Runtime project are configured to use CTest to run. 
-All conformance tests have the ``conformance`` label attached to them which 
-allows them to be run independently. To run all the conformance tests, execute 
-the following command from the build directory.
+Conformance tests are ran as part of the main ``test`` target, but can also be
+executed using the `check-unified-runtime-conformance` target as follows:
 
 .. code-block:: console
      
-    ctest -L "conformance"
+    cmake --build build --target check-unified-runtime-conformance
+
+A specific device/adapter can be specified by using ``ONEAPI_DEVICE_SELECTOR``
+in the same way as in the `DPC++ compiler
+<https://github.com/intel/llvm/blob/sycl/sycl/doc/EnvironmentVariables.md#oneapi_device_selector>`_.
+
+A number of tests are skipped on certain adapters due to being known failures.
+To force all tests to run, including known failures,
+``UR_CTS_ALSO_RUN_KNOWN_FAILURES`` may be set to ``1`` as an environment
+variable.
 
 Experimental Features
 =====================

scripts/core/INTRO.rst

@@ -417,12 +417,12 @@ CTS Environment Variables
 
 The following environment variables are used by the CTS runner and can be used to specify the platform that the test
 framework should run on. This can be used during development and testing to run CTS tests in case multiple platforms
-are available. If both filters are specified, then they both must match a platform for it to be selected. If there are
-no valid platforms, then the tests will fail. Command line arguments take priority over these variables.
+are available. Conformance tests will be skipped if there are no valid platforms.
 
-.. envvar:: UR_CTS_BACKEND
+.. envvar:: ONEAPI_DEVICE_SELECTOR
 
-    A (case insensitive) backend to force the test to use. For example, `opencl`, `level_zero`, `hip` and so on.
+   If ``ONEAPI_DEVICE_SELECTOR`` is set in the environment, only devices
+   matched by the selector will be tested.
 
 .. envvar:: UR_CTS_ALSO_RUN_KNOWN_FAILURES
 

scripts/ctest_parser.py

@@ -31,8 +31,9 @@ configure_file(lit.site.cfg.py.in lit.site.cfg.py)
 add_custom_target(deps_check-unified-runtime)
 if(UR_STANDALONE_BUILD)
   add_custom_target(check-unified-runtime
-    COMMAND "${URLIT_LIT_BINARY}" "${CMAKE_CURRENT_BINARY_DIR}" -v
+    COMMAND "${URLIT_LIT_BINARY}" "${CMAKE_CURRENT_BINARY_DIR}" -sv
     DEPENDS deps_check-unified-runtime
+    USES_TERMINAL
   )
 else()
   # Use the LLVM method to add the test suite - this also registers it under `check-all`
@@ -60,7 +61,8 @@ function(add_ur_lit_testsuite suite)
 
   if(UR_STANDALONE_BUILD)
     add_custom_target(${TARGET}
-      COMMAND "${URLIT_LIT_BINARY}" "${CMAKE_CURRENT_BINARY_DIR}" -v
+      COMMAND "${URLIT_LIT_BINARY}" "${CMAKE_CURRENT_BINARY_DIR}" -sv
+      USES_TERMINAL
     )
   else()
     add_lit_testsuite(${TARGET} "Running Unified Runtime ${suite} tests"

test/CMakeLists.txt

@@ -3,94 +3,61 @@
 # See LICENSE.TXT
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
-set(UR_CONFORMANCE_TEST_DIR ${CMAKE_CURRENT_SOURCE_DIR})
+add_ur_lit_testsuite(conformance DEPENDS urinfo)
+
+set(UR_CONFORMANCE_TEST_DIR "${CMAKE_CURRENT_SOURCE_DIR}" CACHE INTERNAL
+    "Internal cache variable for conformance test root")
 
 set(UR_CONFORMANCE_DEVICE_BINARIES_DIR
     "${CMAKE_CURRENT_BINARY_DIR}/device_binaries" CACHE INTERNAL
     "Internal cache variable for device binaries directory")
 
-function(add_test_adapter name adapter backend)
-    set(TEST_TARGET_NAME test-${name})
-    set(TEST_NAME ${name}-${adapter})
-
-    set(TEST_COMMAND
-        "${PROJECT_BINARY_DIR}/bin/${TEST_TARGET_NAME} --gtest_filter=*${backend}*"
-    )
-
-    function(do_add_test tname env)
-        separate_arguments(TEST_COMMAND)
-        add_test(NAME ${tname}
-            COMMAND ${TEST_COMMAND}
-            WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
-        )
-
-        set_tests_properties(${tname} PROPERTIES
-            ENVIRONMENT "${env}"
-            LABELS "conformance;${adapter}")
-    endfunction()
-
-    do_add_test(${TEST_NAME} UR_ADAPTERS_FORCE_LOAD="$<TARGET_FILE:ur_${adapter}>")
-    if(UR_CONFORMANCE_TEST_LOADER)
-        do_add_test(${TEST_NAME}-loader "")
-    endif()
-endfunction()
-
 function(add_conformance_test name)
-    set(TEST_TARGET_NAME test-${name})
-    add_ur_executable(${TEST_TARGET_NAME}
+    add_gtest_test(${name}
         ${ARGN}
         ${UR_CONFORMANCE_TEST_DIR}/source/environment.cpp
-        ${UR_CONFORMANCE_TEST_DIR}/source/main.cpp)
-    target_link_libraries(${TEST_TARGET_NAME}
-        PRIVATE
-        ${PROJECT_NAME}::loader
-        ${PROJECT_NAME}::headers
-        ${PROJECT_NAME}::testing
-        ${PROJECT_NAME}::common
-        GTest::gtest_main)
-
-    if(UR_USE_CFI)
-        target_compile_definitions(${TEST_TARGET_NAME} PRIVATE UR_USE_CFI)
-    endif()
-
-    if(UR_BUILD_ADAPTER_CUDA OR UR_BUILD_ADAPTER_ALL)
-        add_test_adapter(${name} adapter_cuda CUDA)
-    endif()
-    if(UR_BUILD_ADAPTER_HIP OR UR_BUILD_ADAPTER_ALL)
-        add_test_adapter(${name} adapter_hip HIP)
-    endif()
-    if(UR_BUILD_ADAPTER_L0 OR UR_BUILD_ADAPTER_ALL)
-        add_test_adapter(${name} adapter_level_zero Level_Zero)
-    endif()
-    if(UR_BUILD_ADAPTER_L0_V2)
-        add_test_adapter(${name} adapter_level_zero_v2 Level_Zero)
-    endif()
-    if(UR_BUILD_ADAPTER_OPENCL OR UR_BUILD_ADAPTER_ALL)
-        add_test_adapter(${name} adapter_opencl OpenCL)
-    endif()
-    if(UR_BUILD_ADAPTER_NATIVE_CPU OR UR_BUILD_ADAPTER_ALL)
-        add_test_adapter(${name} adapter_native_cpu NATIVE_CPU)
-    endif()
+        ${UR_CONFORMANCE_TEST_DIR}/source/main.cpp
+    )
 endfunction()
 
-function(add_conformance_test_with_kernels_environment name)
+function(add_conformance_kernels_test name)
     add_conformance_test(${name} ${ARGN})
-    target_compile_definitions("test-${name}" PRIVATE KERNELS_ENVIRONMENT PRIVATE
+    target_compile_definitions(${name}-test PRIVATE KERNELS_ENVIRONMENT PRIVATE
       KERNELS_DEFAULT_DIR="${UR_CONFORMANCE_DEVICE_BINARIES_DIR}")
-    target_include_directories("test-${name}" PRIVATE ${UR_CONFORMANCE_DEVICE_BINARIES_DIR})
-    add_dependencies("test-${name}" generate_device_binaries)
+    target_include_directories(${name}-test PRIVATE ${UR_CONFORMANCE_DEVICE_BINARIES_DIR})
+    add_dependencies(${name}-test generate_device_binaries kernel_names_header)
 endfunction()
 
-function(add_conformance_test_with_devices_environment name)
+function(add_conformance_devices_test name)
     add_conformance_test(${name} ${ARGN})
-    target_compile_definitions("test-${name}" PRIVATE DEVICES_ENVIRONMENT)
+    target_compile_definitions(${name}-test PRIVATE DEVICES_ENVIRONMENT)
 endfunction()
 
-function(add_conformance_test_with_platform_environment name)
+function(add_conformance_platform_test name)
     add_conformance_test(${name} ${ARGN})
-    target_compile_definitions("test-${name}" PRIVATE PLATFORM_ENVIRONMENT)
+    target_compile_definitions(${name}-test PRIVATE PLATFORM_ENVIRONMENT)
 endfunction()
 
+foreach(adapter ${UR_ADAPTERS_LIST})
+    if(NOT "${adapter}" STREQUAL "mock")
+        if(UR_STANDALONE_BUILD)
+            add_custom_target(check-unified-runtime-conformance-${adapter}
+                COMMAND "${URLIT_LIT_BINARY}" "${CMAKE_CURRENT_BINARY_DIR}"
+                    -v -Dselector=${adapter}:*
+                DEPENDS deps_check-unified-runtime-conformance
+            )
+        else()
+            add_lit_testsuite(check-unified-runtime-conformance-${adapter}
+                "Running Unified Runtime ${adapter} conformance tests"
+                ${CMAKE_CURRENT_BINARY_DIR}
+                DEPENDS FileCheck deps_check-unified-runtime-conformance
+                PARAMS selector=${adapter}:*
+                EXCLUDE_FROM_CHECK_ALL
+            )
+        endif()
+    endif()
+endforeach()
+
 add_subdirectory(testing)
 
 add_subdirectory(adapter)

test/conformance/CMakeLists.txt

@@ -3,7 +3,7 @@
 # See LICENSE.TXT
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
-add_conformance_test_with_devices_environment(context 
+add_conformance_devices_test(context
     urContextCreate.cpp
     urContextCreateWithNativeHandle.cpp
     urContextGetInfo.cpp