Move dpctl_ext.tensor to dpnp.tensor (#2829)

This PR proposes a refactoring that migrates `dpctl_ext.tensor` module
into `dpnp` package as `dpnp.tensor`

Changes:

1. Moved `dpctl_ext/tensor/` directory to `dpnp/tensor/`
2. Updated all imports from `dpctl_ext.tensor` to `dpnp.tensor` across
the codebase
3. Consolidated build: removed dpctl_ext/CMakeLists.txt, added
build_dpnp_tensor_ext() to dpnp/CMakeLists.txt
4. Added `DPNP_BUILD_COMPONENTS` CMake option
(`ALL/TENSOR_ONLY/SKIP_TENSOR`) for staged builds
5. Split coverage workflow into two steps to avoid memory issues
6. Updated include paths in all backend extension CMake files
7. Removed `dpctl_ext/` directory and cleaned up `.gitignore`

Author: vlad-perevezentsev

.gitignore

@@ -28,12 +28,8 @@ dpnp_pytest.*
 example3
 
 *dpnp_backend*
+dpnp/include/dpnp/tensor/*.h
 dpnp/**/*.cpython*.so
 dpnp/**/*.pyd
 *~
 core
-
-# TODO: revert to `dpctl/`
-# when dpnp fully migrates dpctl/tensor
-dpctl_ext/**/*.cpython*.so
-dpctl_ext/include/

CMakeLists.txt

@@ -37,12 +37,23 @@ project(
 )
 
 option(DPNP_GENERATE_COVERAGE "Enable build DPNP with coverage instrumentation" OFF)
+option(
+    DPNP_TENSOR_GENERATE_COVERAGE_FOR_PYBIND11_EXTENSIONS
+    "Build dpnp tensor pybind11 offloading extensions with coverage instrumentation"
+    OFF
+)
 option(DPNP_BACKEND_TESTS "Enable building of DPNP backend test suite" OFF)
 option(
     DPNP_WITH_REDIST
     "Build DPNP assuming DPC++ redistributable is installed into Python prefix"
     OFF
 )
+option(
+    DPNP_TENSOR_OFFLOAD_COMPRESS
+    "Build dpnp tensor using offload section compression feature of DPC++ to reduce \
+size of shared object with offloading sections"
+    OFF
+)
 
 set(CMAKE_CXX_STANDARD 17)
 set(CMAKE_CXX_STANDARD_REQUIRED True)
@@ -344,5 +355,4 @@ if(DEFINED SKBUILD)
     set(_ignore_me ${SKBUILD})
 endif()
 
-add_subdirectory(dpctl_ext)
 add_subdirectory(dpnp)

dpctl_ext/CMakeLists.txt

@@ -86,11 +86,96 @@ function(build_dpnp_cython_ext _trgt _src _dest)
     install(TARGETS ${_trgt} LIBRARY DESTINATION ${_dest})
 endfunction()
 
+function(build_dpnp_tensor_ext _trgt _src _dest)
+    set(options SYCL)
+    cmake_parse_arguments(BUILD_DPNP_TENSOR "${options}" "RELATIVE_PATH" "" ${ARGN})
+    add_cython_target(${_trgt} ${_src} CXX OUTPUT_VAR _generated_src)
+    set(_cythonize_trgt "${_trgt}_cythonize_pyx")
+    python_add_library(${_trgt} MODULE WITH_SOABI ${_generated_src})
+    if(BUILD_DPNP_TENSOR_SYCL)
+        add_sycl_to_target(TARGET ${_trgt} SOURCES ${_generated_src})
+        target_compile_options(${_trgt} PRIVATE -fno-sycl-id-queries-fit-in-int)
+        target_link_options(${_trgt} PRIVATE -fsycl-device-code-split=per_kernel)
+        if(DPNP_TENSOR_OFFLOAD_COMPRESS)
+            target_link_options(${_trgt} PRIVATE --offload-compress)
+        endif()
+        if(_dpnp_sycl_targets)
+            # make fat binary
+            target_compile_options(
+                ${_trgt}
+                PRIVATE ${_dpnp_sycl_target_compile_options}
+            )
+            target_link_options(${_trgt} PRIVATE ${_dpnp_sycl_target_link_options})
+        endif()
+    endif()
+    target_link_libraries(${_trgt} PRIVATE Python::NumPy)
+    if(DPNP_GENERATE_COVERAGE)
+        target_compile_definitions(${_trgt} PRIVATE CYTHON_TRACE=1 CYTHON_TRACE_NOGIL=1)
+        if(BUILD_DPNP_TENSOR_SYCL)
+            target_compile_options(${_trgt} PRIVATE -fno-sycl-use-footer)
+        endif()
+    endif()
+    # Dpctl
+    target_include_directories(${_trgt} PRIVATE ${Dpctl_INCLUDE_DIR})
+    target_link_directories(${_trgt} PRIVATE ${Dpctl_INCLUDE_DIR}/..)
+    target_link_libraries(${_trgt} PRIVATE DPCTLSyclInterface)
+    set(_linker_options "LINKER:${DPNP_LDFLAGS}")
+    target_link_options(${_trgt} PRIVATE ${_linker_options})
+    get_filename_component(_name_wle ${_generated_src} NAME_WLE)
+    get_filename_component(_generated_src_dir ${_generated_src} DIRECTORY)
+    set(_generated_public_h "${_generated_src_dir}/${_name_wle}.h")
+    set(_generated_api_h "${_generated_src_dir}/${_name_wle}_api.h")
+
+    # TODO: create separate folder inside build folder that contains only
+    #   headers related to this target and appropriate folder structure to
+    #   eliminate shadow dependencies
+    # Go up two levels to build root for "dpnp/tensor/_usmarray.h" resolution
+    get_filename_component(_parent_dir ${_generated_src_dir} DIRECTORY)
+    get_filename_component(_build_root ${_parent_dir} DIRECTORY)
+    # TODO: do not set directory if we did not generate header
+    target_include_directories(${_trgt} INTERFACE ${_build_root})
+    set(_rpath_value "$ORIGIN")
+    if(BUILD_DPNP_TENSOR_RELATIVE_PATH)
+        set(_rpath_value "${_rpath_value}/${BUILD_DPNP_TENSOR_RELATIVE_PATH}")
+    endif()
+    if(DPNP_WITH_REDIST)
+        set(_rpath_value "${_rpath_value}:${_rpath_value}/../../..")
+    endif()
+    set_target_properties(${_trgt} PROPERTIES INSTALL_RPATH ${_rpath_value})
+
+    install(TARGETS ${_trgt} LIBRARY DESTINATION ${_dest})
+    install(
+        FILES ${_generated_api_h}
+        DESTINATION ${CMAKE_INSTALL_PREFIX}/dpnp/include/${_dest}
+        OPTIONAL
+    )
+    install(
+        FILES ${_generated_public_h}
+        DESTINATION ${CMAKE_INSTALL_PREFIX}/dpnp/include/${_dest}
+        OPTIONAL
+    )
+    if(DPNP_GENERATE_COVERAGE)
+        get_filename_component(_original_src_dir ${_src} DIRECTORY)
+        file(RELATIVE_PATH _rel_dir ${CMAKE_SOURCE_DIR} ${_original_src_dir})
+        install(FILES ${_generated_src} DESTINATION ${CMAKE_INSTALL_PREFIX}/${_rel_dir})
+    endif()
+
+    # Create target with headers only, because python is managing all the
+    # library imports at runtime
+    set(_trgt_headers ${_trgt}_headers)
+    add_library(${_trgt_headers} INTERFACE)
+    add_dependencies(${_trgt_headers} ${_trgt})
+    get_target_property(_trgt_headers_dir ${_trgt} INTERFACE_INCLUDE_DIRECTORIES)
+    target_include_directories(${_trgt_headers} INTERFACE ${_trgt_headers_dir})
+endfunction()
+
 function(build_dpnp_cython_ext_with_backend _trgt _src _dest)
     build_dpnp_cython_ext(${_trgt} ${_src} ${_dest})
     target_link_libraries(${_trgt} PRIVATE dpnp_backend_library)
 endfunction()
 
+add_subdirectory(tensor)
+
 add_subdirectory(backend)
 add_subdirectory(backend/extensions/blas)
 add_subdirectory(backend/extensions/fft)

dpctl_ext/__init__.py

@@ -60,9 +60,7 @@
                 [os.getenv("PATH", ""), dll_path]
             )
 
-# TODO: revert to `from dpctl.tensor...`
-# when dpnp fully migrates dpctl/tensor
-from dpctl_ext.tensor import __array_api_version__, DLDeviceType
+from .tensor import __array_api_version__, DLDeviceType
 
 from .dpnp_array import dpnp_array as ndarray
 from .dpnp_array_api_info import __array_namespace_info__

dpnp/CMakeLists.txt

@@ -71,7 +71,7 @@ target_include_directories(
     PRIVATE
         ${CMAKE_CURRENT_SOURCE_DIR}/../common
         ${CMAKE_SOURCE_DIR}/dpnp/backend/include
-        ${CMAKE_SOURCE_DIR}/dpctl_ext/tensor/libtensor/include
+        ${CMAKE_SOURCE_DIR}/dpnp/tensor/libtensor/include
 )
 
 # treat below headers as system to suppress the warnings there during the build