Migrate dpctl.tensor into dpnp.tensor

.github/workflows/build-sphinx.yml

@@ -26,7 +26,7 @@ jobs:
     name: Build and Deploy Docs
 
     runs-on: ubuntu-22.04
-    timeout-minutes: 60
+    timeout-minutes: 90
 
     permissions:
       # Needed to cancel any previous runs that are not completed for a given workflow

.github/workflows/check-onemath.yaml

@@ -74,7 +74,7 @@ jobs:
         os: [ubuntu-22.04] # windows-2022 - no DFT support for Windows in oneMKL
 
     runs-on: ${{ matrix.os }}
-    timeout-minutes: 60
+    timeout-minutes: 120
 
     defaults:
       run:
@@ -133,6 +133,14 @@ jobs:
         if: env.rerun-tests-on-failure != 'true'
         run: |
           python -m pytest -ra --pyargs dpnp.tests
+        env:
+          SKIP_TENSOR_TESTS: 1
+          SYCL_CACHE_PERSISTENT: 1
+
+      - name: Run tensor tests
+        if: env.rerun-tests-on-failure != 'true'
+        run: |
+          python -m pytest -ra --pyargs dpnp.tests.tensor
         env:
           SYCL_CACHE_PERSISTENT: 1
 
@@ -150,6 +158,24 @@ jobs:
             mamba activate ${{ env.test-env-name }}
 
             python -m pytest -ra --pyargs dpnp.tests
+        env:
+          SKIP_TENSOR_TESTS: 1
+          SYCL_CACHE_PERSISTENT: 1
+
+      - name: ReRun tensor tests on Linux
+        if: env.rerun-tests-on-failure == 'true'
+        id: run_tensor_tests
+        uses: nick-fields/retry@ce71cc2ab81d554ebbe88c79ab5975992d79ba08 # v3.0.2
+        with:
+          timeout_minutes: ${{ env.rerun-tests-timeout }}
+          max_attempts: ${{ env.rerun-tests-max-attempts }}
+          retry_on: any
+          command: |
+            . $CONDA/etc/profile.d/conda.sh
+            . $CONDA/etc/profile.d/mamba.sh
+            mamba activate ${{ env.test-env-name }}
+
+            python -m pytest -ra --pyargs dpnp.tests.tensor
         env:
           SYCL_CACHE_PERSISTENT: 1
 
@@ -239,6 +265,14 @@ jobs:
         if: env.rerun-tests-on-failure != 'true'
         run: |
           python -m pytest -ra --pyargs dpnp.tests
+        env:
+          SKIP_TENSOR_TESTS: 1
+          SYCL_CACHE_PERSISTENT: 1
+
+      - name: Run tensor tests
+        if: env.rerun-tests-on-failure != 'true'
+        run: |
+          python -m pytest -ra --pyargs dpnp.tests.tensor
         env:
           SYCL_CACHE_PERSISTENT: 1
 
@@ -256,5 +290,23 @@ jobs:
             mamba activate ${{ env.test-env-name }}
 
             python -m pytest -ra --pyargs dpnp.tests
+        env:
+          SKIP_TENSOR_TESTS: 1
+          SYCL_CACHE_PERSISTENT: 1
+
+      - name: ReRun tensor tests on Linux
+        if: env.rerun-tests-on-failure == 'true'
+        id: run_tensor_tests_branch
+        uses: nick-fields/retry@ce71cc2ab81d554ebbe88c79ab5975992d79ba08 # v3.0.2
+        with:
+          timeout_minutes: ${{ env.rerun-tests-timeout }}
+          max_attempts: ${{ env.rerun-tests-max-attempts }}
+          retry_on: any
+          command: |
+            . $CONDA/etc/profile.d/conda.sh
+            . $CONDA/etc/profile.d/mamba.sh
+            mamba activate ${{ env.test-env-name }}
+
+            python -m pytest -ra --pyargs dpnp.tests.tensor
         env:
           SYCL_CACHE_PERSISTENT: 1

.github/workflows/conda-package.yml

@@ -37,7 +37,7 @@ jobs:
       actions: write
 
     runs-on: ${{ matrix.os }}
-    timeout-minutes: 60
+    timeout-minutes: 90
 
     defaults:
       run:
@@ -220,6 +220,7 @@ jobs:
       - name: Run tests
         if: env.rerun-tests-on-failure != 'true'
         run: |
+          export SKIP_TENSOR_TESTS=1
           if [[ "${{ matrix.python }}" == "${{ env.python-ver-test-all-dtypes }}" ]]; then
             export DPNP_TEST_ALL_INT_TYPES=1
             python -m pytest -ra --pyargs ${{ env.package-name }}.tests
@@ -239,6 +240,7 @@ jobs:
             . $CONDA/etc/profile.d/conda.sh
             . $CONDA/etc/profile.d/mamba.sh
             mamba activate ${{ env.test-env-name }}
+            export SKIP_TENSOR_TESTS=1
 
             if [[ "${{ matrix.python }}" == "${{ env.python-ver-test-all-dtypes }}" ]]; then
               export DPNP_TEST_ALL_INT_TYPES=1
@@ -247,6 +249,26 @@ jobs:
               python -m pytest -n auto -ra --pyargs ${{ env.package-name }}.tests
             fi
 
+      - name: Run tensor tests
+        if: env.rerun-tests-on-failure != 'true'
+        run: |
+          python -m pytest -n auto -ra --pyargs dpnp.tests.tensor
+
+      - name: Run tensor tests
+        if: env.rerun-tests-on-failure == 'true'
+        id: run_tests_tensor_linux
+        uses: nick-fields/retry@ce71cc2ab81d554ebbe88c79ab5975992d79ba08 # v3.0.2
+        with:
+          timeout_minutes: ${{ env.rerun-tests-timeout }}
+          max_attempts: ${{ env.rerun-tests-max-attempts }}
+          retry_on: any
+          command: |
+            . $CONDA/etc/profile.d/conda.sh
+            . $CONDA/etc/profile.d/mamba.sh
+            mamba activate ${{ env.test-env-name }}
+
+            python -m pytest -n auto -ra --pyargs dpnp.tests.tensor
+
   test_windows:
     name: Test
 
@@ -382,6 +404,7 @@ jobs:
         if: env.rerun-tests-on-failure != 'true'
         shell: pwsh
         run: |
+          $env:SKIP_TENSOR_TESTS=1
           if (${{ matrix.python }} -eq ${{ env.python-ver-test-all-dtypes }}) {
             $env:DPNP_TEST_ALL_INT_TYPES=1
             python -m pytest -ra --pyargs ${{ env.package-name }}.tests
@@ -399,13 +422,32 @@ jobs:
           retry_on: any
           shell: pwsh
           command: |
+            $env:SKIP_TENSOR_TESTS=1
             if ( ${{ matrix.python }} -eq ${{ env.python-ver-test-all-dtypes }} ) {
               $env:DPNP_TEST_ALL_INT_TYPES=1
               python -m pytest -ra --pyargs ${{ env.package-name }}.tests
             } else {
               python -m pytest -n auto -ra --pyargs ${{ env.package-name }}.tests
             }
 
+      - name: Run tensor tests
+        if: env.rerun-tests-on-failure != 'true'
+        shell: pwsh
+        run: |
+          python -m pytest -n auto -ra --pyargs dpnp.tests.tensor
+
+      - name: Run tensor tests
+        if: env.rerun-tests-on-failure == 'true'
+        id: run_tests_tensor_win
+        uses: nick-fields/retry@ce71cc2ab81d554ebbe88c79ab5975992d79ba08 # v3.0.2
+        with:
+          timeout_minutes: ${{ env.rerun-tests-timeout }}
+          max_attempts: ${{ env.rerun-tests-max-attempts }}
+          retry_on: any
+          shell: pwsh
+          command: |
+            python -m pytest -n auto -ra --pyargs dpnp.tests.tensor
+
   upload:
     name: Upload
 

.github/workflows/generate_coverage.yaml

@@ -11,7 +11,7 @@ jobs:
     name: Generate coverage and push to Coveralls.io
 
     runs-on: ubuntu-latest
-    timeout-minutes: 120
+    timeout-minutes: 150
 
     permissions:
       # Needed to cancel any previous runs that are not completed for a given workflow
@@ -122,14 +122,15 @@ jobs:
         uses: nick-fields/retry@ad984534de44a9489a53aefd81eb77f87c70dc60 # v4.0.0
         with:
           shell: bash
-          timeout_minutes: 60
+          timeout_minutes: 120
           max_attempts: 5
           retry_on: error
           command: |
             . $CONDA/etc/profile.d/conda.sh
             conda activate coverage
             [ -f /opt/intel/oneapi/setvars.sh ] && source /opt/intel/oneapi/setvars.sh
             git clean -fxd
+            export SKIP_TENSOR_TESTS=1
             python scripts/gen_coverage.py
 
       - name: Total number of coverage attempts

.gitignore

@@ -28,6 +28,7 @@ dpnp_pytest.*
 example3
 
 *dpnp_backend*
+dpnp/include/dpnp/tensor/*.h
 dpnp/**/*.cpython*.so
 dpnp/**/*.pyd
 *~

CMakeLists.txt

@@ -37,12 +37,23 @@ project(
 )
 
 option(DPNP_GENERATE_COVERAGE "Enable build DPNP with coverage instrumentation" OFF)
+option(
+    DPNP_TENSOR_GENERATE_COVERAGE_FOR_PYBIND11_EXTENSIONS
+    "Build dpnp tensor pybind11 offloading extensions with coverage instrumentation"
+    OFF
+)
 option(DPNP_BACKEND_TESTS "Enable building of DPNP backend test suite" OFF)
 option(
     DPNP_WITH_REDIST
     "Build DPNP assuming DPC++ redistributable is installed into Python prefix"
     OFF
 )
+option(
+    DPNP_TENSOR_OFFLOAD_COMPRESS
+    "Build dpnp tensor using offload section compression feature of DPC++ to reduce \
+size of shared object with offloading sections"
+    OFF
+)
 
 set(CMAKE_CXX_STANDARD 17)
 set(CMAKE_CXX_STANDARD_REQUIRED True)
@@ -106,7 +117,6 @@ find_package(Cython REQUIRED)
 
 find_package(Dpctl REQUIRED)
 message(STATUS "Dpctl_INCLUDE_DIR=" ${Dpctl_INCLUDE_DIR})
-message(STATUS "Dpctl_TENSOR_INCLUDE_DIR=" ${Dpctl_TENSOR_INCLUDE_DIR})
 
 option(DPNP_USE_ONEMATH "Build DPNP with oneMath" OFF)
 set(DPNP_TARGET_CUDA

doc/conf.py

@@ -6,6 +6,7 @@
 # http://www.sphinx-doc.org/en/master/config
 
 from datetime import datetime
+from urllib.parse import urljoin
 
 from sphinx.ext.autodoc import FunctionDocumenter
 from sphinx.ext.napoleon import NumpyDocstring, docstring
@@ -231,6 +232,9 @@ def _can_document_member(member, *args, **kwargs):
 
 autosummary_generate = True
 
+_DPCTL_021_BASE = "https://intelpython.github.io/dpctl/0.21.1/"
+_DPCTL_021_INV = urljoin(_DPCTL_021_BASE, "objects.inv")
+
 intersphinx_mapping = {
     "python": ("https://docs.python.org/3/", None),
     "numpy": ("https://numpy.org/doc/stable/", None),
@@ -302,3 +306,65 @@ def _parse_returns_section_patched(self, section: str) -> list[str]:
 
 
 NumpyDocstring._parse_returns_section = _parse_returns_section_patched
+
+
+# TODO: Remove once dpnp.tensor docs are generated in dpnp
+def _load_dpctl_tensor_inventory(app):
+    """Load dpctl 0.21.1 inventory for dpnp.tensor fallback only."""
+    from sphinx.ext.intersphinx import fetch_inventory
+    from sphinx.util import logging
+
+    logger = logging.getLogger(__name__)
+
+    try:
+        inv = fetch_inventory(app, _DPCTL_021_BASE, _DPCTL_021_INV)
+    except Exception as exc:
+        logger.warning(
+            "Failed to load dpctl 0.21.1 inventory from %s: %s",
+            _DPCTL_021_INV,
+            exc,
+        )
+        inv = {}
+
+    app.builder.env._dpctl_tensor_021_inventory = inv
+
+
+# TODO: Remove once dpnp.tensor docs are generated in dpnp
+def _resolve_dpnp_tensor_refs(app, env, node, contnode):
+    """Resolve dpnp.tensor.* references to dpctl 0.21.1 documentation.
+
+    This temporary workaround is needed because dpnp.tensor documentation
+    is not generated yet, while the corresponding API is still documented
+    in dpctl 0.21.1.
+    """
+    from docutils import nodes as docutils_nodes
+
+    target = node.get("reftarget", "")
+    if not target.startswith("dpnp.tensor"):
+        return None
+
+    dpctl_target = target.replace("dpnp.tensor", "dpctl.tensor", 1)
+    dpctl_tensor_inv = getattr(env, "_dpctl_tensor_021_inventory", {})
+
+    for _objtype, objects in dpctl_tensor_inv.items():
+        if dpctl_target not in objects:
+            continue
+
+        item = objects[dpctl_target]
+        location = item.uri
+        if location.endswith("$"):
+            location = location[:-1] + dpctl_target
+
+        refuri = urljoin(_DPCTL_021_BASE, location)
+        newnode = docutils_nodes.reference(
+            "", "", internal=False, refuri=refuri
+        )
+        newnode += contnode.deepcopy()
+        return newnode
+
+    return None
+
+
+def setup(app):
+    app.connect("builder-inited", _load_dpctl_tensor_inventory, priority=400)
+    app.connect("missing-reference", _resolve_dpnp_tensor_refs, priority=400)

doc/index.rst

@@ -13,6 +13,7 @@ Data Parallel Extension for NumPy*
    overview
    quick_start_guide
    reference/index
+   tensor
 
 .. toctree::
    :maxdepth: 1

doc/reference/exceptions.rst

@@ -20,7 +20,7 @@ Exceptions
 .. exception:: DLPackCreationError
 
    Given when constructing DLPack capsule from either :class:`dpnp.ndarray` or
-   :class:`dpctl.tensor.usm_ndarray` based on a USM allocation
+   :class:`dpnp.tensor.usm_ndarray` based on a USM allocation
    on a partitioned SYCL device.
 
    .. rubric:: Examples