diff --git a/debian/libibverbs1.symbols b/debian/libibverbs1.symbols index 5378bd801..6f90cffde 100644 --- a/debian/libibverbs1.symbols +++ b/debian/libibverbs1.symbols @@ -15,6 +15,7 @@ libibverbs.so.1 libibverbs1 #MINVER# IBVERBS_1.15@IBVERBS_1.15 59 IBVERBS_1.16@IBVERBS_1.16 62 IBVERBS_1.17@IBVERBS_1.17 64 + IBVERBS_1.18@IBVERBS_1.18 65 (symver)IBVERBS_PRIVATE_64 64 _ibv_query_gid_ex@IBVERBS_1.11 32 _ibv_query_gid_table@IBVERBS_1.11 32 @@ -64,6 +65,7 @@ libibverbs.so.1 libibverbs1 #MINVER# ibv_dofork_range@IBVERBS_1.1 1.1.6 ibv_dontfork_range@IBVERBS_1.1 1.1.6 ibv_event_type_str@IBVERBS_1.1 1.1.6 + ibv_export_buf_dmabuf_fd@IBVERBS_1.18 65 ibv_fork_init@IBVERBS_1.1 1.1.6 ibv_free_buf@IBVERBS_1.17 64 ibv_free_device_list@IBVERBS_1.0 1.1.6 diff --git a/libibverbs/CMakeLists.txt b/libibverbs/CMakeLists.txt index 7fe44d861..eaf25a1ae 100644 --- a/libibverbs/CMakeLists.txt +++ b/libibverbs/CMakeLists.txt @@ -22,7 +22,7 @@ configure_file("libibverbs.map.in" rdma_library(ibverbs "${CMAKE_CURRENT_BINARY_DIR}/libibverbs.map" # See Documentation/versioning.md - 1 1.17.${PACKAGE_VERSION} + 1 1.18.${PACKAGE_VERSION} all_providers.c cmd.c cmd_ah.c diff --git a/libibverbs/libibverbs.map.in b/libibverbs/libibverbs.map.in index dd7fb8a88..7329bce5a 100644 --- a/libibverbs/libibverbs.map.in +++ b/libibverbs/libibverbs.map.in @@ -184,6 +184,11 @@ IBVERBS_1.17 { ibv_reg_buf_mr; } IBVERBS_1.16; +IBVERBS_1.18 { + global: + ibv_export_buf_dmabuf_fd; +} IBVERBS_1.17; + /* If any symbols in this stanza change ABI then the entire staza gets a new symbol version. See the top level CMakeLists.txt for this setting. */ diff --git a/libibverbs/man/CMakeLists.txt b/libibverbs/man/CMakeLists.txt index f0e2e7ee0..f8eca8668 100644 --- a/libibverbs/man/CMakeLists.txt +++ b/libibverbs/man/CMakeLists.txt @@ -89,6 +89,7 @@ rdma_man_pages( rdma_alias_man_pages( ibv_alloc_buf.3 ibv_free_buf.3 ibv_alloc_buf.3 ibv_reg_buf_mr.3 + ibv_alloc_buf.3 ibv_export_buf_dmabuf_fd.3 ibv_alloc_dm.3 ibv_free_dm.3 ibv_alloc_dm.3 ibv_reg_dm_mr.3 ibv_alloc_dm.3 ibv_memcpy_to_dm.3 diff --git a/libibverbs/man/ibv_alloc_buf.3.md b/libibverbs/man/ibv_alloc_buf.3.md index 8809c6b72..5409b3c00 100644 --- a/libibverbs/man/ibv_alloc_buf.3.md +++ b/libibverbs/man/ibv_alloc_buf.3.md @@ -10,7 +10,7 @@ title: ibv_alloc_buf # NAME -ibv_alloc_buf, ibv_free_buf, ibv_reg_buf_mr - allocate provider-aware buffers and register them as memory regions +ibv_alloc_buf, ibv_free_buf, ibv_reg_buf_mr, ibv_export_buf_dmabuf_fd - allocate provider-aware buffers and register them as memory regions # SYNOPSIS @@ -23,6 +23,8 @@ void ibv_free_buf(struct ibv_buf *buf); struct ibv_mr *ibv_reg_buf_mr(struct ibv_pd *pd, struct ibv_buf *buf, void *addr, size_t length, int access); + +int ibv_export_buf_dmabuf_fd(struct ibv_buf *buf); ``` # DESCRIPTION @@ -30,8 +32,9 @@ struct ibv_mr *ibv_reg_buf_mr(struct ibv_pd *pd, struct ibv_buf *buf, void *addr **ibv_alloc_buf()** allocates a buffer using the allocation method selected by the provider for the protection domain *pd*. On success it returns the mapped address and stores an opaque buffer handle in *buf*. The handle is used by -**ibv_free_buf()**, **ibv_reg_buf_mr()**, and **ibv_reg_mr_ex()** with -**IBV_REG_MR_MASK_BUF**, and must not be interpreted by applications. +**ibv_free_buf()**, **ibv_reg_buf_mr()**, **ibv_reg_mr_ex()** with +**IBV_REG_MR_MASK_BUF**, and **ibv_export_buf_dmabuf_fd()**; it must not be +interpreted by applications. **ibv_free_buf()** releases a buffer handle returned by **ibv_alloc_buf()**. The protection domain used for allocation must remain valid until the buffer is @@ -53,6 +56,14 @@ For provider allocations backed by a DMA-buf, it registers the corresponding DMA-buf range using the metadata stored in the opaque *buf* handle. +**ibv_export_buf_dmabuf_fd()** exports a new file descriptor for a buffer that +is backed by a DMA-buf. This allows applications to pass provider-allocated +memory to other APIs that import DMA-buf file descriptors. The returned file +descriptor is owned by the caller and should be closed with **close(2)** when +no longer needed. Calling +**ibv_export_buf_dmabuf_fd()** does not transfer ownership of the buffer handle +or the internal descriptor used by libibverbs. + # ARGUMENTS *pd* @@ -62,7 +73,7 @@ the opaque *buf* handle. : Size of the buffer to allocate, in bytes (**ibv_alloc_buf()**). *buf* -: For **ibv_alloc_buf()**, an output parameter set on success to an opaque buffer handle. For **ibv_free_buf()** and **ibv_reg_buf_mr()**, the buffer handle returned by **ibv_alloc_buf()** to be released or registered, respectively. +: For **ibv_alloc_buf()**, an output parameter set on success to an opaque buffer handle. For **ibv_free_buf()** and **ibv_reg_buf_mr()**, the buffer handle returned by **ibv_alloc_buf()** to be released or registered, respectively. For **ibv_export_buf_dmabuf_fd()**, the buffer handle whose DMA-buf fd should be exported. *addr* : The start address to register (**ibv_reg_buf_mr()**): the buffer base returned by **ibv_alloc_buf()** or an address within that buffer. @@ -83,6 +94,10 @@ if the request fails. **ibv_free_buf()** does not return a value. +**ibv_export_buf_dmabuf_fd()** returns a file descriptor >= 0 on success, or -1 +if the request fails with *errno* set. If *buf* is not backed by a DMA-buf, +*errno* is set to **ENODATA**. + # NOTES Applications running in a CoCo guest that need unprotected/shared memory should diff --git a/libibverbs/verbs.c b/libibverbs/verbs.c index 0565be208..bb5233b53 100644 --- a/libibverbs/verbs.c +++ b/libibverbs/verbs.c @@ -40,6 +40,7 @@ #include #include #include +#include #include #include #include @@ -440,6 +441,16 @@ void *ibv_alloc_buf(struct ibv_pd *pd, size_t size, struct ibv_buf **buf) return get_ops(pd->context)->alloc_buf(pd, size, buf); } +int ibv_export_buf_dmabuf_fd(struct ibv_buf *buf) +{ + if (buf->dmabuf_fd < 0) { + errno = ENODATA; + return -1; + } + + return fcntl(buf->dmabuf_fd, F_DUPFD_CLOEXEC, 0); +} + struct ibv_mr *ibv_reg_buf_mr(struct ibv_pd *pd, struct ibv_buf *buf, void *addr, size_t length, int access) { diff --git a/libibverbs/verbs.h b/libibverbs/verbs.h index 60103d98a..9ed55d423 100644 --- a/libibverbs/verbs.h +++ b/libibverbs/verbs.h @@ -3218,6 +3218,16 @@ ibv_alloc_parent_domain(struct ibv_context *context, */ void *ibv_alloc_buf(struct ibv_pd *pd, size_t size, struct ibv_buf **buf); +/** + * ibv_export_buf_dmabuf_fd - Export a DMA-buf fd for an ibv_buf + * @buf: Handle from ibv_alloc_buf() + * + * Returns a new fd for the DMA-buf backing the buffer on success, or -1 + * on failure with errno set. The returned fd is owned by the caller and + * should be closed with close(2) when no longer needed. + */ +int ibv_export_buf_dmabuf_fd(struct ibv_buf *buf); + /** * ibv_free_buf - Free a buffer allocated with ibv_alloc_buf * @buf: Handle from ibv_alloc_buf() diff --git a/pyverbs/libibverbs.pxd b/pyverbs/libibverbs.pxd index 78b801c59..a8e942aba 100644 --- a/pyverbs/libibverbs.pxd +++ b/pyverbs/libibverbs.pxd @@ -696,6 +696,7 @@ cdef extern from 'infiniband/verbs.h': ibv_mr *ibv_reg_dmabuf_mr(ibv_pd *pd, uint64_t offset, size_t length, uint64_t iova, int fd, int access) void *ibv_alloc_buf(ibv_pd *pd, size_t size, ibv_buf **buf) + int ibv_export_buf_dmabuf_fd(ibv_buf *buf) void ibv_free_buf(ibv_buf *buf) ibv_mr *ibv_reg_buf_mr(ibv_pd *pd, ibv_buf *buf, void *addr, size_t length, int access) diff --git a/pyverbs/mr.pyx b/pyverbs/mr.pyx index 1921f70fa..53f321169 100644 --- a/pyverbs/mr.pyx +++ b/pyverbs/mr.pyx @@ -764,6 +764,16 @@ cdef class Buf(PyverbsCM): else: raise PyverbsError('Unrecognized object type') + def export_dmabuf_fd(self): + """ + Export a dmabuf FD for this Buf object. + :return: A file descriptor (int) for the dmabuf FD + """ + fd = v.ibv_export_buf_dmabuf_fd(self.bufh) + if fd < 0: + raise PyverbsRDMAErrno('Failed to export dmabuf FD for Buf') + return fd + @property def addr(self): return self.addr diff --git a/tests/test_buf.py b/tests/test_buf.py index ec6cfe033..50c75b8aa 100644 --- a/tests/test_buf.py +++ b/tests/test_buf.py @@ -6,6 +6,7 @@ """ import unittest import errno +import os import resource from pyverbs.pyverbs_error import PyverbsRDMAError @@ -199,6 +200,22 @@ def get_mr(self, pd, buf, length, offset=0, via_reg_mr_ex=False): ibv_access_flags.IBV_ACCESS_LOCAL_WRITE, offset=offset, via_reg_mr_ex=via_reg_mr_ex) + def test_buf_export_dmabuf_fd_plain_pd(self): + """Plain PD: no DMA-buf fd is available for ordinary memory.""" + buf = self.get_buf(self.get_pd(), PAGE_SIZE) + with self.assertRaises(PyverbsRDMAError) as cm: + buf.export_dmabuf_fd() + self.assertEqual(cm.exception.error_code, errno.ENODATA, + 'Non-DMA-buf backed buffers must report ENODATA') + + def test_buf_export_dmabuf_fd_on_bounce_device(self): + """DMA-bounce device: a buffer's DMA-buf fd is exported.""" + if not device_has_cc_dma_bounce(self.ctx): + raise unittest.SkipTest('Device does not report CC_DMA_BOUNCE') + buf = self.get_buf(self.get_pd(cc=True), PAGE_SIZE) + fd = buf.export_dmabuf_fd() + os.close(fd) + def check_multiple_mrs_one_buf(self, pd, via_reg_mr_ex=False): """Register two disjoint subranges of one buffer and access them.""" buf = self.get_buf(pd, 2 * PAGE_SIZE)