libibverbs: Add DMA-buf fd export for ibv_buf

debian/libibverbs1.symbols

@@ -15,6 +15,7 @@ libibverbs.so.1 libibverbs1 #MINVER#
  IBVERBS_1.15@IBVERBS_1.15 59
  IBVERBS_1.16@IBVERBS_1.16 62
  IBVERBS_1.17@IBVERBS_1.17 64
+ IBVERBS_1.18@IBVERBS_1.18 65
  (symver)IBVERBS_PRIVATE_64 64
  _ibv_query_gid_ex@IBVERBS_1.11 32
  _ibv_query_gid_table@IBVERBS_1.11 32
@@ -64,6 +65,7 @@ libibverbs.so.1 libibverbs1 #MINVER#
  ibv_dofork_range@IBVERBS_1.1 1.1.6
  ibv_dontfork_range@IBVERBS_1.1 1.1.6
  ibv_event_type_str@IBVERBS_1.1 1.1.6
+ ibv_export_buf_dmabuf_fd@IBVERBS_1.18 65
  ibv_fork_init@IBVERBS_1.1 1.1.6
  ibv_free_buf@IBVERBS_1.17 64
  ibv_free_device_list@IBVERBS_1.0 1.1.6

libibverbs/CMakeLists.txt

@@ -22,7 +22,7 @@ configure_file("libibverbs.map.in"
 
 rdma_library(ibverbs "${CMAKE_CURRENT_BINARY_DIR}/libibverbs.map"
   # See Documentation/versioning.md
-  1 1.17.${PACKAGE_VERSION}
+  1 1.18.${PACKAGE_VERSION}
   all_providers.c
   cmd.c
   cmd_ah.c

libibverbs/libibverbs.map.in

@@ -184,6 +184,11 @@ IBVERBS_1.17 {
 		ibv_reg_buf_mr;
 } IBVERBS_1.16;
 
+IBVERBS_1.18 {
+	global:
+		ibv_export_buf_dmabuf_fd;
+} IBVERBS_1.17;
+
 /* If any symbols in this stanza change ABI then the entire staza gets a new symbol
    version. See the top level CMakeLists.txt for this setting. */
 

libibverbs/man/CMakeLists.txt

@@ -89,6 +89,7 @@ rdma_man_pages(
 rdma_alias_man_pages(
   ibv_alloc_buf.3 ibv_free_buf.3
   ibv_alloc_buf.3 ibv_reg_buf_mr.3
+  ibv_alloc_buf.3 ibv_export_buf_dmabuf_fd.3
   ibv_alloc_dm.3 ibv_free_dm.3
   ibv_alloc_dm.3 ibv_reg_dm_mr.3
   ibv_alloc_dm.3 ibv_memcpy_to_dm.3

libibverbs/man/ibv_alloc_buf.3.md

@@ -10,7 +10,7 @@ title: ibv_alloc_buf
 
 # NAME
 
-ibv_alloc_buf, ibv_free_buf, ibv_reg_buf_mr - allocate provider-aware buffers and register them as memory regions
+ibv_alloc_buf, ibv_free_buf, ibv_reg_buf_mr, ibv_export_buf_dmabuf_fd - allocate provider-aware buffers and register them as memory regions
 
 # SYNOPSIS
 
@@ -23,15 +23,18 @@ void ibv_free_buf(struct ibv_buf *buf);
 
 struct ibv_mr *ibv_reg_buf_mr(struct ibv_pd *pd, struct ibv_buf *buf, void *addr,
                               size_t length, int access);
+
+int ibv_export_buf_dmabuf_fd(struct ibv_buf *buf);
 ```
 
 # DESCRIPTION
 
 **ibv_alloc_buf()** allocates a buffer using the allocation method selected by
 the provider for the protection domain *pd*. On success it returns the mapped
 address and stores an opaque buffer handle in *buf*. The handle is used by
-**ibv_free_buf()**, **ibv_reg_buf_mr()**, and **ibv_reg_mr_ex()** with
-**IBV_REG_MR_MASK_BUF**, and must not be interpreted by applications.
+**ibv_free_buf()**, **ibv_reg_buf_mr()**, **ibv_reg_mr_ex()** with
+**IBV_REG_MR_MASK_BUF**, and **ibv_export_buf_dmabuf_fd()**; it must not be
+interpreted by applications.
 
 **ibv_free_buf()** releases a buffer handle returned by **ibv_alloc_buf()**. The
 protection domain used for allocation must remain valid until the buffer is
@@ -53,6 +56,14 @@ For provider allocations backed by a DMA-buf, it registers
 the corresponding DMA-buf range using the metadata stored in
 the opaque *buf* handle.
 
+**ibv_export_buf_dmabuf_fd()** exports a new file descriptor for a buffer that
+is backed by a DMA-buf. This allows applications to pass provider-allocated
+memory to other APIs that import DMA-buf file descriptors. The returned file
+descriptor is owned by the caller and should be closed with **close(2)** when
+no longer needed. Calling
+**ibv_export_buf_dmabuf_fd()** does not transfer ownership of the buffer handle
+or the internal descriptor used by libibverbs.
+
 # ARGUMENTS
 
 *pd*
@@ -62,7 +73,7 @@ the opaque *buf* handle.
 :	Size of the buffer to allocate, in bytes (**ibv_alloc_buf()**).
 
 *buf*
-:	For **ibv_alloc_buf()**, an output parameter set on success to an opaque buffer handle. For **ibv_free_buf()** and **ibv_reg_buf_mr()**, the buffer handle returned by **ibv_alloc_buf()** to be released or registered, respectively.
+:	For **ibv_alloc_buf()**, an output parameter set on success to an opaque buffer handle. For **ibv_free_buf()** and **ibv_reg_buf_mr()**, the buffer handle returned by **ibv_alloc_buf()** to be released or registered, respectively. For **ibv_export_buf_dmabuf_fd()**, the buffer handle whose DMA-buf fd should be exported.
 
 *addr*
 :	The start address to register (**ibv_reg_buf_mr()**): the buffer base returned by **ibv_alloc_buf()** or an address within that buffer.
@@ -83,6 +94,10 @@ if the request fails.
 
 **ibv_free_buf()** does not return a value.
 
+**ibv_export_buf_dmabuf_fd()** returns a file descriptor >= 0 on success, or -1
+if the request fails with *errno* set. If *buf* is not backed by a DMA-buf,
+*errno* is set to **ENODATA**.
+
 # NOTES
 
 Applications running in a CoCo guest that need unprotected/shared memory should

libibverbs/verbs.c

@@ -40,6 +40,7 @@
 #include <unistd.h>
 #include <stdlib.h>
 #include <errno.h>
+#include <fcntl.h>
 #include <string.h>
 #include <linux/ip.h>
 #include <dirent.h>
@@ -440,6 +441,16 @@ void *ibv_alloc_buf(struct ibv_pd *pd, size_t size, struct ibv_buf **buf)
 	return get_ops(pd->context)->alloc_buf(pd, size, buf);
 }
 
+int ibv_export_buf_dmabuf_fd(struct ibv_buf *buf)
+{
+	if (buf->dmabuf_fd < 0) {
+		errno = ENODATA;
+		return -1;
+	}
+
+	return fcntl(buf->dmabuf_fd, F_DUPFD_CLOEXEC, 0);
+}
+
 struct ibv_mr *ibv_reg_buf_mr(struct ibv_pd *pd, struct ibv_buf *buf,
 			      void *addr, size_t length, int access)
 {

libibverbs/verbs.h

@@ -3218,6 +3218,16 @@ ibv_alloc_parent_domain(struct ibv_context *context,
  */
 void *ibv_alloc_buf(struct ibv_pd *pd, size_t size, struct ibv_buf **buf);
 
+/**
+ * ibv_export_buf_dmabuf_fd - Export a DMA-buf fd for an ibv_buf
+ * @buf: Handle from ibv_alloc_buf()
+ *
+ * Returns a new fd for the DMA-buf backing the buffer on success, or -1
+ * on failure with errno set. The returned fd is owned by the caller and
+ * should be closed with close(2) when no longer needed.
+ */
+int ibv_export_buf_dmabuf_fd(struct ibv_buf *buf);
+
 /**
  * ibv_free_buf - Free a buffer allocated with ibv_alloc_buf
  * @buf: Handle from ibv_alloc_buf()

pyverbs/libibverbs.pxd

@@ -696,6 +696,7 @@ cdef extern from 'infiniband/verbs.h':
     ibv_mr *ibv_reg_dmabuf_mr(ibv_pd *pd, uint64_t offset, size_t length,
                               uint64_t iova, int fd, int access)
     void *ibv_alloc_buf(ibv_pd *pd, size_t size, ibv_buf **buf)
+    int ibv_export_buf_dmabuf_fd(ibv_buf *buf)
     void ibv_free_buf(ibv_buf *buf)
     ibv_mr *ibv_reg_buf_mr(ibv_pd *pd, ibv_buf *buf, void *addr,
                            size_t length, int access)

pyverbs/mr.pyx

@@ -764,6 +764,16 @@ cdef class Buf(PyverbsCM):
         else:
             raise PyverbsError('Unrecognized object type')
 
+    def export_dmabuf_fd(self):
+        """
+        Export a dmabuf FD for this Buf object.
+        :return: A file descriptor (int) for the dmabuf FD
+        """
+        fd = v.ibv_export_buf_dmabuf_fd(self.bufh)
+        if fd < 0:
+            raise PyverbsRDMAErrno('Failed to export dmabuf FD for Buf')
+        return fd
+
     @property
     def addr(self):
         return <uintptr_t>self.addr

tests/test_buf.py

@@ -6,6 +6,7 @@
 """
 import unittest
 import errno
+import os
 import resource
 
 from pyverbs.pyverbs_error import PyverbsRDMAError
@@ -199,6 +200,22 @@ def get_mr(self, pd, buf, length, offset=0, via_reg_mr_ex=False):
                                ibv_access_flags.IBV_ACCESS_LOCAL_WRITE,
                                offset=offset, via_reg_mr_ex=via_reg_mr_ex)
 
+    def test_buf_export_dmabuf_fd_plain_pd(self):
+        """Plain PD: no DMA-buf fd is available for ordinary memory."""
+        buf = self.get_buf(self.get_pd(), PAGE_SIZE)
+        with self.assertRaises(PyverbsRDMAError) as cm:
+            buf.export_dmabuf_fd()
+        self.assertEqual(cm.exception.error_code, errno.ENODATA,
+                         'Non-DMA-buf backed buffers must report ENODATA')
+
+    def test_buf_export_dmabuf_fd_on_bounce_device(self):
+        """DMA-bounce device: a buffer's DMA-buf fd is exported."""
+        if not device_has_cc_dma_bounce(self.ctx):
+            raise unittest.SkipTest('Device does not report CC_DMA_BOUNCE')
+        buf = self.get_buf(self.get_pd(cc=True), PAGE_SIZE)
+        fd = buf.export_dmabuf_fd()
+        os.close(fd)
+
     def check_multiple_mrs_one_buf(self, pd, via_reg_mr_ex=False):
         """Register two disjoint subranges of one buffer and access them."""
         buf = self.get_buf(pd, 2 * PAGE_SIZE)