Merge pull request #1678 from yishaih/verbs_misc

yishaih · web-flow · commit 8b9cdb7c6bd2 · 2026-02-08T11:11:58.000+02:00
verbs: Add support for exporting dma-buf file descriptors
diff --git a/debian/ibverbs-providers.symbols b/debian/ibverbs-providers.symbols
@@ -33,6 +33,7 @@ libmlx5.so.1 ibverbs-providers #MINVER#
  MLX5_1.23@MLX5_1.23 40
  MLX5_1.24@MLX5_1.24 42
  MLX5_1.25@MLX5_1.25 54
+ MLX5_1.26@MLX5_1.26 62
  mlx5dv_init_obj@MLX5_1.0 13
  mlx5dv_init_obj@MLX5_1.2 15
  mlx5dv_query_device@MLX5_1.0 13
@@ -55,6 +56,7 @@ libmlx5.so.1 ibverbs-providers #MINVER#
  mlx5dv_devx_obj_modify@MLX5_1.7 21
  mlx5dv_devx_obj_query@MLX5_1.7 21
  mlx5dv_devx_query_eqn@MLX5_1.7 21
+ mlx5dv_devx_uar_export_dmabuf_fd@MLX5_1.26 62
  mlx5dv_devx_umem_dereg@MLX5_1.7 21
  mlx5dv_devx_umem_reg@MLX5_1.7 21
  mlx5dv_open_device@MLX5_1.7 21
diff --git a/debian/libibverbs1.symbols b/debian/libibverbs1.symbols
@@ -58,6 +58,7 @@ libibverbs.so.1 libibverbs1 #MINVER#
  ibv_destroy_srq@IBVERBS_1.1 1.1.6
  ibv_detach_mcast@IBVERBS_1.0 1.1.6
  ibv_detach_mcast@IBVERBS_1.1 1.1.6
+ ibv_dm_export_dmabuf_fd@IBVERBS_1.16 62
  ibv_dofork_range@IBVERBS_1.1 1.1.6
  ibv_dontfork_range@IBVERBS_1.1 1.1.6
  ibv_event_type_str@IBVERBS_1.1 1.1.6
diff --git a/kernel-headers/rdma/bnxt_re-abi.h b/kernel-headers/rdma/bnxt_re-abi.h
@@ -56,6 +56,7 @@ enum {
 	BNXT_RE_UCNTX_CMASK_DBR_PACING_ENABLED = 0x08ULL,
 	BNXT_RE_UCNTX_CMASK_POW2_DISABLED = 0x10ULL,
 	BNXT_RE_UCNTX_CMASK_MSN_TABLE_ENABLED = 0x40,
+	BNXT_RE_UCNTX_CMASK_QP_RATE_LIMIT_ENABLED = 0x80ULL,
 };
 
 enum bnxt_re_wqe_mode {
@@ -215,4 +216,19 @@ enum bnxt_re_toggle_mem_methods {
 	BNXT_RE_METHOD_GET_TOGGLE_MEM = (1U << UVERBS_ID_NS_SHIFT),
 	BNXT_RE_METHOD_RELEASE_TOGGLE_MEM,
 };
+
+struct bnxt_re_packet_pacing_caps {
+	__u32 qp_rate_limit_min;
+	__u32 qp_rate_limit_max; /* In kbps */
+	/* Corresponding bit will be set if qp type from
+	 * 'enum ib_qp_type' is supported, e.g.
+	 * supported_qpts |= 1 << IB_QPT_RC
+	 */
+	__u32 supported_qpts;
+	__u32 reserved;
+};
+
+struct bnxt_re_query_device_ex_resp {
+	struct bnxt_re_packet_pacing_caps packet_pacing_caps;
+};
 #endif /* __BNXT_RE_UVERBS_ABI_H__*/
diff --git a/kernel-headers/rdma/ib_user_ioctl_cmds.h b/kernel-headers/rdma/ib_user_ioctl_cmds.h
@@ -56,6 +56,7 @@ enum uverbs_default_objects {
 	UVERBS_OBJECT_COUNTERS,
 	UVERBS_OBJECT_ASYNC_EVENT,
 	UVERBS_OBJECT_DMAH,
+	UVERBS_OBJECT_DMABUF,
 };
 
 enum {
@@ -263,6 +264,15 @@ enum uverbs_methods_dmah {
 	UVERBS_METHOD_DMAH_FREE,
 };
 
+enum uverbs_attrs_alloc_dmabuf_cmd_attr_ids {
+	UVERBS_ATTR_ALLOC_DMABUF_HANDLE,
+	UVERBS_ATTR_ALLOC_DMABUF_PGOFF,
+};
+
+enum uverbs_methods_dmabuf {
+	UVERBS_METHOD_DMABUF_ALLOC,
+};
+
 enum uverbs_attrs_reg_dm_mr_cmd_attr_ids {
 	UVERBS_ATTR_REG_DM_MR_HANDLE,
 	UVERBS_ATTR_REG_DM_MR_OFFSET,
diff --git a/kernel-headers/rdma/mana-abi.h b/kernel-headers/rdma/mana-abi.h
@@ -17,6 +17,9 @@
 #define MANA_IB_UVERBS_ABI_VERSION 1
 
 enum mana_ib_create_cq_flags {
+	/* Reserved for backward compatibility. Legacy
+	 * kernel versions use it to create CQs in RNIC
+	 */
 	MANA_IB_CREATE_RNIC_CQ	= 1 << 0,
 };
 
diff --git a/libibverbs/CMakeLists.txt b/libibverbs/CMakeLists.txt
@@ -29,6 +29,7 @@ rdma_library(ibverbs "${CMAKE_CURRENT_BINARY_DIR}/libibverbs.map"
   cmd_cq.c
   cmd_device.c
   cmd_dm.c
+  cmd_dmabuf.c
   cmd_dmah.c
   cmd_fallback.c
   cmd_flow.c
diff --git a/libibverbs/cmd_dmabuf.c b/libibverbs/cmd_dmabuf.c
@@ -0,0 +1,24 @@
+// SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
+/*
+ * Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved
+ */
+
+#include <infiniband/cmd_write.h>
+
+int ibv_cmd_export_dmabuf_fd(struct ibv_context *ctx, off_t pg_off)
+{
+	DECLARE_COMMAND_BUFFER(cmd, UVERBS_OBJECT_DMABUF,
+			       UVERBS_METHOD_DMABUF_ALLOC, 2);
+	struct ib_uverbs_attr *handle;
+	int ret;
+
+	handle = fill_attr_out_fd(cmd, UVERBS_ATTR_ALLOC_DMABUF_HANDLE, 0);
+	fill_attr_in_uint64(cmd, UVERBS_ATTR_ALLOC_DMABUF_PGOFF, pg_off);
+
+	ret = execute_ioctl(ctx, cmd);
+	if (ret)
+		/* errno is set to a postive value internally */
+		return -1;
+
+	return read_attr_fd(UVERBS_ATTR_ALLOC_DMABUF_HANDLE, handle);
+}
diff --git a/libibverbs/driver.h b/libibverbs/driver.h
@@ -407,6 +407,7 @@ struct verbs_context_ops {
 	int (*destroy_wq)(struct ibv_wq *wq);
 	int (*detach_mcast)(struct ibv_qp *qp, const union ibv_gid *gid,
 			    uint16_t lid);
+	int (*dm_export_dmabuf_fd)(struct ibv_dm *dm);
 	void (*free_context)(struct ibv_context *context);
 	int (*free_dm)(struct ibv_dm *dm);
 	int (*get_srq_num)(struct ibv_srq *srq, uint32_t *srq_num);
@@ -743,6 +744,7 @@ int ibv_cmd_alloc_dm(struct ibv_context *ctx,
 		     struct verbs_dm *dm,
 		     struct ibv_command_buffer *link);
 int ibv_cmd_free_dm(struct verbs_dm *dm);
+int ibv_cmd_export_dmabuf_fd(struct ibv_context *ctx, off_t pg_off);
 int ibv_cmd_alloc_dmah(struct ibv_context *ctx, struct verbs_dmah *st,
 		       struct ibv_dmah_init_attr *attr);
 int ibv_cmd_free_dmah(struct verbs_dmah *dmah);
diff --git a/libibverbs/dummy_ops.c b/libibverbs/dummy_ops.c
@@ -285,6 +285,12 @@ static int detach_mcast(struct ibv_qp *qp, const union ibv_gid *gid,
 	return EOPNOTSUPP;
 }
 
+static int dm_export_dmabuf_fd(struct ibv_dm *dm)
+{
+	errno = EOPNOTSUPP;
+	return -1;
+}
+
 static void free_context(struct ibv_context *ctx)
 {
 	return;
@@ -569,6 +575,7 @@ const struct verbs_context_ops verbs_dummy_ops = {
 	destroy_srq,
 	destroy_wq,
 	detach_mcast,
+	dm_export_dmabuf_fd,
 	free_context,
 	free_dm,
 	get_srq_num,
@@ -698,6 +705,7 @@ void verbs_set_ops(struct verbs_context *vctx,
 	SET_PRIV_OP(ctx, destroy_srq);
 	SET_OP(vctx, destroy_wq);
 	SET_PRIV_OP(ctx, detach_mcast);
+	SET_OP(vctx, dm_export_dmabuf_fd);
 	SET_PRIV_OP_IC(ctx, free_context);
 	SET_OP(vctx, free_dm);
 	SET_OP(vctx, get_srq_num);
diff --git a/libibverbs/libibverbs.map.in b/libibverbs/libibverbs.map.in
@@ -173,6 +173,7 @@ IBVERBS_1.15 {
 
 IBVERBS_1.16 {
 	global:
+		ibv_dm_export_dmabuf_fd;
 		ibv_query_port_speed;
 } IBVERBS_1.15;
 
@@ -220,6 +221,7 @@ IBVERBS_PRIVATE_@IBVERBS_PABI_VERSION@ {
 		ibv_cmd_destroy_srq;
 		ibv_cmd_destroy_wq;
 		ibv_cmd_detach_mcast;
+		ibv_cmd_export_dmabuf_fd;
 		ibv_cmd_free_dm;
 		ibv_cmd_free_dmah;
 		ibv_cmd_get_context;
diff --git a/libibverbs/man/CMakeLists.txt b/libibverbs/man/CMakeLists.txt
@@ -28,6 +28,7 @@ rdma_man_pages(
   ibv_create_wq.3
   ibv_devices.1
   ibv_devinfo.1
+  ibv_dm_export_dmabuf_fd.3.md
   ibv_event_type_str.3.md
   ibv_fork_init.3.md
   ibv_get_async_event.3
diff --git a/libibverbs/man/ibv_dm_export_dmabuf_fd.3.md b/libibverbs/man/ibv_dm_export_dmabuf_fd.3.md
@@ -0,0 +1,49 @@
+---
+date: 2025-12-7
+footer: libibverbs
+header: "Libibverbs Programmer's Manual"
+layout: page
+license: 'Licensed under the OpenIB.org BSD license (FreeBSD Variant) - See COPYING.md'
+section: 3
+title: ibv_dm_export_dmabuf_fd
+---
+
+# NAME
+
+ibv_dm_export_dmabuf_fd - export dmabuf fd for a given ibv_dm.
+
+# SYNOPSIS
+
+```c
+#include <infiniband/verbs.h>
+
+int ibv_dm_export_dmabuf_fd(struct ibv_dm *dm);
+```
+
+# DESCRIPTION
+
+**ibv_dm_export_dmabuf_fd()** exports a dmabuf fd that is associated with the given
+*dm*.
+
+The returned fd can be later used for DMA and RDMA operations associated with it.
+
+Once the usage has been ended close() should be called while supplying the fd.
+
+This call will release resources that were earlier allocated using the **ibv_dm_export_dmabuf_fd()** API.
+
+# ARGUMENTS
+
+*dm*
+:	An ibv_dm pointer to export the dmabuf fd for its memory.
+
+# RETURN VALUE
+
+**ibv_dm_export_dmabuf_fd()** returns an fd number >= 0 upon success, or -1 if the request fails and errno set to the error.
+
+# SEE ALSO
+
+**ibv_reg_mr_ex**(3)
+
+# AUTHOR
+
+Yishai Hadas <yishaih@nvidia.com>
diff --git a/libibverbs/verbs.c b/libibverbs/verbs.c
@@ -412,6 +412,11 @@ int ibv_dealloc_dmah(struct ibv_dmah *dmah)
 	return get_ops(dmah->context)->dealloc_dmah(dmah);
 }
 
+int ibv_dm_export_dmabuf_fd(struct ibv_dm *dm)
+{
+	return get_ops(dm->context)->dm_export_dmabuf_fd(dm);
+}
+
 struct ibv_mr *ibv_reg_dmabuf_mr(struct ibv_pd *pd, uint64_t offset,
 				 size_t length, uint64_t iova, int fd,
 				 int access)
diff --git a/libibverbs/verbs.h b/libibverbs/verbs.h
@@ -2183,6 +2183,7 @@ struct ibv_values_ex {
 
 struct verbs_context {
 	/*  "grows up" - new fields go here */
+	int (*dm_export_dmabuf_fd)(struct ibv_dm *dm);
 	struct ibv_mr *(*reg_mr_ex)(struct ibv_pd *pd,
 				    struct ibv_mr_init_attr *mr_init_attr);
 	int (*dealloc_dmah)(struct ibv_dmah *dmah);
@@ -2837,6 +2838,8 @@ int ibv_free_dm(struct ibv_dm *dm)
 	return vctx->free_dm(dm);
 }
 
+int ibv_dm_export_dmabuf_fd(struct ibv_dm *dm);
+
 /**
  * ibv_memcpy_to/from_dm - copy to/from device allocated memory
  * @dm - The DM to copy to/from
diff --git a/providers/mlx5/CMakeLists.txt b/providers/mlx5/CMakeLists.txt
@@ -15,7 +15,7 @@ if (ENABLE_LTTNG AND LTTNGUST_FOUND)
 endif()
 
 rdma_shared_provider(mlx5 libmlx5.map
-  1 1.25.${PACKAGE_VERSION}
+  1 1.26.${PACKAGE_VERSION}
   ${TRACE_FILE}
   buf.c
   cq.c
diff --git a/providers/mlx5/libmlx5.map b/providers/mlx5/libmlx5.map
@@ -235,3 +235,8 @@ MLX5_1.25 {
 		mlx5dv_get_data_direct_sysfs_path;
 		mlx5dv_reg_dmabuf_mr;
 } MLX5_1.24;
+
+MLX5_1.26 {
+	global:
+		mlx5dv_devx_uar_export_dmabuf_fd;
+} MLX5_1.25;
diff --git a/providers/mlx5/man/CMakeLists.txt b/providers/mlx5/man/CMakeLists.txt
@@ -23,6 +23,7 @@ rdma_man_pages(
   mlx5dv_devx_qp_modify.3.md
   mlx5dv_devx_query_eqn.3.md
   mlx5dv_devx_subscribe_devx_event.3.md
+  mlx5dv_devx_uar_export_dmabuf_fd.3.md
   mlx5dv_devx_umem_reg.3.md
   mlx5dv_dm_map_op_addr.3.md
   mlx5dv_dr_flow.3.md
diff --git a/providers/mlx5/man/mlx5dv_devx_uar_export_dmabuf_fd.3.md b/providers/mlx5/man/mlx5dv_devx_uar_export_dmabuf_fd.3.md
@@ -0,0 +1,47 @@
+---
+layout: page
+title: mlx5dv_devx_uar_export_dmabuf_fd
+section: 3
+tagline: Verbs
+---
+
+# NAME
+
+mlx5dv_devx_uar_export_dmabuf_fd - export dmabuf fd for a given mlx5dv_devx_uar.
+
+# SYNOPSIS
+
+```c
+#include <infiniband/mlx5dv.h>
+
+int mlx5dv_devx_uar_export_dmabuf_fd(struct mlx5dv_devx_uar *devx_uar);
+```
+
+# DESCRIPTION
+
+**mlx5dv_devx_uar_export_dmabuf_fd()** exports a dmabuf fd that is associated with the given
+*devx_uar*.
+
+The returned fd can be later used for DMA and RDMA operations associated with it.
+
+Once the usage has been ended, close() should be called while supplying the fd.
+
+This call will release resources that were earlier allocated using the **mlx5dv_devx_uar_export_dmabuf_fd()** API.
+
+# ARGUMENTS
+
+*devx_uar*
+:	An mlx5dv_devx_uar pointer to export the dmabuf fd for its memory.
+
+# RETURN VALUE
+
+**mlx5dv_devx_uar_export_dmabuf_fd()** returns an fd number >= 0 upon success, or -1 if the request fails and errno set to the error.
+
+# SEE ALSO
+
+**ibv_reg_mr_ex**(3)
+**ibv_dm_export_dmabuf_fd**(3)
+
+# AUTHOR
+
+Yishai Hadas <yishaih@nvidia.com>
diff --git a/providers/mlx5/mlx5.c b/providers/mlx5/mlx5.c
@@ -177,6 +177,7 @@ static const struct verbs_context_ops mlx5_ctx_common_ops = {
 	.dealloc_dmah = mlx5_dealloc_dmah,
 	.reg_mr_ex = mlx5_reg_mr_ex,
 	.query_port_speed = mlx5_query_port_speed,
+	.dm_export_dmabuf_fd = mlx5_dm_export_dmabuf_fd,
 };
 
 static const struct verbs_context_ops mlx5_ctx_cqev1_ops = {
diff --git a/providers/mlx5/mlx5.h b/providers/mlx5/mlx5.h
@@ -645,6 +645,7 @@ struct mlx5_dm {
 	void			       *mmap_va;
 	void			       *start_va;
 	uint64_t			remote_va;
+	off_t pg_off;
 };
 
 struct mlx5_mr {
@@ -1268,6 +1269,7 @@ int mlx5_modify_flow_action_esp(struct ibv_flow_action *action,
 struct ibv_dm *mlx5_alloc_dm(struct ibv_context *context,
 			     struct ibv_alloc_dm_attr *dm_attr);
 int mlx5_free_dm(struct ibv_dm *ibdm);
+int mlx5_dm_export_dmabuf_fd(struct ibv_dm *dm);
 struct ibv_mr *mlx5_reg_dm_mr(struct ibv_pd *pd, struct ibv_dm *ibdm,
 			      uint64_t dm_offset, size_t length,
 			      unsigned int acc);
@@ -1635,6 +1637,7 @@ struct mlx5_dv_context_ops {
 					size_t length, uint64_t iova, int fd,
 					int access, int mlx5_access);
 	int (*get_data_direct_sysfs_path)(struct ibv_context *context, char *buf, size_t buf_len);
+	int (*devx_uar_export_dmabuf_fd)(struct mlx5dv_devx_uar *dv_devx_uar);
 };
 
 struct mlx5_dv_context_ops *mlx5_get_dv_ops(struct ibv_context *context);
diff --git a/providers/mlx5/mlx5dv.h b/providers/mlx5/mlx5dv.h
@@ -1819,6 +1819,7 @@ struct mlx5dv_devx_uar *mlx5dv_devx_alloc_uar(struct ibv_context *context,
 					      uint32_t flags);
 void mlx5dv_devx_free_uar(struct mlx5dv_devx_uar *devx_uar);
 
+int mlx5dv_devx_uar_export_dmabuf_fd(struct mlx5dv_devx_uar *devx_uar);
 
 struct mlx5dv_var {
 	uint32_t page_id;
diff --git a/providers/mlx5/verbs.c b/providers/mlx5/verbs.c
