bnxt_re/lib: Direct Verbs: Support CQ and QP verbs

The following Direct Verb (DV) APIs have been implemented in
this patch. These are implemented by enhancing the driver
specific udata in existing verbs.

CQ Direct Verbs:
----------------
- bnxt_re_dv_create_cq():
  Create a CQ of requested size (cqe). The application must have
  already registered this memory using bnxt_re_dv_umem_reg().
  The CQ umem-handle and umem-offset provided by the application
  are translated into an address for mapping and passed to the
  driver.

- bnxt_re_dv_destroy_cq():
  Destroy the DV_CQ created earlier.

QP Direct Verbs:
----------------
- bnxt_re_dv_create_qp():
  Create a QP using specified params (struct bnxt_re_dv_qp_init_attr).
  The application must have already registered SQ/RQ memory using
  bnxt_re_dv_umem_reg(). The SQ/RQ umem-handle and umem-offset provided
  by the application are translated into an address for mapping and
  passed to the driver.

- bnxt_re_dv_destroy_qp():
  Destroy the DV_QP created earlier.

- bnxt_re_dv_modify_qp():
  Modify QP attributes of the DV_QP.

- bnxt_re_dv_query_qp():
  Return QP attributes of the DV_QP.

Note:
-----
Some applications might want to allocate memory for all resources of a
given type (CQ/QP) in one big chunk and then register that entire
memory once using dv_umem_reg(). At the time of creating each individual
resource, the application should pass a specific offset/length in the
umem registered memory.

Signed-off-by: Sriharsha Basavapatna <sriharsha.basavapatna@broadcom.com>
Co-developed-by: Kalesh AP <kalesh-anakkur.purayil@broadcom.com>
Signed-off-by: Kalesh AP <kalesh-anakkur.purayil@broadcom.com>
Co-developed-by: Selvin Xavier <selvin.xavier@broadcom.com>
Signed-off-by: Selvin Xavier <selvin.xavier@broadcom.com>

Author: sbasavapatna

providers/bnxt_re/CMakeLists.txt

@@ -1,7 +1,19 @@
-rdma_provider(bnxt_re
+rdma_shared_provider(bnxt_re bnxt_re.map
+	1 1.1.${PACKAGE_VERSION}
 	db.c
 	main.c
 	memory.c
 	verbs.c
 	dv.c
 )
+
+publish_headers(infiniband
+  bnxt_re_dv.h
+)
+
+rdma_pkg_config("bnxt_re" "libibverbs" "${CMAKE_THREAD_LIBS_INIT}")
+
+if (ENABLE_LTTNG AND LTTNGUST_FOUND)
+  target_include_directories(bnxt_re PUBLIC ".")
+  target_link_libraries(bnxt_re LINK_PRIVATE LTTng::UST)
+endif()

providers/bnxt_re/bnxt_re-abi.h

@@ -47,11 +47,11 @@
 
 DECLARE_DRV_CMD(ubnxt_re_pd, IB_USER_VERBS_CMD_ALLOC_PD,
 		empty, bnxt_re_pd_resp);
-DECLARE_DRV_CMD(ubnxt_re_cq, IB_USER_VERBS_CMD_CREATE_CQ,
+DECLARE_DRV_CMD(ubnxt_re_cq, IB_USER_VERBS_EX_CMD_CREATE_CQ,
 		bnxt_re_cq_req, bnxt_re_cq_resp);
 DECLARE_DRV_CMD(ubnxt_re_resize_cq, IB_USER_VERBS_CMD_RESIZE_CQ,
 		bnxt_re_resize_cq_req, empty);
-DECLARE_DRV_CMD(ubnxt_re_qp, IB_USER_VERBS_CMD_CREATE_QP,
+DECLARE_DRV_CMD(ubnxt_re_qp, IB_USER_VERBS_EX_CMD_CREATE_QP,
 		bnxt_re_qp_req, bnxt_re_qp_resp);
 DECLARE_DRV_CMD(ubnxt_re_cntx, IB_USER_VERBS_CMD_GET_CONTEXT,
 		bnxt_re_uctx_req, bnxt_re_uctx_resp);
@@ -226,6 +226,7 @@ enum {
 	BNXT_RE_COMP_MASK_UCNTX_DBR_PACING_ENABLED = 0x02,
 	BNXT_RE_COMP_MASK_UCNTX_POW2_DISABLED = 0x04,
 	BNXT_RE_COMP_MASK_UCNTX_MSN_TABLE_ENABLED = 0x08,
+	BNXT_RE_COMP_MASK_UCNTX_UAPI_COMPAT_SUPPORTED = 0x10,
 };
 
 enum bnxt_re_que_flags_mask {

providers/bnxt_re/bnxt_re.map

@@ -1,10 +1,17 @@
-{
+/* Export symbols should be added below according to
+   Documentation/versioning.md document. */
+BNXT_RE_1.1 {
 	global:
-		openib_driver_init;
 		bnxt_re_dv_alloc_db_region;
 		bnxt_re_dv_free_db_region;
 		bnxt_re_dv_umem_reg;
 		bnxt_re_dv_umem_dereg;
 		bnxt_re_dv_get_default_db_region;
+		bnxt_re_dv_create_cq;
+		bnxt_re_dv_destroy_cq;
+		bnxt_re_dv_create_qp;
+		bnxt_re_dv_destroy_qp;
+		bnxt_re_dv_modify_qp;
+		bnxt_re_dv_query_qp;
 	local: *;
 };

providers/bnxt_re/bnxt_re_dv.h

@@ -64,6 +64,48 @@ struct bnxt_re_dv_umem_reg_attr {
 };
 
 struct bnxt_re_dv_umem;
+struct bnxt_re_dv_cq_init_attr {
+	uint64_t cq_handle;
+	void *umem_handle; /* umem_handle from umem_reg */
+	uint64_t cq_umem_offset;	/* offset into umem */
+	uint32_t ncqe;
+};
+
+struct bnxt_re_dv_cq_attr {
+	uint32_t ncqe; /* no. of entries */
+	uint32_t cqe_size; /* size of entries */
+};
+
+struct bnxt_re_dv_qp_init_attr {
+	/* Standard ibv params */
+	enum ibv_qp_type qp_type;
+	uint32_t max_send_wr;
+	uint32_t max_recv_wr;
+	uint32_t max_send_sge;
+	uint32_t max_recv_sge;
+	uint32_t max_inline_data;
+	struct ibv_cq *send_cq;
+	struct ibv_cq *recv_cq;
+	struct ibv_srq *srq;
+
+	/* DV params */
+	uint64_t qp_handle;	/* to match with cqe */
+	void *dbr_handle;	/* dbr_handle from alloc_dbr */
+	void *sq_umem_handle;	/* umem_handle from umem_reg */
+	uint64_t sq_umem_offset;	/* offset into umem */
+	uint32_t sq_len;	/* sq length including MSN area */
+	uint32_t sq_slots;	/* sq length in slots */
+	void *rq_umem_handle;	/* umem_handle from umem_reg */
+	uint64_t rq_umem_offset;	/* offset into umem */
+	uint32_t sq_wqe_sz;     /* sq wqe size */
+	uint32_t sq_psn_sz;     /* sq psn size */
+	uint32_t sq_npsn;       /* sq num psn entries */
+	uint32_t rq_len;	/* rq length */
+	uint32_t rq_slots;	/* rq length in slots */
+	uint32_t rq_wqe_sz;     /* rq wqe size */
+	uint64_t comp_mask;	/* compatibility mask for future updates */
+};
+
 struct bnxt_re_dv_db_region_attr *
 bnxt_re_dv_alloc_db_region(struct ibv_context *ctx);
 int bnxt_re_dv_free_db_region(struct ibv_context *ctx,
@@ -73,6 +115,15 @@ int bnxt_re_dv_get_default_db_region(struct ibv_context *ibvctx,
 struct bnxt_re_dv_umem *bnxt_re_dv_umem_reg(struct ibv_context *ibvctx,
 					    struct bnxt_re_dv_umem_reg_attr *in);
 int bnxt_re_dv_umem_dereg(struct bnxt_re_dv_umem *umem);
+struct ibv_cq *bnxt_re_dv_create_cq(struct ibv_context *ibvctx,
+				    struct bnxt_re_dv_cq_init_attr *cq_attr);
+int bnxt_re_dv_destroy_cq(struct ibv_cq *ibv_cq);
+struct ibv_qp *bnxt_re_dv_create_qp(struct ibv_pd *pd,
+				    struct bnxt_re_dv_qp_init_attr *qp_attr);
+int bnxt_re_dv_destroy_qp(struct ibv_qp *ibvqp);
+int bnxt_re_dv_modify_qp(struct ibv_qp *ibv_qp, struct ibv_qp_attr *attr,
+			 int attr_mask);
+int bnxt_re_dv_query_qp(void *qp_handle, struct ib_uverbs_qp_attr *attr);
 #ifdef __cplusplus
 }
 #endif