providers/mana: UC QP support

Implement creation and destruction of UC QPs.
Add processing of UC send CQEs.

Signed-off-by: Konstantin Taranov <kotaranov@microsoft.com>

Author: Konstantin Taranov

providers/mana/cq.c

@@ -284,6 +284,25 @@ static inline int advance_send_completions(struct mana_qp *qp, uint32_t psn,
 	return produced;
 }
 
+static inline int handle_requester_cqe(struct mana_qp *qp, struct gdma_cqe *cqe, struct ibv_wc *wc)
+{
+	struct mana_gdma_queue *send_queue = mana_ib_get_sreq(qp);
+	struct shadow_wqe_header *wqe;
+	int produced = 0;
+
+	while (!produced && (wqe = shadow_queue_get_next_to_consume(&qp->shadow_sq)) != NULL) {
+		send_queue->cons_idx += wqe->posted_wqe_size_in_bu;
+		send_queue->cons_idx &= GDMA_QUEUE_OFFSET_MASK;
+		if (wqe->flags != MANA_NO_SIGNAL_WC) {
+			fill_verbs_from_shadow_wqe(qp, wc, wqe);
+			produced++;
+		}
+		shadow_queue_advance_consumer(&qp->shadow_sq);
+	}
+
+	return produced;
+}
+
 static inline int handle_rc_requester_cqe(struct mana_qp *qp, struct gdma_cqe *cqe,
 					  struct ibv_wc *wc, int nwc, bool *consumed)
 {
@@ -443,6 +462,8 @@ static inline int mana_handle_cqe(struct mana_context *ctx, struct gdma_cqe *cqe
 		return handle_error_cqe(qp, cqe, wc, nwc, consumed);
 	else if (cqe->rdma_cqe.cqe_type == CQE_TYPE_ARMED_CMPL)
 		return handle_rc_requester_cqe(qp, cqe, wc, nwc, consumed);
+	else if (cqe->is_sq && cqe->rdma_cqe.cqe_type == CQE_TYPE_UD_SEND)
+		return handle_requester_cqe(qp, cqe, wc);
 	else
 		return handle_responder_cqe(qp, cqe, wc);
 }

providers/mana/mana.h

@@ -49,7 +49,8 @@ enum user_queue_types {
 	USER_RNIC_SEND_QUEUE_RESPONDER = 1,
 	USER_RNIC_RECV_QUEUE_REQUESTER = 2,
 	USER_RNIC_RECV_QUEUE_RESPONDER = 3,
-	USER_RNIC_QUEUE_TYPE_MAX = 4,
+	USER_RNIC_SEND_QUEUE_MM = 4,
+	USER_RNIC_QUEUE_TYPE_MAX = 5,
 };
 
 #define QUEUE_TYPE_MASK 0x3

providers/mana/qp.c

@@ -31,6 +31,9 @@ DECLARE_DRV_CMD(mana_create_qp_ex, IB_USER_VERBS_EX_CMD_CREATE_QP,
 DECLARE_DRV_CMD(mana_create_rc_qp, IB_USER_VERBS_CMD_CREATE_QP,
 		mana_ib_create_rc_qp, mana_ib_create_rc_qp_resp);
 
+DECLARE_DRV_CMD(mana_create_uc_qp, IB_USER_VERBS_CMD_CREATE_QP,
+		mana_ib_create_uc_qp, mana_ib_create_uc_qp_resp);
+
 static struct ibv_qp *mana_create_qp_raw(struct ibv_pd *ibpd,
 					 struct ibv_qp_init_attr *attr)
 {
@@ -216,56 +219,149 @@ static uint32_t get_queue_size(struct ibv_qp_init_attr *attr, enum user_queue_ty
 	uint32_t size = 0;
 	uint32_t sges = 0;
 
-	if (attr->qp_type == IBV_QPT_RC) {
-		switch (type) {
-		case USER_RNIC_SEND_QUEUE_REQUESTER:
-			/* WQE must have at least one SGE */
-			/* For write with imm we need one extra SGE */
-			sges = max(1U, attr->cap.max_send_sge) + 1;
-			size = attr->cap.max_send_wr * get_large_wqe_size(sges);
-			break;
-		case USER_RNIC_SEND_QUEUE_RESPONDER:
-			size = MANA_PAGE_SIZE;
-			break;
-		case USER_RNIC_RECV_QUEUE_REQUESTER:
-			size = MANA_PAGE_SIZE;
-			break;
-		case USER_RNIC_RECV_QUEUE_RESPONDER:
-			/* WQE must have at least one SGE */
-			sges = max(1U, attr->cap.max_recv_sge);
-			size = attr->cap.max_recv_wr * get_wqe_size(sges);
-			break;
-		default:
-			return 0;
-		}
+	switch (type) {
+	case USER_RNIC_SEND_QUEUE_REQUESTER:
+		/* WQE must have at least one SGE */
+		/* For write with imm we need one extra SGE */
+		sges = max(1U, attr->cap.max_send_sge) + 1;
+		size = align_hw_size(attr->cap.max_send_wr * get_large_wqe_size(sges));
+		break;
+	case USER_RNIC_SEND_QUEUE_RESPONDER:
+		if (attr->qp_type == IBV_QPT_RC)
+			size = align_hw_size(MANA_PAGE_SIZE);
+		break;
+	case USER_RNIC_RECV_QUEUE_REQUESTER:
+		if (attr->qp_type == IBV_QPT_RC)
+			size = align_hw_size(MANA_PAGE_SIZE);
+		break;
+	case USER_RNIC_RECV_QUEUE_RESPONDER:
+		/* WQE must have at least one SGE */
+		sges = max(1U, attr->cap.max_recv_sge);
+		size = align_hw_size(attr->cap.max_recv_wr * get_wqe_size(sges));
+		break;
+	case USER_RNIC_SEND_QUEUE_MM:
+		sges = max(1U, attr->cap.max_send_sge) + 1;
+		size = align_hw_size(attr->cap.max_send_wr * get_large_wqe_size(sges));
+		break;
+	default:
+		return 0;
 	}
 
-	size = align_hw_size(size);
-
 	if (attr->qp_type == IBV_QPT_RC && type == USER_RNIC_SEND_QUEUE_REQUESTER)
 		size += sizeof(struct mana_ib_rollback_shared_mem);
 
 	return size;
 }
 
-static struct ibv_qp *mana_create_qp_rnic(struct ibv_pd *ibpd,
-					  struct ibv_qp_init_attr *attr)
+static int mana_create_cmd_qp_rc(struct mana_qp *qp, struct ibv_pd *ibpd,
+				 struct ibv_qp_init_attr *attr)
 {
-	struct mana_context *ctx = to_mctx(ibpd->context);
 	struct mana_ib_create_rc_qp_resp *qp_resp_drv;
 	struct mana_create_rc_qp_resp qp_resp = {};
 	struct mana_ib_create_rc_qp *qp_cmd_drv;
 	struct mana_create_rc_qp qp_cmd = {};
+	int ret, i;
+
+	qp_cmd_drv = &qp_cmd.drv_payload;
+	qp_resp_drv = &qp_resp.drv_payload;
+
+	for (i = 0; i < USER_RNIC_QUEUE_TYPE_MAX; ++i) {
+		if (i == USER_RNIC_SEND_QUEUE_MM)
+			continue;
+		qp_cmd_drv->queue_buf[i] = (uintptr_t)qp->rnic_qp.queues[i].buffer;
+		qp_cmd_drv->queue_size[i] = qp->rnic_qp.queues[i].size;
+	}
+
+	ret = ibv_cmd_create_qp(ibpd, &qp->ibqp.qp, attr, &qp_cmd.ibv_cmd,
+				sizeof(qp_cmd), &qp_resp.ibv_resp,
+				sizeof(qp_resp));
+	if (ret) {
+		verbs_err(verbs_get_ctx(ibpd->context), "Create QP failed\n");
+		return ret;
+	}
+
+	for (i = 0; i < USER_RNIC_QUEUE_TYPE_MAX; ++i) {
+		if (i == USER_RNIC_SEND_QUEUE_MM)
+			continue;
+		qp->rnic_qp.queues[i].id = qp_resp_drv->queue_id[i];
+	}
+
+	return 0;
+}
+
+enum {
+	MANA_UC_UDATA_SQR = 0,
+	MANA_UC_UDATA_RQR = 1,
+	MANA_UC_UDATA_SMQ = 2,
+};
+
+static int mana_create_cmd_qp_uc(struct mana_qp *qp, struct ibv_pd *ibpd,
+				 struct ibv_qp_init_attr *attr)
+{
+	struct mana_ib_create_uc_qp_resp *qp_resp_drv;
+	struct mana_create_uc_qp_resp qp_resp = {};
+	struct mana_ib_create_uc_qp *qp_cmd_drv;
+	struct mana_create_uc_qp qp_cmd = {};
+	int ret;
+
+	qp_cmd_drv = &qp_cmd.drv_payload;
+	qp_resp_drv = &qp_resp.drv_payload;
+
+	qp_cmd_drv->queue_buf[MANA_UC_UDATA_SQR] =
+		(uintptr_t)qp->rnic_qp.queues[USER_RNIC_SEND_QUEUE_REQUESTER].buffer;
+	qp_cmd_drv->queue_size[MANA_UC_UDATA_SQR] =
+		qp->rnic_qp.queues[USER_RNIC_SEND_QUEUE_REQUESTER].size;
+
+	qp_cmd_drv->queue_buf[MANA_UC_UDATA_RQR] =
+		(uintptr_t)qp->rnic_qp.queues[USER_RNIC_RECV_QUEUE_RESPONDER].buffer;
+	qp_cmd_drv->queue_size[MANA_UC_UDATA_RQR] =
+		qp->rnic_qp.queues[USER_RNIC_RECV_QUEUE_RESPONDER].size;
+
+	qp_cmd_drv->queue_buf[MANA_UC_UDATA_SMQ] =
+		(uintptr_t)qp->rnic_qp.queues[USER_RNIC_SEND_QUEUE_MM].buffer;
+	qp_cmd_drv->queue_size[MANA_UC_UDATA_SMQ] =
+		qp->rnic_qp.queues[USER_RNIC_SEND_QUEUE_MM].size;
+
+	ret = ibv_cmd_create_qp(ibpd, &qp->ibqp.qp, attr, &qp_cmd.ibv_cmd,
+				sizeof(qp_cmd), &qp_resp.ibv_resp,
+				sizeof(qp_resp));
+	if (ret) {
+		verbs_err(verbs_get_ctx(ibpd->context), "Create QP failed\n");
+		return ret;
+	}
+
+	qp->rnic_qp.queues[USER_RNIC_SEND_QUEUE_REQUESTER].id =
+		qp_resp_drv->queue_id[MANA_UC_UDATA_SQR];
+	qp->rnic_qp.queues[USER_RNIC_RECV_QUEUE_RESPONDER].id =
+		qp_resp_drv->queue_id[MANA_UC_UDATA_RQR];
+	qp->rnic_qp.queues[USER_RNIC_SEND_QUEUE_MM].id =
+		qp_resp_drv->queue_id[MANA_UC_UDATA_SMQ];
+
+	return 0;
+}
+
+static int mana_create_cmd_qp(struct mana_qp *qp, struct ibv_pd *ibpd,
+			      struct ibv_qp_init_attr *attr)
+{
+	if (attr->qp_type == IBV_QPT_RC)
+		return mana_create_cmd_qp_rc(qp, ibpd, attr);
+	else if (attr->qp_type == IBV_QPT_UC)
+		return mana_create_cmd_qp_uc(qp, ibpd, attr);
+	else
+		return -EOPNOTSUPP;
+}
+
+static struct ibv_qp *mana_create_qp_rnic(struct ibv_pd *ibpd,
+					  struct ibv_qp_init_attr *attr)
+{
+	struct mana_context *ctx = to_mctx(ibpd->context);
 	struct mana_qp *qp;
 	int ret, i;
 
 	qp = calloc(1, sizeof(*qp));
 	if (!qp)
 		return NULL;
 
-	qp_cmd_drv = &qp_cmd.drv_payload;
-	qp_resp_drv = &qp_resp.drv_payload;
-
 	pthread_spin_init(&qp->sq_lock, PTHREAD_PROCESS_PRIVATE);
 	pthread_spin_init(&qp->rq_lock, PTHREAD_PROCESS_PRIVATE);
 	qp->sq_sig_all = attr->sq_sig_all;
@@ -291,29 +387,18 @@ static struct ibv_qp *mana_create_qp_rnic(struct ibv_pd *ibpd,
 
 		if (qp->rnic_qp.queues[i].size != 0 && !qp->rnic_qp.queues[i].buffer) {
 			verbs_err(verbs_get_ctx(ibpd->context),
-				  "Failed to allocate memory for RC queue %d\n", i);
+				  "Failed to allocate memory for queue %d\n", i);
 			errno = ENOMEM;
 			goto destroy_queues;
 		}
-
-		qp_cmd_drv->queue_buf[i] = (uintptr_t)qp->rnic_qp.queues[i].buffer;
-		qp_cmd_drv->queue_size[i] = qp->rnic_qp.queues[i].size;
 	}
 
-	mana_ib_init_rb_shmem(qp);
-
-	ret = ibv_cmd_create_qp(ibpd, &qp->ibqp.qp, attr, &qp_cmd.ibv_cmd,
-				sizeof(qp_cmd), &qp_resp.ibv_resp,
-				sizeof(qp_resp));
+	ret = mana_create_cmd_qp(qp, ibpd, attr);
 	if (ret) {
-		verbs_err(verbs_get_ctx(ibpd->context), "Create QP failed\n");
 		errno = ret;
-		goto free_rb;
+		goto destroy_queues;
 	}
 
-	for (i = 0; i < USER_RNIC_QUEUE_TYPE_MAX; ++i)
-		qp->rnic_qp.queues[i].id = qp_resp_drv->queue_id[i];
-
 	qp->ibqp.qp.qp_num = qp->rnic_qp.queues[USER_RNIC_RECV_QUEUE_RESPONDER].id;
 
 	ret = mana_store_qp(ctx, qp);
@@ -322,12 +407,12 @@ static struct ibv_qp *mana_create_qp_rnic(struct ibv_pd *ibpd,
 		goto destroy_qp;
 	}
 
+	mana_ib_init_rb_shmem(qp);
+
 	return &qp->ibqp.qp;
 
 destroy_qp:
 	ibv_cmd_destroy_qp(&qp->ibqp.qp);
-free_rb:
-	mana_ib_deinit_rb_shmem(qp);
 destroy_queues:
 	while (i-- > 0)
 		mana_dealloc_mem(qp->rnic_qp.queues[i].buffer, qp->rnic_qp.queues[i].size);
@@ -346,6 +431,7 @@ struct ibv_qp *mana_create_qp(struct ibv_pd *ibpd,
 	case IBV_QPT_RAW_PACKET:
 		return mana_create_qp_raw(ibpd, attr);
 	case IBV_QPT_RC:
+	case IBV_QPT_UC:
 		return mana_create_qp_rnic(ibpd, attr);
 	default:
 		verbs_err(verbs_get_ctx(ibpd->context),
@@ -382,7 +468,8 @@ static void mana_ib_modify_rnic_qp(struct mana_qp *qp, struct ibv_qp_attr *attr,
 			if (attr_mask & IBV_QP_SQ_PSN) {
 				qp->sq_ssn = 1;
 				qp->sq_psn = attr->sq_psn;
-				gdma_arm_normal_cqe(mana_ib_get_rreq(qp), attr->sq_psn);
+				if (qp->ibqp.qp.qp_type == IBV_QPT_RC)
+					gdma_arm_normal_cqe(mana_ib_get_rreq(qp), attr->sq_psn);
 			}
 			break;
 		default:
@@ -397,7 +484,7 @@ int mana_modify_qp(struct ibv_qp *ibqp, struct ibv_qp_attr *attr, int attr_mask)
 	struct ibv_modify_qp cmd = {};
 	int err;
 
-	if (ibqp->qp_type != IBV_QPT_RC)
+	if (ibqp->qp_type != IBV_QPT_RC && ibqp->qp_type != IBV_QPT_UC)
 		return EOPNOTSUPP;
 
 	pthread_spin_lock(&qp->sq_lock);
@@ -443,7 +530,7 @@ int mana_destroy_qp(struct ibv_qp *ibqp)
 	struct mana_context *ctx = to_mctx(ibqp->context);
 	int ret, i;
 
-	if (ibqp->qp_type == IBV_QPT_RC) {
+	if (ibqp->qp_type == IBV_QPT_RC || ibqp->qp_type == IBV_QPT_UC) {
 		mana_remove_qp(ctx, qp);
 		mana_drain_cqes(qp);
 	}
@@ -459,6 +546,7 @@ int mana_destroy_qp(struct ibv_qp *ibqp)
 		ctx->extern_alloc.free(qp->raw_qp.send_buf, ctx->extern_alloc.data);
 		break;
 	case IBV_QPT_RC:
+	case IBV_QPT_UC:
 		pthread_spin_destroy(&qp->sq_lock);
 		pthread_spin_destroy(&qp->rq_lock);
 		destroy_shadow_queue(&qp->shadow_sq);

providers/mana/rollback.h

@@ -39,6 +39,8 @@ static inline struct mana_ib_rollback_shared_mem
 
 static inline void mana_ib_init_rb_shmem(struct mana_qp *qp)
 {
+	if (qp->ibqp.qp.qp_type != IBV_QPT_RC)
+		return;
 	// take some bytes for rollback memory
 	struct mana_gdma_queue *req_sq =
 		&qp->rnic_qp.queues[USER_RNIC_SEND_QUEUE_REQUESTER];
@@ -54,6 +56,8 @@ static inline void mana_ib_init_rb_shmem(struct mana_qp *qp)
 
 static inline void mana_ib_deinit_rb_shmem(struct mana_qp *qp)
 {
+	if (qp->ibqp.qp.qp_type != IBV_QPT_RC)
+		return;
 	// return back bytes for rollback memory
 	struct mana_gdma_queue *req_sq =
 		&qp->rnic_qp.queues[USER_RNIC_SEND_QUEUE_REQUESTER];
@@ -62,6 +66,9 @@ static inline void mana_ib_deinit_rb_shmem(struct mana_qp *qp)
 
 static inline void mana_ib_reset_rb_shmem(struct mana_qp *qp)
 {
+	if (qp->ibqp.qp.qp_type != IBV_QPT_RC)
+		return;
+
 	struct mana_ib_rollback_shared_mem *rb_shmem =
 		mana_ib_get_rollback_sh_mem(qp);
 
@@ -71,6 +78,9 @@ static inline void mana_ib_reset_rb_shmem(struct mana_qp *qp)
 
 static inline void mana_ib_update_shared_mem_right_offset(struct mana_qp *qp, uint32_t offset_in_bu)
 {
+	if (qp->ibqp.qp.qp_type != IBV_QPT_RC)
+		return;
+
 	struct mana_ib_rollback_shared_mem *rb_shmem =
 			mana_ib_get_rollback_sh_mem(qp);
 

providers/mana/wr.c

@@ -191,6 +191,7 @@ int mana_post_recv(struct ibv_qp *ibqp, struct ibv_recv_wr *wr,
 {
 	switch (ibqp->qp_type) {
 	case IBV_QPT_RC:
+	case IBV_QPT_UC:
 		return mana_ib_post_recv(ibqp, wr, bad);
 	default:
 		verbs_err(verbs_get_ctx(ibqp->context), "QPT not supported %d\n", ibqp->qp_type);
@@ -350,7 +351,7 @@ mana_ib_post_send_request(struct mana_qp *qp, struct ibv_send_wr *wr,
 			       &send_oob, oob_sge, num_sge, MTU_SIZE(qp->mtu), flags, &gdma_wqe);
 	if (ret) {
 		verbs_err(verbs_get_ctx(qp->ibqp.qp.context),
-			  "rc post send error, ret %d\n", ret);
+			  "post send error, ret %d\n", ret);
 		goto cleanup;
 	}
 
@@ -430,6 +431,7 @@ int mana_post_send(struct ibv_qp *ibqp, struct ibv_send_wr *wr,
 {
 	switch (ibqp->qp_type) {
 	case IBV_QPT_RC:
+	case IBV_QPT_UC:
 		return mana_ib_post_send(ibqp, wr, bad);
 	default:
 		verbs_err(verbs_get_ctx(ibqp->context), "QPT not supported %d\n", ibqp->qp_type);