Merge pull request #1739 from amzn/processing-hints

EFA work request processing hints

Author: rleon

debian/ibverbs-providers.symbols

@@ -187,6 +187,7 @@ libefa.so.1 ibverbs-providers #MINVER#
  EFA_1.3@EFA_1.3 50
  EFA_1.4@EFA_1.4 59
  EFA_1.5@EFA_1.5 63
+ EFA_1.6@EFA_1.6 64
  efadv_create_driver_qp@EFA_1.0 24
  efadv_create_qp_ex@EFA_1.1 26
  efadv_query_device@EFA_1.1 26
@@ -198,6 +199,7 @@ libefa.so.1 ibverbs-providers #MINVER#
  efadv_query_cq@EFA_1.4 59
  efadv_get_max_sq_depth@EFA_1.5 63
  efadv_get_max_rq_depth@EFA_1.5 63
+ efadv_qp_from_ibv_qp_ex@EFA_1.6 64
 libhns.so.1 ibverbs-providers #MINVER#
 * Build-Depends-Package: libibverbs-dev
  HNS_1.0@HNS_1.0 51

providers/efa/CMakeLists.txt

@@ -3,7 +3,7 @@ if (ENABLE_LTTNG AND LTTNGUST_FOUND)
 endif()
 
 rdma_shared_provider(efa libefa.map
-	1 1.5.${PACKAGE_VERSION}
+	1 1.6.${PACKAGE_VERSION}
 	${TRACE_FILE}
 	efa.c
 	verbs.c

providers/efa/efa.h

@@ -175,6 +175,7 @@ struct efa_sq {
 
 struct efa_qp {
 	struct verbs_qp verbs_qp;
+	struct efadv_qp dv_qp;
 	struct efa_sq sq;
 	struct efa_rq rq;
 	int page_size;
@@ -239,6 +240,11 @@ static inline struct efa_qp *to_efa_qp_ex(struct ibv_qp_ex *ibvqpx)
 	return container_of(ibvqpx, struct efa_qp, verbs_qp.qp_ex);
 }
 
+static inline struct efa_qp *efadv_qp_to_efa_qp(struct efadv_qp *efadv_qp)
+{
+	return container_of(efadv_qp, struct efa_qp, dv_qp);
+}
+
 static inline struct efa_ah *to_efa_ah(struct ibv_ah *ibvah)
 {
 	return container_of(ibvah, struct efa_ah, ibvah);

providers/efa/efa_io_defs.h

@@ -65,6 +65,11 @@ enum efa_io_comp_status {
 	EFA_IO_COMP_STATUS_REMOTE_ERROR_FEATURE_MISMATCH = 18,
 };
 
+enum efa_io_processing_hint {
+	/* Optimize for throughput */
+	EFA_IO_PROCESSING_HINT_BURST_PPS_SENSITIVE  = 1 << 0,
+};
+
 struct efa_io_tx_meta_desc {
 	/* Verbs-generated Request ID */
 	uint16_t req_id;
@@ -114,7 +119,15 @@ struct efa_io_tx_meta_desc {
 
 	uint16_t ah;
 
-	uint16_t reserved;
+	/*
+	 * control flags
+	 * 1:0 : processing_hints - Bitmask of enum
+	 *    efa_io_processing_hint
+	 * 7:2 : reserved - MBZ
+	 */
+	uint8_t ctrl3;
+
+	uint8_t reserved;
 
 	/* Queue key */
 	uint32_t qkey;
@@ -327,6 +340,7 @@ struct efa_io_rx_cdesc_ex {
 #define EFA_IO_TX_META_DESC_FIRST_MASK                      BIT(2)
 #define EFA_IO_TX_META_DESC_LAST_MASK                       BIT(3)
 #define EFA_IO_TX_META_DESC_COMP_REQ_MASK                   BIT(4)
+#define EFA_IO_TX_META_DESC_PROCESSING_HINTS_MASK           GENMASK(1, 0)
 
 /* tx_buf_desc */
 #define EFA_IO_TX_BUF_DESC_LKEY_MASK                        GENMASK(23, 0)

providers/efa/efadv.h

@@ -87,12 +87,17 @@ enum {
 	EFADV_QP_FLAGS_INLINE_WRITE = 1 << 1,
 };
 
+enum {
+	EFADV_WR_EX_WITH_PROCESSING_HINTS = 1 << 0,
+};
+
 struct efadv_qp_init_attr {
 	uint64_t comp_mask;
 	uint32_t driver_qp_type;
 	uint16_t flags;
 	uint8_t sl;
 	uint8_t reserved;
+	uint64_t wr_flags;
 };
 
 struct ibv_qp *efadv_create_qp_ex(struct ibv_context *ibvctx,
@@ -113,12 +118,22 @@ struct efadv_wq_attr {
 int efadv_query_qp_wqs(struct ibv_qp *ibvqp, struct efadv_wq_attr *sq_attr,
 		       struct efadv_wq_attr *rq_attr, uint32_t inlen);
 
-struct efadv_cq {
+enum efadv_wr_processing_hint {
+	EFADV_WR_PROCESSING_HINT_BURST_PPS_SENSITIVE = 1 << 0,
+};
+
+struct efadv_qp {
 	uint64_t comp_mask;
-	int (*wc_read_sgid)(struct efadv_cq *efadv_cq, union ibv_gid *sgid);
-	bool (*wc_is_unsolicited)(struct efadv_cq *efadv_cq);
+	void (*wr_set_processing_hints)(struct efadv_qp *efadv_qp, uint32_t hints);
 };
 
+struct efadv_qp *efadv_qp_from_ibv_qp_ex(struct ibv_qp_ex *ibvqpx);
+
+static inline void efadv_wr_set_processing_hints(struct efadv_qp *efadv_qp, uint32_t hints)
+{
+	efadv_qp->wr_set_processing_hints(efadv_qp, hints);
+}
+
 enum {
 	EFADV_WC_EX_WITH_SGID = 1 << 0,
 	EFADV_WC_EX_WITH_IS_UNSOLICITED = 1 << 1,
@@ -156,6 +171,12 @@ struct efadv_cq_attr {
 
 int efadv_query_cq(struct ibv_cq *ibvcq, struct efadv_cq_attr *attr, uint32_t inlen);
 
+struct efadv_cq {
+	uint64_t comp_mask;
+	int (*wc_read_sgid)(struct efadv_cq *efadv_cq, union ibv_gid *sgid);
+	bool (*wc_is_unsolicited)(struct efadv_cq *efadv_cq);
+};
+
 struct efadv_cq *efadv_cq_from_ibv_cq_ex(struct ibv_cq_ex *ibvcqx);
 
 static inline int efadv_wc_read_sgid(struct efadv_cq *efadv_cq,

providers/efa/libefa.map

@@ -35,3 +35,8 @@ EFA_1.5 {
 		efadv_get_max_sq_depth;
 		efadv_get_max_rq_depth;
 } EFA_1.4;
+
+EFA_1.6 {
+	global:
+		efadv_qp_from_ibv_qp_ex;
+} EFA_1.5;

providers/efa/man/CMakeLists.txt

@@ -7,4 +7,5 @@ rdma_man_pages(
   efadv_query_device.3.md
   efadv_query_mr.3.md
   efadv_query_qp_wqs.3.md
+  efadv_wr_set_processing_hints.3.md
 )

providers/efa/man/efadv_create_qp_ex.3.md

@@ -46,7 +46,8 @@ struct efadv_qp_init_attr {
 	uint32_t driver_qp_type;
 	uint16_t flags;
 	uint8_t sl;
-	uint8_t reserved[1];
+	uint8_t reserved;
+	uint64_t wr_flags;
 };
 ```
 
@@ -74,6 +75,15 @@ struct efadv_qp_init_attr {
 *sl*
 :	Service Level - 0 value implies default level.
 
+*wr_flags*
+:	A bitwise OR of the values described below. Controls which
+	EFA-specific work request setter functions are available on
+	the QP. Use **efadv_qp_from_ibv_qp_ex()** to get the
+	efadv_qp for accessing the work request setter interface.
+
+	EFADV_WR_EX_WITH_PROCESSING_HINTS:
+		Enable **efadv_wr_set_processing_hints()** on this QP.
+
 # RETURN VALUE
 
 efadv_create_qp_ex() returns a pointer to the created QP, or NULL if the request fails.

providers/efa/man/efadv_wr_set_processing_hints.3.md

@@ -0,0 +1,62 @@
+---
+layout: page
+title: EFADV_WR_SET_PROCESSING_HINTS
+section: 3
+tagline: Verbs
+date: 2026-05-05
+header: "EFA Direct Verbs Manual"
+footer: efa
+---
+
+# NAME
+
+efadv_wr_set_processing_hints - Set processing hints on the current
+work request
+
+# SYNOPSIS
+
+```c
+#include <infiniband/efadv.h>
+
+static inline void efadv_wr_set_processing_hints(
+	struct efadv_qp *efadv_qp,
+	uint32_t hints);
+```
+
+# DESCRIPTION
+
+**efadv_wr_set_processing_hints()** sets processing hints on the
+current work request being built. Hints allow the application to
+communicate intended usage patterns to the device, which may use them
+to optimize processing.
+
+This function is a work request setter and must be called after the
+work request opcode function (e.g. **ibv_wr_send()**) and before
+**ibv_wr_complete()** or the next work request opcode call.
+
+Use **efadv_qp_from_ibv_qp_ex()** to get the efadv_qp for accessing
+this interface.
+
+The QP must be created with **EFADV_WR_EX_WITH_PROCESSING_HINTS** set
+in *efadv_qp_init_attr.wr_flags* to use this function.
+
+The *hints* argument is a bitmask of **efadv_wr_processing_hint**
+values:
+
+```c
+enum efadv_wr_processing_hint {
+	EFADV_WR_PROCESSING_HINT_BURST_PPS_SENSITIVE = 1 << 0,
+};
+```
+
+*EFADV_WR_PROCESSING_HINT_BURST_PPS_SENSITIVE*
+:	Optimize for throughput in bursty, packet-rate sensitive
+	workloads.
+
+# SEE ALSO
+
+**efadv**(7), **efadv_create_qp_ex**(3), **ibv_wr_start**(3)
+
+# AUTHORS
+
+Michael Margolin <mrgolin@amazon.com>

providers/efa/verbs.c

@@ -1777,8 +1777,9 @@ static void efa_unlock_cqs(struct ibv_qp *ibvqp)
 	}
 }
 
-static void efa_qp_fill_wr_pfns(struct ibv_qp_ex *ibvqpx,
+static void efa_qp_fill_wr_pfns(struct efa_qp *qp,
 				struct ibv_qp_init_attr_ex *attr_ex,
+				struct efadv_qp_init_attr *efa_attr,
 				uint16_t wqe_size);
 
 static int efa_check_qp_attr(struct efa_context *ctx,
@@ -1853,6 +1854,12 @@ static int efa_check_qp_attr(struct efa_context *ctx,
 		}
 	}
 
+	if (!check_comp_mask(efa_attr->wr_flags, EFADV_WR_EX_WITH_PROCESSING_HINTS)) {
+		verbs_err(&ctx->ibvctx, "Unsupported wr_flags[%" PRIx64 "]\n",
+			  efa_attr->wr_flags);
+		return EOPNOTSUPP;
+	}
+
 	if (!attr->recv_cq || !attr->send_cq) {
 		verbs_err(&ctx->ibvctx, "Send/Receive CQ not provided\n");
 		return EINVAL;
@@ -1983,7 +1990,7 @@ static struct ibv_qp *create_qp(struct ibv_context *ibvctx,
 	pthread_spin_unlock(&ctx->qp_table_lock);
 
 	if (attr->comp_mask & IBV_QP_INIT_ATTR_SEND_OPS_FLAGS) {
-		efa_qp_fill_wr_pfns(&qp->verbs_qp.qp_ex, attr, qp->sq.wqe_size);
+		efa_qp_fill_wr_pfns(qp, attr, efa_attr, qp->sq.wqe_size);
 		qp->verbs_qp.comp_mask |= VERBS_QP_EX;
 	}
 
@@ -2185,6 +2192,13 @@ int efa_query_qp_data_in_order(struct ibv_qp *ibvqp, enum ibv_wr_opcode op,
 	return caps;
 }
 
+struct efadv_qp *efadv_qp_from_ibv_qp_ex(struct ibv_qp_ex *ibvqpx)
+{
+	struct efa_qp *qp = to_efa_qp_ex(ibvqpx);
+
+	return &qp->dv_qp;
+}
+
 int efa_destroy_qp(struct ibv_qp *ibvqp)
 {
 	struct efa_context *ctx = to_efa_context(ibvqp->context);
@@ -2912,6 +2926,20 @@ static void efa_send_wr_set_addr(struct ibv_qp_ex *ibvqpx,
 			efa_wqe_get_data_length(qp->sq));
 }
 
+static void efa_send_wr_set_processing_hints(struct efadv_qp *efadv_qp, uint32_t hints)
+{
+	struct efa_qp *qp = efadv_qp_to_efa_qp(efadv_qp);
+	uint8_t wqe_hints = 0;
+
+	if (unlikely(qp->wr_session_err))
+		return;
+
+	if (hints & EFADV_WR_PROCESSING_HINT_BURST_PPS_SENSITIVE)
+		wqe_hints |= EFA_IO_PROCESSING_HINT_BURST_PPS_SENSITIVE;
+
+	EFA_SET(&qp->sq.curr_tx_wqe.md->ctrl3, EFA_IO_TX_META_DESC_PROCESSING_HINTS, wqe_hints);
+}
+
 static void efa_send_wr_start(struct ibv_qp_ex *ibvqpx)
 {
 	struct efa_qp *qp = to_efa_qp_ex(ibvqpx);
@@ -3012,11 +3040,13 @@ static void efa_send_wr_abort(struct ibv_qp_ex *ibvqpx)
 		pthread_spin_unlock(&sq->wq.wqlock);
 }
 
-static void efa_qp_fill_wr_pfns(struct ibv_qp_ex *ibvqpx,
+static void efa_qp_fill_wr_pfns(struct efa_qp *qp,
 				struct ibv_qp_init_attr_ex *attr_ex,
+				struct efadv_qp_init_attr *efa_attr,
 				uint16_t wqe_size)
 {
 	bool use_64 = wqe_size == EFA_IO_TX_DESC_SIZE_64;
+	struct ibv_qp_ex *ibvqpx = &qp->verbs_qp.qp_ex;
 
 	ibvqpx->wr_start = efa_send_wr_start;
 	ibvqpx->wr_complete = efa_send_wr_complete;
@@ -3045,6 +3075,9 @@ static void efa_qp_fill_wr_pfns(struct ibv_qp_ex *ibvqpx,
 	ibvqpx->wr_set_sge = efa_send_wr_set_sge;
 	ibvqpx->wr_set_sge_list = efa_send_wr_set_sge_list;
 	ibvqpx->wr_set_ud_addr = efa_send_wr_set_addr;
+
+	if (efa_attr->wr_flags & EFADV_WR_EX_WITH_PROCESSING_HINTS)
+		qp->dv_qp.wr_set_processing_hints = efa_send_wr_set_processing_hints;
 }
 
 static int efa_post_recv_validate(struct efa_qp *qp, struct ibv_recv_wr *wr)