efa: Add Completion Counters support

Implement completion counters for the EFA provider. Each counter
object has two event counters (success and error) with set, inc, and
read operations. Counter values are accessed directly from userspace
through mapped memory.

Add efadv_create_comp_cntr() for extended creation with external
memory options. Buffer memory is described to the kernel via
efa_uverbs_buffer_desc struct passed as a driver-specific ioctl
attribute. Three modes are supported per counter: user-supplied VA,
user-supplied dmabuf, or provider-internal memory.

Signed-off-by: Michael Margolin <mrgolin@amazon.com>

Author: mrgolin

debian/ibverbs-providers.symbols

@@ -200,6 +200,7 @@ libefa.so.1 ibverbs-providers #MINVER#
  efadv_get_max_sq_depth@EFA_1.5 63
  efadv_get_max_rq_depth@EFA_1.5 63
  efadv_qp_from_ibv_qp_ex@EFA_1.6 64
+ efadv_create_comp_cntr@EFA_1.6 64
 libhns.so.1 ibverbs-providers #MINVER#
 * Build-Depends-Package: libibverbs-dev
  HNS_1.0@HNS_1.0 51

providers/efa/efa.c

@@ -33,6 +33,7 @@ static const struct verbs_context_ops efa_ctx_ops = {
 	.create_ah = efa_create_ah,
 	.create_cq = efa_create_cq,
 	.create_cq_ex = efa_create_cq_ex,
+	.create_comp_cntr = efa_create_comp_cntr,
 	.create_qp = efa_create_qp,
 	.create_qp_ex = efa_create_qp_ex,
 	.cq_event = efa_cq_event,
@@ -41,18 +42,26 @@ static const struct verbs_context_ops efa_ctx_ops = {
 	.dereg_mr = efa_dereg_mr,
 	.destroy_ah = efa_destroy_ah,
 	.destroy_cq = efa_destroy_cq,
+	.destroy_comp_cntr = efa_destroy_comp_cntr,
 	.destroy_qp = efa_destroy_qp,
+	.inc_comp_cntr = efa_inc_comp_cntr,
+	.inc_err_comp_cntr = efa_inc_err_comp_cntr,
 	.modify_qp = efa_modify_qp,
 	.poll_cq = efa_poll_cq,
 	.post_recv = efa_post_recv,
 	.post_send = efa_post_send,
+	.qp_attach_comp_cntr = efa_qp_attach_comp_cntr,
 	.query_device_ex = efa_query_device_ex,
 	.query_port = efa_query_port,
 	.query_qp = efa_query_qp,
 	.query_qp_data_in_order = efa_query_qp_data_in_order,
+	.read_comp_cntr = efa_read_comp_cntr,
+	.read_err_comp_cntr = efa_read_err_comp_cntr,
 	.reg_dmabuf_mr = efa_reg_dmabuf_mr,
 	.reg_mr = efa_reg_mr,
 	.req_notify_cq = efa_arm_cq,
+	.set_comp_cntr = efa_set_comp_cntr,
+	.set_err_comp_cntr = efa_set_err_comp_cntr,
 	.free_context = efa_free_context,
 };
 

providers/efa/efa.h

@@ -105,6 +105,14 @@ struct efa_cq {
 	struct efa_sub_cq sub_cq_arr[];
 };
 
+struct efa_comp_cntr {
+	struct ibv_comp_cntr ibv_comp_cntr;
+	uint64_t comp_val __attribute__((aligned(8)));
+	uint64_t err_val __attribute__((aligned(8)));
+	uint64_t *comp_ptr;
+	uint64_t *err_ptr;
+};
+
 struct efa_wq {
 	uint64_t *wrid;
 	/* wrid_idx_pool: Pool of free indexes in the wrid array, used to select the
@@ -230,6 +238,11 @@ static inline struct efa_cq *efadv_cq_to_efa_cq(struct efadv_cq *efadv_cq)
 	return container_of(efadv_cq, struct efa_cq, dv_cq);
 }
 
+static inline struct efa_comp_cntr *to_efa_comp_cntr(struct ibv_comp_cntr *ibvcc)
+{
+	return container_of(ibvcc, struct efa_comp_cntr, ibv_comp_cntr);
+}
+
 static inline struct efa_qp *to_efa_qp(struct ibv_qp *ibvqp)
 {
 	return container_of(ibvqp, struct efa_qp, verbs_qp.qp);

providers/efa/efadv.h

@@ -23,6 +23,7 @@ enum {
 	EFADV_DEVICE_ATTR_CAPS_RDMA_WRITE = 1 << 3,
 	EFADV_DEVICE_ATTR_CAPS_UNSOLICITED_WRITE_RECV = 1 << 4,
 	EFADV_DEVICE_ATTR_CAPS_CQ_WITH_EXT_MEM_DMABUF = 1 << 5,
+	EFADV_DEVICE_ATTR_CAPS_COMP_CNTR = 1 << 6,
 };
 
 struct efadv_device_attr {
@@ -190,6 +191,40 @@ static inline bool efadv_wc_is_unsolicited(struct efadv_cq *efadv_cq)
 	return efadv_cq->wc_is_unsolicited(efadv_cq);
 }
 
+enum {
+	EFADV_MEMORY_LOCATION_VA,
+	EFADV_MEMORY_LOCATION_DMABUF,
+};
+
+struct efadv_memory_location {
+	uint8_t *ptr;
+	struct {
+		uint64_t offset;
+		int32_t fd;
+		uint32_t reserved;
+	} dmabuf;
+	uint8_t type;
+	uint8_t reserved[7];
+};
+
+enum {
+	EFADV_COMP_CNTR_INIT_WITH_COMP_EXTERNAL_MEM = 1 << 0,
+	EFADV_COMP_CNTR_INIT_WITH_ERR_EXTERNAL_MEM = 1 << 1,
+};
+
+struct efadv_comp_cntr_init_attr {
+	uint64_t comp_mask;
+	uint32_t flags;
+	uint32_t reserved;
+	struct efadv_memory_location comp_cntr_ext_mem;
+	struct efadv_memory_location err_cntr_ext_mem;
+};
+
+struct ibv_comp_cntr *efadv_create_comp_cntr(struct ibv_context *ibvctx,
+					     struct ibv_comp_cntr_init_attr *attr,
+					     struct efadv_comp_cntr_init_attr *efa_attr,
+					     uint32_t inlen);
+
 enum {
 	EFADV_MR_ATTR_VALIDITY_RECV_IC_ID = 1 << 0,
 	EFADV_MR_ATTR_VALIDITY_RDMA_READ_IC_ID = 1 << 1,

providers/efa/libefa.map

@@ -39,4 +39,5 @@ EFA_1.5 {
 EFA_1.6 {
 	global:
 		efadv_qp_from_ibv_qp_ex;
+		efadv_create_comp_cntr;
 } EFA_1.5;

providers/efa/man/efadv_create_comp_cntr.3.md

@@ -0,0 +1,118 @@
+---
+layout: page
+title: EFADV_CREATE_COMP_CNTR
+section: 3
+tagline: Verbs
+date: 2026-04-27
+header: "EFA Direct Verbs Manual"
+footer: efa
+---
+
+# NAME
+
+efadv_create_comp_cntr - Create EFA specific Completion Counter
+
+# SYNOPSIS
+
+```c
+#include <infiniband/efadv.h>
+
+struct ibv_comp_cntr *efadv_create_comp_cntr(struct ibv_context *context,
+					     struct ibv_comp_cntr_init_attr *attr,
+					     struct efadv_comp_cntr_init_attr *efa_attr,
+					     uint32_t inlen);
+```
+
+# DESCRIPTION
+
+**efadv_create_comp_cntr()** creates a Completion Counter with EFA specific
+properties, such as external memory for the counter values.
+
+The argument *attr* is an ibv_comp_cntr_init_attr struct, as defined in
+<infiniband/verbs.h>.
+
+Compatibility is handled using the comp_mask and inlen fields.
+
+```c
+enum {
+	EFADV_MEMORY_LOCATION_VA,
+	EFADV_MEMORY_LOCATION_DMABUF,
+};
+
+struct efadv_memory_location {
+	uint8_t *ptr;
+	struct {
+		uint64_t offset;
+		int32_t fd;
+		uint32_t reserved;
+	} dmabuf;
+	uint8_t type;
+	uint8_t reserved[7];
+};
+
+struct efadv_comp_cntr_init_attr {
+	uint64_t comp_mask;
+	uint32_t flags;
+	uint32_t reserved;
+	struct efadv_memory_location comp_cntr_ext_mem;
+	struct efadv_memory_location err_cntr_ext_mem;
+};
+```
+
+*inlen*
+:	In: Size of struct efadv_comp_cntr_init_attr.
+
+*comp_mask*
+:	Compatibility mask.
+
+*flags*
+:	A bitwise OR of the various values described below.
+
+	**EFADV_COMP_CNTR_INIT_WITH_COMP_EXTERNAL_MEM**:
+		Use application-provided memory for the completion count, as
+		described by *comp_cntr_ext_mem*.
+
+	**EFADV_COMP_CNTR_INIT_WITH_ERR_EXTERNAL_MEM**:
+		Use application-provided memory for the error count, as
+		described by *err_cntr_ext_mem*.
+
+*comp_cntr_ext_mem*
+:	Memory location for the completion count when using external memory.
+
+*err_cntr_ext_mem*
+:	Memory location for the error count when using external memory.
+
+## efadv_memory_location
+
+The external memory is described by an **efadv_memory_location** structure
+which supports two modes:
+
+*type*
+:	**EFADV_MEMORY_LOCATION_VA** for a virtual address, or
+	**EFADV_MEMORY_LOCATION_DMABUF** for a DMA-BUF reference.
+
+*ptr*
+:	Virtual address pointer. Required when type is
+	**EFADV_MEMORY_LOCATION_VA**. When type is
+	**EFADV_MEMORY_LOCATION_DMABUF**, may optionally be set to provide a
+	process-accessible mapping of the DMA-BUF memory.
+
+*dmabuf.fd*
+:	DMA-BUF file descriptor (used when type is
+	**EFADV_MEMORY_LOCATION_DMABUF**).
+
+*dmabuf.offset*
+:	Offset within the DMA-BUF.
+
+# RETURN VALUE
+
+efadv_create_comp_cntr() returns a pointer to the created ibv_comp_cntr, or
+NULL if the request fails.
+
+# SEE ALSO
+
+**efadv**(7), **ibv_create_comp_cntr**(3), **ibv_qp_attach_comp_cntr**(3)
+
+# AUTHORS
+
+Michael Margolin <mrgolin@amazon.com>

providers/efa/man/efadv_query_device.3.md

@@ -92,6 +92,9 @@ struct efadv_device_attr {
 		Indicates that creating CQs with external memory buffers by passing dmabuf is
 		supported.
 
+	EFADV_DEVICE_ATTR_CAPS_COMP_CNTR:
+		Completion counters are supported.
+
 *max_rdma_size*
 :	Maximum RDMA transfer size in bytes.