Skip to content

Commit 5220723

Browse files
committed
libibverbs: Add common infrastructure for Completion Counters verbs
Add kernel ioctl interface, provider ops, command helpers, and public API dispatch to enable providers to implement the Completion Counters verbs introduced in the previous commit. Reviewed-by: Daniel Kinsbursky <dkinsb@amazon.com> Signed-off-by: Michael Margolin <mrgolin@amazon.com>
1 parent 18dc896 commit 5220723

8 files changed

Lines changed: 313 additions & 1 deletion

File tree

debian/libibverbs1.symbols

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ libibverbs.so.1 libibverbs1 #MINVER#
1414
IBVERBS_1.14@IBVERBS_1.14 36
1515
IBVERBS_1.15@IBVERBS_1.15 59
1616
IBVERBS_1.16@IBVERBS_1.16 62
17+
IBVERBS_1.17@IBVERBS_1.17 64
1718
(symver)IBVERBS_PRIVATE_59 59
1819
_ibv_query_gid_ex@IBVERBS_1.11 32
1920
_ibv_query_gid_table@IBVERBS_1.11 32
@@ -36,6 +37,7 @@ libibverbs.so.1 libibverbs1 #MINVER#
3637
ibv_create_ah@IBVERBS_1.1 1.1.6
3738
ibv_create_ah_from_wc@IBVERBS_1.1 1.1.6
3839
ibv_create_comp_channel@IBVERBS_1.0 1.1.6
40+
ibv_create_comp_cntr@IBVERBS_1.17 64
3941
ibv_create_cq@IBVERBS_1.0 1.1.6
4042
ibv_create_cq@IBVERBS_1.1 1.1.6
4143
ibv_create_qp@IBVERBS_1.0 1.1.6
@@ -50,6 +52,7 @@ libibverbs.so.1 libibverbs1 #MINVER#
5052
ibv_destroy_ah@IBVERBS_1.0 1.1.6
5153
ibv_destroy_ah@IBVERBS_1.1 1.1.6
5254
ibv_destroy_comp_channel@IBVERBS_1.0 1.1.6
55+
ibv_destroy_comp_cntr@IBVERBS_1.17 64
5356
ibv_destroy_cq@IBVERBS_1.0 1.1.6
5457
ibv_destroy_cq@IBVERBS_1.1 1.1.6
5558
ibv_destroy_qp@IBVERBS_1.0 1.1.6
@@ -82,6 +85,8 @@ libibverbs.so.1 libibverbs1 #MINVER#
8285
ibv_import_dm@IBVERBS_1.13 35
8386
ibv_import_mr@IBVERBS_1.10 31
8487
ibv_import_pd@IBVERBS_1.10 31
88+
ibv_inc_comp_cntr@IBVERBS_1.17 64
89+
ibv_inc_err_comp_cntr@IBVERBS_1.17 64
8590
ibv_init_ah_from_wc@IBVERBS_1.1 1.1.6
8691
ibv_is_fork_initialized@IBVERBS_1.13 35
8792
ibv_modify_qp@IBVERBS_1.0 1.1.6
@@ -92,6 +97,7 @@ libibverbs.so.1 libibverbs1 #MINVER#
9297
ibv_open_device@IBVERBS_1.0 1.1.6
9398
ibv_open_device@IBVERBS_1.1 1.1.6
9499
ibv_port_state_str@IBVERBS_1.1 1.1.6
100+
ibv_qp_attach_comp_cntr@IBVERBS_1.17 64
95101
ibv_qp_to_qp_ex@IBVERBS_1.6 24
96102
ibv_query_device@IBVERBS_1.0 1.1.6
97103
ibv_query_device@IBVERBS_1.1 1.1.6
@@ -110,6 +116,8 @@ libibverbs.so.1 libibverbs1 #MINVER#
110116
ibv_query_srq@IBVERBS_1.1 1.1.6
111117
ibv_rate_to_mbps@IBVERBS_1.1 1.1.8
112118
ibv_rate_to_mult@IBVERBS_1.0 1.1.6
119+
ibv_read_comp_cntr@IBVERBS_1.17 64
120+
ibv_read_err_comp_cntr@IBVERBS_1.17 64
113121
ibv_read_sysfs_file@IBVERBS_1.0 1.1.6
114122
ibv_reg_dmabuf_mr@IBVERBS_1.12 34
115123
ibv_reg_mr@IBVERBS_1.0 1.1.6
@@ -122,7 +130,9 @@ libibverbs.so.1 libibverbs1 #MINVER#
122130
ibv_resize_cq@IBVERBS_1.0 1.1.6
123131
ibv_resize_cq@IBVERBS_1.1 1.1.6
124132
ibv_resolve_eth_l2_from_gid@IBVERBS_1.1 1.2.0
133+
ibv_set_comp_cntr@IBVERBS_1.17 64
125134
ibv_set_ece@IBVERBS_1.10 31
135+
ibv_set_err_comp_cntr@IBVERBS_1.17 64
126136
ibv_unimport_dm@IBVERBS_1.13 35
127137
ibv_unimport_mr@IBVERBS_1.10 31
128138
ibv_unimport_pd@IBVERBS_1.10 31

libibverbs/CMakeLists.txt

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,11 +21,12 @@ configure_file("libibverbs.map.in"
2121

2222
rdma_library(ibverbs "${CMAKE_CURRENT_BINARY_DIR}/libibverbs.map"
2323
# See Documentation/versioning.md
24-
1 1.16.${PACKAGE_VERSION}
24+
1 1.17.${PACKAGE_VERSION}
2525
all_providers.c
2626
cmd.c
2727
cmd_ah.c
2828
cmd_counters.c
29+
cmd_comp_cntr.c
2930
cmd_cq.c
3031
cmd_device.c
3132
cmd_dm.c

libibverbs/cmd_comp_cntr.c

Lines changed: 131 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,131 @@
1+
// SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
2+
/*
3+
* Copyright Amazon.com, Inc. or its affiliates. All rights reserved.
4+
*/
5+
6+
#include <infiniband/cmd_write.h>
7+
8+
int ibv_cmd_create_comp_cntr(struct ibv_context *context,
9+
struct ibv_comp_cntr *comp_cntr,
10+
struct ibv_command_buffer *link)
11+
{
12+
DECLARE_COMMAND_BUFFER_LINK(cmdb, UVERBS_OBJECT_COMP_CNTR,
13+
UVERBS_METHOD_COMP_CNTR_CREATE, 3, link);
14+
struct ib_uverbs_attr *handle;
15+
int ret;
16+
17+
comp_cntr->context = context;
18+
19+
handle = fill_attr_out_obj(cmdb, UVERBS_ATTR_CREATE_COMP_CNTR_HANDLE);
20+
fill_attr_out_ptr(cmdb, UVERBS_ATTR_CREATE_COMP_CNTR_RESP_COUNT_MAX_VALUE,
21+
&comp_cntr->comp_count_max_value);
22+
fill_attr_out_ptr(cmdb, UVERBS_ATTR_CREATE_COMP_CNTR_RESP_ERR_COUNT_MAX_VALUE,
23+
&comp_cntr->err_count_max_value);
24+
25+
ret = execute_ioctl(context, cmdb);
26+
if (ret)
27+
return errno;
28+
29+
comp_cntr->handle = read_attr_obj(UVERBS_ATTR_CREATE_COMP_CNTR_HANDLE, handle);
30+
return 0;
31+
}
32+
33+
int ibv_cmd_destroy_comp_cntr(struct ibv_comp_cntr *comp_cntr)
34+
{
35+
DECLARE_COMMAND_BUFFER(cmdb, UVERBS_OBJECT_COMP_CNTR, UVERBS_METHOD_COMP_CNTR_DESTROY, 1);
36+
int ret;
37+
38+
fill_attr_in_obj(cmdb, UVERBS_ATTR_DESTROY_COMP_CNTR_HANDLE, comp_cntr->handle);
39+
40+
ret = execute_ioctl(comp_cntr->context, cmdb);
41+
if (verbs_is_destroy_err(&ret))
42+
return ret;
43+
44+
return 0;
45+
}
46+
47+
static int ibv_icmd_modify_comp_cntr(struct ibv_comp_cntr *comp_cntr,
48+
uint8_t entry, uint8_t op, uint64_t value)
49+
{
50+
DECLARE_COMMAND_BUFFER(cmdb, UVERBS_OBJECT_COMP_CNTR, UVERBS_METHOD_COMP_CNTR_MODIFY, 4);
51+
52+
fill_attr_in_obj(cmdb, UVERBS_ATTR_MODIFY_COMP_CNTR_HANDLE, comp_cntr->handle);
53+
fill_attr_const_in(cmdb, UVERBS_ATTR_MODIFY_COMP_CNTR_ENTRY, entry);
54+
fill_attr_const_in(cmdb, UVERBS_ATTR_MODIFY_COMP_CNTR_OP, op);
55+
fill_attr_in_uint64(cmdb, UVERBS_ATTR_MODIFY_COMP_CNTR_VALUE, value);
56+
57+
return execute_ioctl(comp_cntr->context, cmdb);
58+
}
59+
60+
int ibv_cmd_set_comp_cntr(struct ibv_comp_cntr *comp_cntr, uint64_t value)
61+
{
62+
return ibv_icmd_modify_comp_cntr(comp_cntr, IB_UVERBS_COMP_CNTR_ENTRY_COMP,
63+
IB_UVERBS_COMP_CNTR_MODIFY_OP_SET, value);
64+
}
65+
66+
int ibv_cmd_set_err_comp_cntr(struct ibv_comp_cntr *comp_cntr, uint64_t value)
67+
{
68+
return ibv_icmd_modify_comp_cntr(comp_cntr, IB_UVERBS_COMP_CNTR_ENTRY_ERR,
69+
IB_UVERBS_COMP_CNTR_MODIFY_OP_SET, value);
70+
}
71+
72+
int ibv_cmd_inc_comp_cntr(struct ibv_comp_cntr *comp_cntr, uint64_t amount)
73+
{
74+
return ibv_icmd_modify_comp_cntr(comp_cntr, IB_UVERBS_COMP_CNTR_ENTRY_COMP,
75+
IB_UVERBS_COMP_CNTR_MODIFY_OP_INC, amount);
76+
}
77+
78+
int ibv_cmd_inc_err_comp_cntr(struct ibv_comp_cntr *comp_cntr, uint64_t amount)
79+
{
80+
return ibv_icmd_modify_comp_cntr(comp_cntr, IB_UVERBS_COMP_CNTR_ENTRY_ERR,
81+
IB_UVERBS_COMP_CNTR_MODIFY_OP_INC, amount);
82+
}
83+
84+
static int ibv_icmd_read_comp_cntr(struct ibv_comp_cntr *comp_cntr,
85+
uint8_t entry, uint64_t *value)
86+
{
87+
DECLARE_COMMAND_BUFFER(cmdb, UVERBS_OBJECT_COMP_CNTR, UVERBS_METHOD_COMP_CNTR_READ, 3);
88+
89+
fill_attr_in_obj(cmdb, UVERBS_ATTR_READ_COMP_CNTR_HANDLE, comp_cntr->handle);
90+
fill_attr_const_in(cmdb, UVERBS_ATTR_READ_COMP_CNTR_ENTRY, entry);
91+
fill_attr_out_ptr(cmdb, UVERBS_ATTR_READ_COMP_CNTR_RESP_VALUE, value);
92+
93+
return execute_ioctl(comp_cntr->context, cmdb);
94+
}
95+
96+
int ibv_cmd_read_comp_cntr(struct ibv_comp_cntr *comp_cntr, uint64_t *value)
97+
{
98+
return ibv_icmd_read_comp_cntr(comp_cntr, IB_UVERBS_COMP_CNTR_ENTRY_COMP, value);
99+
}
100+
101+
int ibv_cmd_read_err_comp_cntr(struct ibv_comp_cntr *comp_cntr, uint64_t *value)
102+
{
103+
return ibv_icmd_read_comp_cntr(comp_cntr, IB_UVERBS_COMP_CNTR_ENTRY_ERR, value);
104+
}
105+
106+
int ibv_cmd_qp_attach_comp_cntr(struct ibv_qp *qp, struct ibv_comp_cntr *comp_cntr,
107+
struct ibv_qp_attach_comp_cntr_attr *attr)
108+
{
109+
DECLARE_COMMAND_BUFFER(cmdb, UVERBS_OBJECT_QP, UVERBS_METHOD_QP_ATTACH_COMP_CNTR, 3);
110+
uint32_t op_mask = 0;
111+
112+
fill_attr_in_obj(cmdb, UVERBS_ATTR_QP_ATTACH_COMP_CNTR_HANDLE, qp->handle);
113+
fill_attr_in_obj(cmdb, UVERBS_ATTR_QP_ATTACH_COMP_CNTR_CNTR_HANDLE, comp_cntr->handle);
114+
115+
if (attr->op_mask & IBV_QP_ATTACH_COMP_CNTR_OP_SEND)
116+
op_mask |= IB_UVERBS_QP_ATTACH_COMP_CNTR_OP_SEND;
117+
if (attr->op_mask & IBV_QP_ATTACH_COMP_CNTR_OP_RECV)
118+
op_mask |= IB_UVERBS_QP_ATTACH_COMP_CNTR_OP_RECV;
119+
if (attr->op_mask & IBV_QP_ATTACH_COMP_CNTR_OP_RDMA_READ)
120+
op_mask |= IB_UVERBS_QP_ATTACH_COMP_CNTR_OP_RDMA_READ;
121+
if (attr->op_mask & IBV_QP_ATTACH_COMP_CNTR_OP_REMOTE_RDMA_READ)
122+
op_mask |= IB_UVERBS_QP_ATTACH_COMP_CNTR_OP_REMOTE_RDMA_READ;
123+
if (attr->op_mask & IBV_QP_ATTACH_COMP_CNTR_OP_RDMA_WRITE)
124+
op_mask |= IB_UVERBS_QP_ATTACH_COMP_CNTR_OP_RDMA_WRITE;
125+
if (attr->op_mask & IBV_QP_ATTACH_COMP_CNTR_OP_REMOTE_RDMA_WRITE)
126+
op_mask |= IB_UVERBS_QP_ATTACH_COMP_CNTR_OP_REMOTE_RDMA_WRITE;
127+
128+
fill_attr_in_uint32(cmdb, UVERBS_ATTR_QP_ATTACH_COMP_CNTR_OP_MASK, op_mask);
129+
130+
return execute_ioctl(qp->context, cmdb);
131+
}

libibverbs/cmd_device.c

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -694,6 +694,10 @@ int ibv_cmd_query_device_any(struct ibv_context *context,
694694
else
695695
attr->phys_port_cnt_ex = attr->orig_attr.phys_port_cnt;
696696
}
697+
698+
if (CAN_COPY(max_comp_cntr, max_comp_cntr))
699+
attr->max_comp_cntr = resp->max_comp_cntr;
700+
697701
#undef CAN_COPY
698702

699703
return 0;

libibverbs/driver.h

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -364,6 +364,8 @@ struct verbs_context_ops {
364364
void (*cq_event)(struct ibv_cq *cq);
365365
struct ibv_ah *(*create_ah)(struct ibv_pd *pd,
366366
struct ibv_ah_attr *attr);
367+
struct ibv_comp_cntr *(*create_comp_cntr)(struct ibv_context *context,
368+
struct ibv_comp_cntr_init_attr *attr);
367369
struct ibv_counters *(*create_counters)(struct ibv_context *context,
368370
struct ibv_counters_init_attr *init_attr);
369371
struct ibv_cq *(*create_cq)(struct ibv_context *context, int cqe,
@@ -397,6 +399,7 @@ struct verbs_context_ops {
397399
int (*dealloc_td)(struct ibv_td *td);
398400
int (*dereg_mr)(struct verbs_mr *vmr);
399401
int (*destroy_ah)(struct ibv_ah *ah);
402+
int (*destroy_comp_cntr)(struct ibv_comp_cntr *comp_cntr);
400403
int (*destroy_counters)(struct ibv_counters *counters);
401404
int (*destroy_cq)(struct ibv_cq *cq);
402405
int (*destroy_flow)(struct ibv_flow *flow);
@@ -417,6 +420,9 @@ struct verbs_context_ops {
417420
uint32_t mr_handle);
418421
struct ibv_pd *(*import_pd)(struct ibv_context *context,
419422
uint32_t pd_handle);
423+
int (*inc_comp_cntr)(struct ibv_comp_cntr *comp_cntr, uint64_t amount);
424+
int (*inc_err_comp_cntr)(struct ibv_comp_cntr *comp_cntr,
425+
uint64_t amount);
420426
int (*modify_cq)(struct ibv_cq *cq, struct ibv_modify_cq_attr *attr);
421427
int (*modify_flow_action_esp)(struct ibv_flow_action *action,
422428
struct ibv_flow_action_esp_attr *attr);
@@ -441,6 +447,9 @@ struct verbs_context_ops {
441447
struct ibv_ops_wr **bad_op);
442448
int (*post_srq_recv)(struct ibv_srq *srq, struct ibv_recv_wr *recv_wr,
443449
struct ibv_recv_wr **bad_recv_wr);
450+
int (*qp_attach_comp_cntr)(struct ibv_qp *qp,
451+
struct ibv_comp_cntr *comp_cntr,
452+
struct ibv_qp_attach_comp_cntr_attr *attr);
444453
int (*query_device_ex)(struct ibv_context *context,
445454
const struct ibv_query_device_ex_input *input,
446455
struct ibv_device_attr_ex *attr,
@@ -457,10 +466,13 @@ struct verbs_context_ops {
457466
int (*query_rt_values)(struct ibv_context *context,
458467
struct ibv_values_ex *values);
459468
int (*query_srq)(struct ibv_srq *srq, struct ibv_srq_attr *srq_attr);
469+
int (*read_comp_cntr)(struct ibv_comp_cntr *comp_cntr, uint64_t *value);
460470
int (*read_counters)(struct ibv_counters *counters,
461471
uint64_t *counters_value,
462472
uint32_t ncounters,
463473
uint32_t flags);
474+
int (*read_err_comp_cntr)(struct ibv_comp_cntr *comp_cntr,
475+
uint64_t *value);
464476
struct ibv_mr *(*reg_dm_mr)(struct ibv_pd *pd, struct ibv_dm *dm,
465477
uint64_t dm_offset, size_t length,
466478
unsigned int access);
@@ -475,7 +487,10 @@ struct verbs_context_ops {
475487
int (*rereg_mr)(struct verbs_mr *vmr, int flags, struct ibv_pd *pd,
476488
void *addr, size_t length, int access);
477489
int (*resize_cq)(struct ibv_cq *cq, int cqe);
490+
int (*set_comp_cntr)(struct ibv_comp_cntr *comp_cntr, uint64_t value);
478491
int (*set_ece)(struct ibv_qp *qp, struct ibv_ece *ece);
492+
int (*set_err_comp_cntr)(struct ibv_comp_cntr *comp_cntr,
493+
uint64_t value);
479494
void (*unimport_dm)(struct ibv_dm *dm);
480495
void (*unimport_mr)(struct ibv_mr *mr);
481496
void (*unimport_pd)(struct ibv_pd *pd);
@@ -748,6 +763,21 @@ int ibv_cmd_export_dmabuf_fd(struct ibv_context *ctx, off_t pg_off);
748763
int ibv_cmd_alloc_dmah(struct ibv_context *ctx, struct verbs_dmah *st,
749764
struct ibv_dmah_init_attr *attr);
750765
int ibv_cmd_free_dmah(struct verbs_dmah *dmah);
766+
767+
int ibv_cmd_create_comp_cntr(struct ibv_context *context,
768+
struct ibv_comp_cntr *comp_cntr,
769+
struct ibv_command_buffer *link);
770+
int ibv_cmd_destroy_comp_cntr(struct ibv_comp_cntr *comp_cntr);
771+
int ibv_cmd_set_comp_cntr(struct ibv_comp_cntr *comp_cntr, uint64_t value);
772+
int ibv_cmd_set_err_comp_cntr(struct ibv_comp_cntr *comp_cntr, uint64_t value);
773+
int ibv_cmd_inc_comp_cntr(struct ibv_comp_cntr *comp_cntr, uint64_t amount);
774+
int ibv_cmd_inc_err_comp_cntr(struct ibv_comp_cntr *comp_cntr, uint64_t amount);
775+
int ibv_cmd_read_comp_cntr(struct ibv_comp_cntr *comp_cntr, uint64_t *value);
776+
int ibv_cmd_read_err_comp_cntr(struct ibv_comp_cntr *comp_cntr, uint64_t *value);
777+
int ibv_cmd_qp_attach_comp_cntr(struct ibv_qp *qp,
778+
struct ibv_comp_cntr *comp_cntr,
779+
struct ibv_qp_attach_comp_cntr_attr *attr);
780+
751781
int ibv_cmd_reg_dm_mr(struct ibv_pd *pd, struct verbs_dm *dm,
752782
uint64_t offset, size_t length,
753783
unsigned int access, struct verbs_mr *vmr,

0 commit comments

Comments
 (0)