Skip to content

Commit 0ef418c

Browse files
JeynmannZJeynmannZ
committed
UCP/PROTO: Enable short and bcopy lane failover
Advertise replay support for AM eager and PUT short/bcopy protocols when their candidate lanes are compatible with hardware PSN failover. Preserve protocol-specific replay metadata for single- and multi-fragment operations.
1 parent 8d188a5 commit 0ef418c

4 files changed

Lines changed: 75 additions & 3 deletions

File tree

‎src/ucp/am/eager_multi.c‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,18 @@ ucp_am_eager_multi_bcopy_proto_probe(const ucp_proto_init_params_t *init_params)
4949
.opt_align_offs = UCP_PROTO_COMMON_OFFSET_INVALID
5050
};
5151

52+
if (init_params->ep_config_key->err_mode ==
53+
UCP_ERR_HANDLING_MODE_FAILOVER) {
54+
if (init_params->ep_config_key->dst_version <
55+
UCP_WIREUP_LANE_STATE_MIN_VERSION) {
56+
return;
57+
}
58+
59+
params.super.flags |= UCP_PROTO_COMMON_INIT_FLAG_FAILOVER;
60+
params.first.tl_v2_cap_flags = UCT_IFACE_FLAG_V2_QUERY_TOKEN;
61+
params.middle.tl_v2_cap_flags = UCT_IFACE_FLAG_V2_QUERY_TOKEN;
62+
}
63+
5264
if (!ucp_am_check_init_params(init_params, UCP_PROTO_AM_OP_ID_MASK,
5365
UCP_PROTO_SELECT_OP_FLAG_AM_RNDV)) {
5466
return;

‎src/ucp/am/eager_single.c‎

Lines changed: 24 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
#include <ucp/proto/proto_common.inl>
1616
#include <ucp/proto/proto_single.h>
1717
#include <ucp/proto/proto_single.inl>
18+
#include <ucp/wireup/wireup.h>
1819

1920

2021
static UCS_F_ALWAYS_INLINE void
@@ -29,6 +30,25 @@ static size_t ucp_am_eager_single_hdr_size(ucp_operation_id_t op_id)
2930
((op_id == UCP_OP_ID_AM_SEND) ? 0 : sizeof(ucp_am_reply_ftr_t));
3031
}
3132

33+
static int
34+
ucp_am_eager_single_enable_failover(const ucp_proto_init_params_t *init_params,
35+
ucp_proto_single_init_params_t *params)
36+
{
37+
if (init_params->ep_config_key->err_mode !=
38+
UCP_ERR_HANDLING_MODE_FAILOVER) {
39+
return 1;
40+
}
41+
42+
if (init_params->ep_config_key->dst_version <
43+
UCP_WIREUP_LANE_STATE_MIN_VERSION) {
44+
return 0;
45+
}
46+
47+
params->super.flags |= UCP_PROTO_COMMON_INIT_FLAG_FAILOVER;
48+
params->tl_v2_cap_flags = UCT_IFACE_FLAG_V2_QUERY_TOKEN;
49+
return 1;
50+
}
51+
3252
static UCS_F_ALWAYS_INLINE ucs_status_t
3353
ucp_am_eager_short_proto_progress_common(uct_pending_req_t *self, int is_reply)
3454
{
@@ -116,7 +136,8 @@ ucp_am_eager_short_probe_common(const ucp_proto_init_params_t *init_params,
116136
.tl_cap_flags = UCT_IFACE_FLAG_AM_SHORT
117137
};
118138

119-
if (!ucp_am_check_init_params(init_params, UCS_BIT(op_id),
139+
if (!ucp_am_eager_single_enable_failover(init_params, &params) ||
140+
!ucp_am_check_init_params(init_params, UCS_BIT(op_id),
120141
UCP_PROTO_SELECT_OP_FLAG_AM_RNDV) ||
121142
!ucp_proto_is_short_supported(select_param)) {
122143
return;
@@ -248,7 +269,8 @@ static void ucp_am_eager_single_bcopy_probe_common(
248269
.tl_cap_flags = UCT_IFACE_FLAG_AM_BCOPY
249270
};
250271

251-
if (!ucp_am_check_init_params(init_params, UCS_BIT(op_id),
272+
if (!ucp_am_eager_single_enable_failover(init_params, &params) ||
273+
!ucp_am_check_init_params(init_params, UCS_BIT(op_id),
252274
UCP_PROTO_SELECT_OP_FLAG_AM_RNDV)) {
253275
return;
254276
}

‎src/ucp/rma/put_am.c‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
#include <ucp/dt/datatype_iter.inl>
1616
#include <ucp/proto/proto_init.h>
1717
#include <ucp/proto/proto_multi.inl>
18+
#include <ucp/wireup/wireup.h>
1819

1920

2021
static size_t ucp_proto_put_am_bcopy_pack(void *dest, void *arg)
@@ -115,6 +116,18 @@ ucp_proto_put_am_bcopy_probe(const ucp_proto_init_params_t *init_params)
115116
.opt_align_offs = UCP_PROTO_COMMON_OFFSET_INVALID
116117
};
117118

119+
if (init_params->ep_config_key->err_mode ==
120+
UCP_ERR_HANDLING_MODE_FAILOVER) {
121+
if (init_params->ep_config_key->dst_version <
122+
UCP_WIREUP_LANE_STATE_MIN_VERSION) {
123+
return;
124+
}
125+
126+
params.super.flags |= UCP_PROTO_COMMON_INIT_FLAG_FAILOVER;
127+
params.first.tl_v2_cap_flags = UCT_IFACE_FLAG_V2_QUERY_TOKEN;
128+
params.middle.tl_v2_cap_flags = UCT_IFACE_FLAG_V2_QUERY_TOKEN;
129+
}
130+
118131
if (!ucp_proto_init_check_op(init_params, UCS_BIT(UCP_OP_ID_PUT))) {
119132
return;
120133
}

‎src/ucp/rma/put_offload.c‎

Lines changed: 26 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
#include <ucp/proto/proto_init.h>
1717
#include <ucp/proto/proto_multi.inl>
1818
#include <ucp/proto/proto_single.inl>
19+
#include <ucp/wireup/wireup.h>
1920

2021
static ucs_status_t ucp_proto_put_offload_short_progress(uct_pending_req_t *self)
2122
{
@@ -84,6 +85,17 @@ ucp_proto_put_offload_short_probe(const ucp_proto_init_params_t *init_params)
8485
.tl_cap_flags = UCT_IFACE_FLAG_PUT_SHORT
8586
};
8687

88+
if (init_params->ep_config_key->err_mode ==
89+
UCP_ERR_HANDLING_MODE_FAILOVER) {
90+
if (init_params->ep_config_key->dst_version <
91+
UCP_WIREUP_LANE_STATE_MIN_VERSION) {
92+
return;
93+
}
94+
95+
params.super.flags |= UCP_PROTO_COMMON_INIT_FLAG_FAILOVER;
96+
params.tl_v2_cap_flags = UCT_IFACE_FLAG_V2_QUERY_TOKEN;
97+
}
98+
8799
if (!ucp_proto_init_check_op(init_params, UCS_BIT(UCP_OP_ID_PUT)) ||
88100
!ucp_proto_is_short_supported(init_params->select_param)) {
89101
return;
@@ -212,7 +224,8 @@ ucp_proto_put_offload_bcopy_probe(const ucp_proto_init_params_t *init_params)
212224
.super.memtype_op = UCT_EP_OP_LAST,
213225
.super.flags = UCP_PROTO_COMMON_INIT_FLAG_RECV_ZCOPY |
214226
UCP_PROTO_COMMON_INIT_FLAG_REMOTE_ACCESS |
215-
UCP_PROTO_COMMON_INIT_FLAG_ERR_HANDLING,
227+
UCP_PROTO_COMMON_INIT_FLAG_ERR_HANDLING |
228+
UCP_PROTO_COMMON_INIT_FLAG_RESUME,
216229
.super.exclude_map = 0,
217230
.super.reg_mem_info = ucp_mem_info_unknown,
218231
.max_lanes = UCP_PROTO_RMA_MAX_BCOPY_LANES,
@@ -225,6 +238,18 @@ ucp_proto_put_offload_bcopy_probe(const ucp_proto_init_params_t *init_params)
225238
.opt_align_offs = UCP_PROTO_COMMON_OFFSET_INVALID
226239
};
227240

241+
if (init_params->ep_config_key->err_mode ==
242+
UCP_ERR_HANDLING_MODE_FAILOVER) {
243+
if (init_params->ep_config_key->dst_version <
244+
UCP_WIREUP_LANE_STATE_MIN_VERSION) {
245+
return;
246+
}
247+
248+
params.super.flags |= UCP_PROTO_COMMON_INIT_FLAG_FAILOVER;
249+
params.first.tl_v2_cap_flags = UCT_IFACE_FLAG_V2_QUERY_TOKEN;
250+
params.middle.tl_v2_cap_flags = UCT_IFACE_FLAG_V2_QUERY_TOKEN;
251+
}
252+
228253
if (!ucp_proto_init_check_op(init_params, UCS_BIT(UCP_OP_ID_PUT))) {
229254
return;
230255
}

0 commit comments

Comments
 (0)