Skip to content

Commit 9953434

Browse files
roiedaninoroiedanino
committed
UCP/RNDV: Use four read paths on ConnectX-9 XDR
1 parent d0e1f7a commit 9953434

14 files changed

Lines changed: 332 additions & 31 deletions

File tree

‎docs/source/faq.md‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -256,7 +256,10 @@ and NUMA locality.
256256

257257
#### Is it possible to use more than 2 rails?
258258

259-
Yes, by setting `UCX_MAX_RNDV_RAILS=<num-rails>`. Currently up to 4 are supported.
259+
Yes, by setting `UCX_MAX_RNDV_RAILS=<num-rails>`. Up to 4 are supported. With
260+
the default `auto` setting, certain transports (for example ConnectX-9 XDR for
261+
RDMA read) may automatically use additional paths on a single device beyond the
262+
two-device baseline.
260263

261264
#### Is it possible that each process would just use the closest device?
262265

‎src/ucp/core/ucp_context.c‎

Lines changed: 31 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
#include <ucs/type/init_once.h>
3030
#include <ucs/vfs/base/vfs_cb.h>
3131
#include <ucs/vfs/base/vfs_obj.h>
32+
#include <limits.h>
3233
#include <string.h>
3334
#include <dlfcn.h>
3435

@@ -227,11 +228,15 @@ static ucs_config_field_t ucp_context_config_table[] = {
227228
ucs_offsetof(ucp_context_config_t, max_eager_lanes), UCS_CONFIG_TYPE_UINT},
228229

229230
{"MAX_RNDV_LANES", NULL,"",
230-
ucs_offsetof(ucp_context_config_t, max_rndv_lanes), UCS_CONFIG_TYPE_UINT},
231+
ucs_offsetof(ucp_context_config_t, max_rndv_lanes_config),
232+
UCS_CONFIG_TYPE_ULUNITS},
231233

232-
{"MAX_RNDV_RAILS", "2",
233-
"Maximal number of devices on which a rendezvous operation may be executed in parallel",
234-
ucs_offsetof(ucp_context_config_t, max_rndv_lanes), UCS_CONFIG_TYPE_UINT},
234+
{"MAX_RNDV_RAILS", "auto",
235+
"Maximal number of devices on which a rendezvous operation may be executed\n"
236+
"in parallel. With 'auto', the default is two devices, while a transport\n"
237+
"may expose additional paths needed by a single device.",
238+
ucs_offsetof(ucp_context_config_t, max_rndv_lanes_config),
239+
UCS_CONFIG_TYPE_ULUNITS},
235240

236241
{"MAX_RMA_LANES", NULL, "",
237242
ucs_offsetof(ucp_context_config_t, max_rma_lanes), UCS_CONFIG_TYPE_UINT},
@@ -2367,6 +2372,23 @@ static double ucp_context_get_protov1_memcpy_bw()
23672372
UCP_CPU_EST_BCOPY_BW_DEFAULT_PROTOV1;
23682373
}
23692374

2375+
static ucs_status_t
2376+
ucp_context_resolve_rndv_config(ucp_context_config_t *config)
2377+
{
2378+
if (config->max_rndv_lanes_config == UCS_ULUNITS_AUTO) {
2379+
config->max_rndv_lanes = UCP_MAX_RNDV_LANES_DEFAULT;
2380+
} else if ((config->max_rndv_lanes_config == UCS_ULUNITS_INF) ||
2381+
(config->max_rndv_lanes_config > UINT_MAX)) {
2382+
ucs_error("maximum number of rendezvous lanes must not exceed %u",
2383+
UINT_MAX);
2384+
return UCS_ERR_INVALID_PARAM;
2385+
} else {
2386+
config->max_rndv_lanes = config->max_rndv_lanes_config;
2387+
}
2388+
2389+
return UCS_OK;
2390+
}
2391+
23702392
static int
23712393
ucp_dynamic_tl_switch_config_valid(const ucp_context_config_t *config)
23722394
{
@@ -2404,6 +2426,11 @@ static ucs_status_t ucp_fill_config(ucp_context_h context,
24042426
goto err;
24052427
}
24062428

2429+
status = ucp_context_resolve_rndv_config(&context->config.ext);
2430+
if (status != UCS_OK) {
2431+
goto err_free_config_ext;
2432+
}
2433+
24072434
if (context->config.ext.estimated_num_eps != UCS_ULUNITS_AUTO) {
24082435
/* num_eps was set via the env variable. Override current value */
24092436
context->config.est_num_eps = context->config.ext.estimated_num_eps;

‎src/ucp/core/ucp_context.h‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,9 @@ typedef enum {
6060
} ucp_reg_devices_mode_t;
6161

6262

63+
#define UCP_MAX_RNDV_LANES_DEFAULT 2
64+
65+
6366
static UCS_F_ALWAYS_INLINE ucp_reg_devices_mode_t
6467
ucp_reg_devices_mode(unsigned long max_hca_per_gpu)
6568
{
@@ -150,6 +153,8 @@ typedef struct ucp_context_config {
150153
int adaptive_progress;
151154
/** Eager-am multi-lane support */
152155
unsigned max_eager_lanes;
156+
/** Raw rendezvous lane limit before resolving the auto value */
157+
unsigned long max_rndv_lanes_config;
153158
/** Rendezvous-get multi-lane support */
154159
unsigned max_rndv_lanes;
155160
/** RMA multi-lane support */

‎src/ucp/proto/proto_multi.c‎

Lines changed: 48 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -103,11 +103,47 @@ ucp_proto_select_add_lane(ucp_proto_lane_selection_t *selection,
103103
selection->dev_count[dev_index]++;
104104
}
105105

106+
static UCS_F_ALWAYS_INLINE void
107+
ucp_proto_multi_limit_device_paths(
108+
const ucp_proto_init_params_t *params,
109+
const ucp_lane_index_t *lanes,
110+
ucp_lane_index_t lane, ucp_lane_index_t max_paths_per_device,
111+
ucp_lane_index_t max_lanes,
112+
const ucp_proto_lane_selection_t *selection,
113+
ucp_lane_map_t *index_map_p)
114+
{
115+
const uct_iface_attr_t *iface_attr;
116+
ucp_rsc_index_t dev_index;
117+
ucp_lane_index_t candidate_index;
118+
119+
if (max_paths_per_device == 0) {
120+
return;
121+
}
122+
123+
iface_attr = ucp_proto_common_get_iface_attr(params, lane);
124+
if (iface_attr->dev_num_paths <= max_lanes) {
125+
return;
126+
}
127+
128+
dev_index = ucp_proto_common_get_dev_index(params, lane);
129+
if (selection->dev_count[dev_index] < max_paths_per_device) {
130+
return;
131+
}
132+
133+
ucs_for_each_bit(candidate_index, *index_map_p) {
134+
if (ucp_proto_common_get_dev_index(params, lanes[candidate_index]) ==
135+
dev_index) {
136+
*index_map_p &= ~UCS_BIT(candidate_index);
137+
}
138+
}
139+
}
140+
106141
static void
107142
ucp_proto_multi_select_bw_lanes(const ucp_proto_init_params_t *params,
108143
const ucp_lane_index_t *lanes,
109144
ucp_lane_index_t num_lanes,
110145
ucp_lane_index_t max_lanes,
146+
ucp_lane_index_t max_paths_per_device,
111147
const ucp_proto_common_tl_perf_t *lanes_perf,
112148
int fixed_first_lane,
113149
ucp_proto_lane_selection_t *selection)
@@ -123,6 +159,9 @@ ucp_proto_multi_select_bw_lanes(const ucp_proto_init_params_t *params,
123159
if (fixed_first_lane) {
124160
ucp_proto_select_add_lane(selection, params, lanes[0]);
125161
index_map &= ~UCS_BIT(0);
162+
ucp_proto_multi_limit_device_paths(params, lanes, lanes[0],
163+
max_paths_per_device, max_lanes,
164+
selection, &index_map);
126165
}
127166

128167
for (i = fixed_first_lane? 1 : 0; i < ucs_min(max_lanes, num_lanes); ++i) {
@@ -136,6 +175,13 @@ ucp_proto_multi_select_bw_lanes(const ucp_proto_init_params_t *params,
136175

137176
ucp_proto_select_add_lane(selection, params, lanes[lane_index]);
138177
index_map &= ~UCS_BIT(lane_index);
178+
ucp_proto_multi_limit_device_paths(params, lanes, lanes[lane_index],
179+
max_paths_per_device, max_lanes,
180+
selection, &index_map);
181+
182+
if (index_map == 0) {
183+
break;
184+
}
139185
}
140186

141187
/* TODO: Aggregate performance:
@@ -409,7 +455,8 @@ ucs_status_t ucp_proto_multi_init(const ucp_proto_multi_init_params_t *params,
409455
}
410456

411457
ucp_proto_multi_select_bw_lanes(&params->super.super, lanes, num_lanes,
412-
params->max_lanes, lanes_perf,
458+
params->max_lanes,
459+
params->max_paths_per_device, lanes_perf,
413460
fixed_first_lane, &selection);
414461

415462
ucs_trace("selected %u lanes for %s", selection.num_lanes,

‎src/ucp/proto/proto_multi.h‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -119,6 +119,10 @@ typedef struct {
119119
/* Maximal number of lanes to select */
120120
ucp_lane_index_t max_lanes;
121121

122+
/* Maximal number of paths to select on a device that exposes more paths
123+
* than max_lanes. Zero means no per-device limit. */
124+
ucp_lane_index_t max_paths_per_device;
125+
122126
/* Minimal chunk size. It defines the minimal size of the fragment to split into
123127
* several parts. The goal is to not split below this limit */
124128
size_t min_chunk;

‎src/ucp/rndv/rndv_am.c‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,9 @@ static void ucp_rndv_am_probe_common(ucp_proto_multi_init_params_t *params)
3232
params->middle.lane_type = UCP_LANE_TYPE_AM_BW;
3333
params->super.hdr_size = sizeof(ucp_request_data_hdr_t);
3434
params->max_lanes = context->config.ext.max_rndv_lanes;
35+
params->max_paths_per_device =
36+
(context->config.ext.max_rndv_lanes_config == UCS_ULUNITS_AUTO) ?
37+
context->config.ext.max_rndv_lanes : 0;
3538
params->opt_align_offs = UCP_PROTO_COMMON_OFFSET_INVALID;
3639

3740
ucp_proto_multi_probe(params);

‎src/ucp/rndv/rndv_get.c‎

Lines changed: 58 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,57 @@
1717

1818
#define UCP_PROTO_RNDV_GET_DESC "read from remote"
1919

20+
static ucp_lane_index_t
21+
ucp_proto_rndv_get_max_lanes(const ucp_proto_init_params_t *init_params)
22+
{
23+
ucp_context_t *context = init_params->worker->context;
24+
const ucp_ep_config_key_t *ep_key = init_params->ep_config_key;
25+
ucp_lane_index_t max_lanes = context->config.ext.max_rndv_lanes;
26+
const uct_iface_attr_t *iface_attr;
27+
ucp_lane_index_t lane;
28+
29+
if (context->config.ext.max_rndv_lanes_config != UCS_ULUNITS_AUTO) {
30+
return max_lanes;
31+
}
32+
33+
for (lane = 0; lane < ep_key->num_lanes; ++lane) {
34+
if (!(ep_key->lanes[lane].lane_types &
35+
UCS_BIT(UCP_LANE_TYPE_RMA_BW))) {
36+
continue;
37+
}
38+
39+
iface_attr = ucp_proto_common_get_iface_attr(init_params, lane);
40+
if (!(iface_attr->cap.flags & UCT_IFACE_FLAG_GET_ZCOPY)) {
41+
continue;
42+
}
43+
44+
max_lanes = ucs_max(max_lanes,
45+
ucs_min(iface_attr->dev_num_paths,
46+
UCP_PROTO_MAX_LANES));
47+
}
48+
49+
return max_lanes;
50+
}
51+
52+
static void
53+
ucp_proto_rndv_get_add_proto(ucp_proto_multi_init_params_t *params)
54+
{
55+
ucp_proto_rndv_bulk_priv_t rpriv;
56+
ucp_proto_perf_t *perf;
57+
ucs_status_t status;
58+
59+
status = ucp_proto_rndv_bulk_init(params, UCP_PROTO_RNDV_GET_DESC,
60+
UCP_PROTO_RNDV_ATS_NAME, &perf, &rpriv);
61+
if (status != UCS_OK) {
62+
return;
63+
}
64+
65+
ucp_proto_select_add_proto(&params->super.super, params->super.cfg_thresh,
66+
params->super.cfg_priority, perf, &rpriv,
67+
UCP_PROTO_MULTI_EXTENDED_PRIV_SIZE(&rpriv,
68+
mpriv));
69+
}
70+
2071
static void
2172
ucp_proto_rndv_get_common_probe(const ucp_proto_init_params_t *init_params,
2273
uint64_t rndv_modes, size_t max_length,
@@ -26,6 +77,8 @@ ucp_proto_rndv_get_common_probe(const ucp_proto_init_params_t *init_params,
2677
const ucp_memory_info_t *reg_mem_info)
2778
{
2879
ucp_context_t *context = init_params->worker->context;
80+
ucp_lane_index_t max_lanes =
81+
ucp_proto_rndv_get_max_lanes(init_params);
2982
ucp_proto_multi_init_params_t params = {
3083
.super.super = *init_params,
3184
.super.cfg_thresh = ucp_proto_rndv_cfg_thresh(context, rndv_modes),
@@ -58,25 +111,18 @@ ucp_proto_rndv_get_common_probe(const ucp_proto_init_params_t *init_params,
58111
.opt_align_offs = ucs_offsetof(uct_iface_attr_t,
59112
cap.get.opt_zcopy_align),
60113
};
61-
ucp_proto_rndv_bulk_priv_t rpriv;
62-
ucp_proto_perf_t *perf;
63-
ucs_status_t status;
64114

65115
if (!ucp_proto_rndv_op_check(init_params, UCP_OP_ID_RNDV_RECV,
66116
support_ppln)) {
67117
return;
68118
}
69119

70-
status = ucp_proto_rndv_bulk_init(&params, UCP_PROTO_RNDV_GET_DESC,
71-
UCP_PROTO_RNDV_ATS_NAME, &perf, &rpriv);
72-
if (status != UCS_OK) {
73-
return;
74-
}
120+
ucp_proto_rndv_get_add_proto(&params);
75121

76-
ucp_proto_select_add_proto(&params.super.super, params.super.cfg_thresh,
77-
params.super.cfg_priority, perf, &rpriv,
78-
UCP_PROTO_MULTI_EXTENDED_PRIV_SIZE(&rpriv,
79-
mpriv));
122+
if (max_lanes > params.max_lanes) {
123+
params.max_lanes = max_lanes;
124+
ucp_proto_rndv_get_add_proto(&params);
125+
}
80126
}
81127

82128
static UCS_F_ALWAYS_INLINE ucs_status_t

‎src/ucp/rndv/rndv_put.c‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -264,6 +264,9 @@ ucp_proto_rndv_put_common_probe(const ucp_proto_init_params_t *init_params,
264264
.super.exclude_map = 0,
265265
.super.reg_mem_info = *reg_mem_info,
266266
.max_lanes = context->config.ext.max_rndv_lanes,
267+
.max_paths_per_device =
268+
(context->config.ext.max_rndv_lanes_config ==
269+
UCS_ULUNITS_AUTO) ? context->config.ext.max_rndv_lanes : 0,
267270
.min_chunk = context->config.ext.min_rndv_chunk_size,
268271
.initial_reg_md_map = initial_reg_md_map,
269272
.first.tl_cap_flags = UCT_IFACE_FLAG_PUT_ZCOPY,

‎src/uct/ib/base/ib_device.c‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -160,7 +160,7 @@ static uct_ib_device_spec_t uct_ib_builtin_device_specs[] = {
160160
UCT_IB_DEVICE_FLAG_DC_V2, 80},
161161
{"ConnectX-9", {0x15b3, 4133},
162162
UCT_IB_DEVICE_FLAG_MELLANOX | UCT_IB_DEVICE_FLAG_MLX5_PRM |
163-
UCT_IB_DEVICE_FLAG_DC_V2, 90},
163+
UCT_IB_DEVICE_FLAG_DC_V2 | UCT_IB_DEVICE_FLAG_XDR_READ_4_PATHS, 90},
164164
{"ConnectX-10", {0x15b3, 4135},
165165
UCT_IB_DEVICE_FLAG_MELLANOX | UCT_IB_DEVICE_FLAG_MLX5_PRM |
166166
UCT_IB_DEVICE_FLAG_DC_V2, 100},

‎src/uct/ib/base/ib_device.h‎

Lines changed: 12 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -90,17 +90,18 @@ typedef enum uct_ib_roce_version {
9090

9191

9292
enum {
93-
UCT_IB_DEVICE_FLAG_MLX4_PRM = UCS_BIT(1), /* Device supports mlx4 PRM */
94-
UCT_IB_DEVICE_FLAG_MLX5_PRM = UCS_BIT(2), /* Device supports mlx5 PRM */
95-
UCT_IB_DEVICE_FLAG_MELLANOX = UCS_BIT(3), /* Mellanox device */
96-
UCT_IB_DEVICE_FLAG_SRQ = UCS_BIT(4), /* Supports SRQ */
97-
UCT_IB_DEVICE_FLAG_LINK_IB = UCS_BIT(5), /* Require only IB */
98-
UCT_IB_DEVICE_FLAG_DC_V1 = UCS_BIT(6), /* Device supports DC ver 1 */
99-
UCT_IB_DEVICE_FLAG_DC_V2 = UCS_BIT(7), /* Device supports DC ver 2 */
100-
UCT_IB_DEVICE_FLAG_AV = UCS_BIT(8), /* Device supports compact AV */
101-
UCT_IB_DEVICE_FLAG_DC = UCT_IB_DEVICE_FLAG_DC_V1 |
102-
UCT_IB_DEVICE_FLAG_DC_V2, /* Device supports DC */
103-
UCT_IB_DEVICE_FAILED = UCS_BIT(9) /* Got fatal error */
93+
UCT_IB_DEVICE_FLAG_MLX4_PRM = UCS_BIT(1), /* Device supports mlx4 PRM */
94+
UCT_IB_DEVICE_FLAG_MLX5_PRM = UCS_BIT(2), /* Device supports mlx5 PRM */
95+
UCT_IB_DEVICE_FLAG_MELLANOX = UCS_BIT(3), /* Mellanox device */
96+
UCT_IB_DEVICE_FLAG_SRQ = UCS_BIT(4), /* Supports SRQ */
97+
UCT_IB_DEVICE_FLAG_LINK_IB = UCS_BIT(5), /* Require only IB */
98+
UCT_IB_DEVICE_FLAG_DC_V1 = UCS_BIT(6), /* Device supports DC ver 1 */
99+
UCT_IB_DEVICE_FLAG_DC_V2 = UCS_BIT(7), /* Device supports DC ver 2 */
100+
UCT_IB_DEVICE_FLAG_AV = UCS_BIT(8), /* Device supports compact AV */
101+
UCT_IB_DEVICE_FLAG_DC = UCT_IB_DEVICE_FLAG_DC_V1 |
102+
UCT_IB_DEVICE_FLAG_DC_V2, /* Device supports DC */
103+
UCT_IB_DEVICE_FAILED = UCS_BIT(9), /* Got fatal error */
104+
UCT_IB_DEVICE_FLAG_XDR_READ_4_PATHS = UCS_BIT(10) /* XDR: 4 read paths */
104105
};
105106

106107

0 commit comments

Comments
 (0)