UCP/RNDV: Use four read paths on ConnectX-9 XDR

Author: roiedanino

docs/source/faq.md

@@ -256,7 +256,10 @@ and NUMA locality.
 
 #### Is it possible to use more than 2 rails?
 
-Yes, by setting `UCX_MAX_RNDV_RAILS=<num-rails>`. Currently up to 4 are supported.
+Yes, by setting `UCX_MAX_RNDV_RAILS=<num-rails>`. Up to 4 are supported. With
+the default `auto` setting, certain transports (for example ConnectX-9 XDR for
+RDMA read) may automatically use additional paths on a single device beyond the
+two-device baseline.
 
 #### Is it possible that each process would just use the closest device?
 

src/ucp/core/ucp_context.c

@@ -29,6 +29,7 @@
 #include <ucs/type/init_once.h>
 #include <ucs/vfs/base/vfs_cb.h>
 #include <ucs/vfs/base/vfs_obj.h>
+#include <limits.h>
 #include <string.h>
 #include <dlfcn.h>
 
@@ -227,11 +228,15 @@ static ucs_config_field_t ucp_context_config_table[] = {
    ucs_offsetof(ucp_context_config_t, max_eager_lanes), UCS_CONFIG_TYPE_UINT},
 
   {"MAX_RNDV_LANES", NULL,"",
-   ucs_offsetof(ucp_context_config_t, max_rndv_lanes), UCS_CONFIG_TYPE_UINT},
+   ucs_offsetof(ucp_context_config_t, max_rndv_lanes_config),
+   UCS_CONFIG_TYPE_ULUNITS},
 
-  {"MAX_RNDV_RAILS", "2",
-   "Maximal number of devices on which a rendezvous operation may be executed in parallel",
-   ucs_offsetof(ucp_context_config_t, max_rndv_lanes), UCS_CONFIG_TYPE_UINT},
+  {"MAX_RNDV_RAILS", "auto",
+   "Maximal number of devices on which a rendezvous operation may be executed\n"
+   "in parallel. With 'auto', the default is two devices, while a transport\n"
+   "may expose additional paths needed by a single device.",
+   ucs_offsetof(ucp_context_config_t, max_rndv_lanes_config),
+   UCS_CONFIG_TYPE_ULUNITS},
 
   {"MAX_RMA_LANES", NULL, "",
    ucs_offsetof(ucp_context_config_t, max_rma_lanes), UCS_CONFIG_TYPE_UINT},
@@ -2367,6 +2372,23 @@ static double ucp_context_get_protov1_memcpy_bw()
                    UCP_CPU_EST_BCOPY_BW_DEFAULT_PROTOV1;
 }
 
+static ucs_status_t
+ucp_context_resolve_rndv_config(ucp_context_config_t *config)
+{
+    if (config->max_rndv_lanes_config == UCS_ULUNITS_AUTO) {
+        config->max_rndv_lanes = UCP_MAX_RNDV_LANES_DEFAULT;
+    } else if ((config->max_rndv_lanes_config == UCS_ULUNITS_INF) ||
+               (config->max_rndv_lanes_config > UINT_MAX)) {
+        ucs_error("maximum number of rendezvous lanes must not exceed %u",
+                  UINT_MAX);
+        return UCS_ERR_INVALID_PARAM;
+    } else {
+        config->max_rndv_lanes = config->max_rndv_lanes_config;
+    }
+
+    return UCS_OK;
+}
+
 static int
 ucp_dynamic_tl_switch_config_valid(const ucp_context_config_t *config)
 {
@@ -2404,6 +2426,11 @@ static ucs_status_t ucp_fill_config(ucp_context_h context,
         goto err;
     }
 
+    status = ucp_context_resolve_rndv_config(&context->config.ext);
+    if (status != UCS_OK) {
+        goto err_free_config_ext;
+    }
+
     if (context->config.ext.estimated_num_eps != UCS_ULUNITS_AUTO) {
         /* num_eps was set via the env variable. Override current value */
         context->config.est_num_eps = context->config.ext.estimated_num_eps;

src/ucp/core/ucp_context.h

@@ -60,6 +60,9 @@ typedef enum {
 } ucp_reg_devices_mode_t;
 
 
+#define UCP_MAX_RNDV_LANES_DEFAULT 2
+
+
 static UCS_F_ALWAYS_INLINE ucp_reg_devices_mode_t
 ucp_reg_devices_mode(unsigned long max_hca_per_gpu)
 {
@@ -150,6 +153,8 @@ typedef struct ucp_context_config {
     int                                    adaptive_progress;
     /** Eager-am multi-lane support */
     unsigned                               max_eager_lanes;
+    /** Raw rendezvous lane limit before resolving the auto value */
+    unsigned long                          max_rndv_lanes_config;
     /** Rendezvous-get multi-lane support */
     unsigned                               max_rndv_lanes;
     /** RMA multi-lane support */

src/ucp/rndv/rndv_get.c

@@ -17,6 +17,57 @@
 
 #define UCP_PROTO_RNDV_GET_DESC "read from remote"
 
+static ucp_lane_index_t
+ucp_proto_rndv_get_max_lanes(const ucp_proto_init_params_t *init_params)
+{
+    ucp_context_t *context                 = init_params->worker->context;
+    const ucp_ep_config_key_t *ep_key      = init_params->ep_config_key;
+    ucp_lane_index_t max_lanes             = context->config.ext.max_rndv_lanes;
+    const uct_iface_attr_t *iface_attr;
+    ucp_lane_index_t lane;
+
+    if (context->config.ext.max_rndv_lanes_config != UCS_ULUNITS_AUTO) {
+        return max_lanes;
+    }
+
+    for (lane = 0; lane < ep_key->num_lanes; ++lane) {
+        if (!(ep_key->lanes[lane].lane_types &
+              UCS_BIT(UCP_LANE_TYPE_RMA_BW))) {
+            continue;
+        }
+
+        iface_attr = ucp_proto_common_get_iface_attr(init_params, lane);
+        if (!(iface_attr->cap.flags & UCT_IFACE_FLAG_GET_ZCOPY)) {
+            continue;
+        }
+
+        max_lanes = ucs_max(max_lanes,
+                            ucs_min(iface_attr->dev_num_paths,
+                                    UCP_PROTO_MAX_LANES));
+    }
+
+    return max_lanes;
+}
+
+static void
+ucp_proto_rndv_get_add_proto(ucp_proto_multi_init_params_t *params)
+{
+    ucp_proto_rndv_bulk_priv_t rpriv;
+    ucp_proto_perf_t *perf;
+    ucs_status_t status;
+
+    status = ucp_proto_rndv_bulk_init(params, UCP_PROTO_RNDV_GET_DESC,
+                                      UCP_PROTO_RNDV_ATS_NAME, &perf, &rpriv);
+    if (status != UCS_OK) {
+        return;
+    }
+
+    ucp_proto_select_add_proto(&params->super.super, params->super.cfg_thresh,
+                               params->super.cfg_priority, perf, &rpriv,
+                               UCP_PROTO_MULTI_EXTENDED_PRIV_SIZE(&rpriv,
+                                                                  mpriv));
+}
+
 static void
 ucp_proto_rndv_get_common_probe(const ucp_proto_init_params_t *init_params,
                                 uint64_t rndv_modes, size_t max_length,
@@ -26,6 +77,8 @@ ucp_proto_rndv_get_common_probe(const ucp_proto_init_params_t *init_params,
                                 const ucp_memory_info_t *reg_mem_info)
 {
     ucp_context_t *context               = init_params->worker->context;
+    ucp_lane_index_t max_lanes           =
+            ucp_proto_rndv_get_max_lanes(init_params);
     ucp_proto_multi_init_params_t params = {
         .super.super         = *init_params,
         .super.cfg_thresh    = ucp_proto_rndv_cfg_thresh(context, rndv_modes),
@@ -58,25 +111,18 @@ ucp_proto_rndv_get_common_probe(const ucp_proto_init_params_t *init_params,
         .opt_align_offs      = ucs_offsetof(uct_iface_attr_t,
                                             cap.get.opt_zcopy_align),
     };
-    ucp_proto_rndv_bulk_priv_t rpriv;
-    ucp_proto_perf_t *perf;
-    ucs_status_t status;
 
     if (!ucp_proto_rndv_op_check(init_params, UCP_OP_ID_RNDV_RECV,
                                  support_ppln)) {
         return;
     }
 
-    status = ucp_proto_rndv_bulk_init(&params, UCP_PROTO_RNDV_GET_DESC,
-                                      UCP_PROTO_RNDV_ATS_NAME, &perf, &rpriv);
-    if (status != UCS_OK) {
-        return;
-    }
+    ucp_proto_rndv_get_add_proto(&params);
 
-    ucp_proto_select_add_proto(&params.super.super, params.super.cfg_thresh,
-                               params.super.cfg_priority, perf, &rpriv,
-                               UCP_PROTO_MULTI_EXTENDED_PRIV_SIZE(&rpriv,
-                                                                  mpriv));
+    if (max_lanes > params.max_lanes) {
+        params.max_lanes = max_lanes;
+        ucp_proto_rndv_get_add_proto(&params);
+    }
 }
 
 static UCS_F_ALWAYS_INLINE ucs_status_t

src/uct/ib/base/ib_device.c

@@ -160,7 +160,7 @@ static uct_ib_device_spec_t uct_ib_builtin_device_specs[] = {
    UCT_IB_DEVICE_FLAG_DC_V2, 80},
   {"ConnectX-9", {0x15b3, 4133},
    UCT_IB_DEVICE_FLAG_MELLANOX | UCT_IB_DEVICE_FLAG_MLX5_PRM |
-   UCT_IB_DEVICE_FLAG_DC_V2, 90},
+   UCT_IB_DEVICE_FLAG_DC_V2 | UCT_IB_DEVICE_FLAG_XDR_READ_4_PATHS, 90},
   {"ConnectX-10", {0x15b3, 4135},
    UCT_IB_DEVICE_FLAG_MELLANOX | UCT_IB_DEVICE_FLAG_MLX5_PRM |
    UCT_IB_DEVICE_FLAG_DC_V2, 100},

src/uct/ib/base/ib_device.h

@@ -90,17 +90,18 @@ typedef enum uct_ib_roce_version {
 
 
 enum {
-    UCT_IB_DEVICE_FLAG_MLX4_PRM = UCS_BIT(1),   /* Device supports mlx4 PRM */
-    UCT_IB_DEVICE_FLAG_MLX5_PRM = UCS_BIT(2),   /* Device supports mlx5 PRM */
-    UCT_IB_DEVICE_FLAG_MELLANOX = UCS_BIT(3),   /* Mellanox device */
-    UCT_IB_DEVICE_FLAG_SRQ      = UCS_BIT(4),   /* Supports SRQ */
-    UCT_IB_DEVICE_FLAG_LINK_IB  = UCS_BIT(5),   /* Require only IB */
-    UCT_IB_DEVICE_FLAG_DC_V1    = UCS_BIT(6),   /* Device supports DC ver 1 */
-    UCT_IB_DEVICE_FLAG_DC_V2    = UCS_BIT(7),   /* Device supports DC ver 2 */
-    UCT_IB_DEVICE_FLAG_AV       = UCS_BIT(8),   /* Device supports compact AV */
-    UCT_IB_DEVICE_FLAG_DC       = UCT_IB_DEVICE_FLAG_DC_V1 |
-                                  UCT_IB_DEVICE_FLAG_DC_V2, /* Device supports DC */
-    UCT_IB_DEVICE_FAILED        = UCS_BIT(9)    /* Got fatal error */
+    UCT_IB_DEVICE_FLAG_MLX4_PRM         = UCS_BIT(1),   /* Device supports mlx4 PRM */
+    UCT_IB_DEVICE_FLAG_MLX5_PRM         = UCS_BIT(2),   /* Device supports mlx5 PRM */
+    UCT_IB_DEVICE_FLAG_MELLANOX         = UCS_BIT(3),   /* Mellanox device */
+    UCT_IB_DEVICE_FLAG_SRQ              = UCS_BIT(4),   /* Supports SRQ */
+    UCT_IB_DEVICE_FLAG_LINK_IB          = UCS_BIT(5),   /* Require only IB */
+    UCT_IB_DEVICE_FLAG_DC_V1            = UCS_BIT(6),   /* Device supports DC ver 1 */
+    UCT_IB_DEVICE_FLAG_DC_V2            = UCS_BIT(7),   /* Device supports DC ver 2 */
+    UCT_IB_DEVICE_FLAG_AV               = UCS_BIT(8),   /* Device supports compact AV */
+    UCT_IB_DEVICE_FLAG_DC               = UCT_IB_DEVICE_FLAG_DC_V1 |
+                                          UCT_IB_DEVICE_FLAG_DC_V2, /* Device supports DC */
+    UCT_IB_DEVICE_FAILED                = UCS_BIT(9),   /* Got fatal error */
+    UCT_IB_DEVICE_FLAG_XDR_READ_4_PATHS = UCS_BIT(10) /* XDR: 4 read paths */
 };
 
 

src/uct/ib/base/ib_iface.c

@@ -38,6 +38,7 @@
  */
 #define UCT_IB_NDR_READ_PATH_BANDWIDTH 38e9
 #define UCT_IB_XDR_READ_PATH_BANDWIDTH 35e9
+#define UCT_IB_XDR_READ_NUM_PATHS      4
 
 /**
  * Minimal NDR single path ratio.
@@ -191,7 +192,8 @@ ucs_config_field_t uct_ib_iface_config_table[] = {
    "to the port link layer:\n"
    " RoCE       - "UCS_PP_MAKE_STRING(UCT_IB_DEV_MAX_PORTS) " for LAG port, otherwise - 1.\n"
    " InfiniBand - As the number of path bits enabled by fabric's LMC value and selected\n"
-   "              by "UCS_DEFAULT_ENV_PREFIX UCT_IB_CONFIG_PREFIX"LID_PATH_BITS configuration.",
+   "              by "UCS_DEFAULT_ENV_PREFIX UCT_IB_CONFIG_PREFIX"LID_PATH_BITS configuration.\n"
+   "              ConnectX-9 XDR ports expose four paths for RDMA read bandwidth.",
    ucs_offsetof(uct_ib_iface_config_t, num_paths), UCS_CONFIG_TYPE_ULUNITS},
 
   {"ROCE_LOCAL_SUBNET", "n",
@@ -1400,6 +1402,8 @@ static unsigned uct_ib_iface_roce_lag_level(uct_ib_iface_t *iface)
 static void uct_ib_iface_set_num_paths(uct_ib_iface_t *iface,
                                        const uct_ib_iface_config_t *config)
 {
+    const uct_ib_device_spec_t *dev_spec;
+
     if (config->num_paths == UCS_ULUNITS_AUTO) {
         if (uct_ib_iface_is_roce(iface)) {
             /* RoCE - number of paths is RoCE LAG level */
@@ -1410,6 +1414,13 @@ static void uct_ib_iface_set_num_paths(uct_ib_iface_t *iface,
             iface->num_paths = iface->path_bits_count;
         }
 
+        dev_spec = uct_ib_device_spec(uct_ib_iface_device(iface));
+        if (uct_ib_iface_port_is_xdr(iface) &&
+            (dev_spec->flags & UCT_IB_DEVICE_FLAG_XDR_READ_4_PATHS)) {
+            iface->num_paths = ucs_max(iface->num_paths,
+                                       UCT_IB_XDR_READ_NUM_PATHS);
+        }
+
         if ((iface->num_paths == 1) &&
             (uct_ib_iface_port_active_speed(iface) >= UCT_IB_SPEED_NDR)) {
             iface->num_paths = 2;

test/gtest/ucp/test_ucp_context.cc

@@ -72,6 +72,54 @@ UCS_TEST_P(test_ucp_context, max_hca_per_gpu_config)
     EXPECT_EQ(2u, ucp_reg_devices_count(e->ucph()->config.ext.max_hca_per_gpu));
 }
 
+UCS_TEST_P(test_ucp_context, rndv_lane_default)
+{
+    entity *e = create_entity();
+
+    EXPECT_EQ(UCS_ULUNITS_AUTO,
+              e->ucph()->config.ext.max_rndv_lanes_config);
+    EXPECT_EQ(UCP_MAX_RNDV_LANES_DEFAULT,
+              e->ucph()->config.ext.max_rndv_lanes);
+}
+
+UCS_TEST_P(test_ucp_context, rndv_lane_config_override)
+{
+    modify_config("MAX_RNDV_RAILS", "3");
+
+    entity *e = create_entity();
+    EXPECT_EQ(3ul, e->ucph()->config.ext.max_rndv_lanes_config);
+    EXPECT_EQ(3u, e->ucph()->config.ext.max_rndv_lanes);
+}
+
+UCS_TEST_P(test_ucp_context, rndv_lane_alias_override)
+{
+    modify_config("MAX_RNDV_LANES", "3");
+
+    entity *e = create_entity();
+    EXPECT_EQ(3ul, e->ucph()->config.ext.max_rndv_lanes_config);
+    EXPECT_EQ(3u, e->ucph()->config.ext.max_rndv_lanes);
+}
+
+UCS_TEST_P(test_ucp_context, rndv_lane_config_invalid_inf)
+{
+    ucs::handle<ucp_config_t*> config;
+    UCS_TEST_CREATE_HANDLE(ucp_config_t*, config, ucp_config_release,
+                           ucp_config_read, NULL, NULL);
+
+    ASSERT_UCS_OK(ucp_config_modify(config.get(), "MAX_RNDV_RAILS", "inf"));
+
+    ucp_context_h ucph;
+    ucs_status_t status;
+    {
+        scoped_log_handler slh(hide_errors_logger);
+        status = ucp_init(&get_variant_ctx_params(), config.get(), &ucph);
+    }
+    EXPECT_EQ(UCS_ERR_INVALID_PARAM, status);
+    if (status == UCS_OK) {
+        ucp_cleanup(ucph);
+    }
+}
+
 UCP_INSTANTIATE_TEST_CASE_TLS(test_ucp_context, all, "all")
 
 class test_ucp_aliases : public test_ucp_context {