Skip to content

UCP/CORE: Print debugging tables on context and ep creation - v1.22.x #11640

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
guy-ealey-morag wants to merge 1 commit into
base: v1.22.x
Choose a base branch
from
Open

UCP/CORE: Print debugging tables on context and ep creation - v1.22.x #11640

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions src/ucp/Makefile.am
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ noinst_HEADERS = \
am/ucp_am.inl \
core/ucp_am.h \
core/ucp_context.h \
core/ucp_tl_info.h \
core/ucp_ep.h \
core/ucp_ep.inl \
core/ucp_ep_vfs.h \
Expand Down Expand Up @@ -85,6 +86,7 @@ noinst_HEADERS = \
wireup/ep_match.h \
wireup/wireup_ep.h \
wireup/wireup.h \
wireup/wireup_lane_info.h \
wireup/wireup_cm.h \
stream/stream.h

Expand All @@ -108,6 +110,7 @@ libucp_la_SOURCES = \
am/eager_multi.c \
am/rndv.c \
core/ucp_context.c \
core/ucp_tl_info.c \
core/ucp_am.c \
core/ucp_ep.c \
core/ucp_ep_vfs.c \
Expand Down Expand Up @@ -177,4 +180,5 @@ libucp_la_SOURCES = \
wireup/select.c \
wireup/wireup_ep.c \
wireup/wireup.c \
wireup/wireup_lane_info.c \
wireup/wireup_cm.c
90 changes: 63 additions & 27 deletions src/ucp/core/ucp_context.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@

#include "ucp_context.h"
#include "ucp_request.h"
#include "ucp_tl_info.h"

#include <ucs/config/parser.h>
#include <ucs/algorithm/qsort_r.h>
Expand Down Expand Up @@ -622,6 +623,15 @@ static ucs_config_field_t ucp_context_config_table[] = {
"resulting performance.",
ucs_offsetof(ucp_context_config_t, node_local_id), UCS_CONFIG_TYPE_ULUNITS},

{"PRINT_TRANSPORT_TABLES", "auto",
"Print tables of available transports/devices and per-endpoint lane\n"
"configuration during initialization. The value is interpreted as follows:\n"
" 'y' : Always print the tables\n"
" 'n' : Never print the tables\n"
" 'auto' : Print the tables when UCX_LOG_LEVEL is 'debug' or higher",
ucs_offsetof(ucp_context_config_t, print_transport_tables),
UCS_CONFIG_TYPE_ON_OFF_AUTO},

{NULL}
};

Expand Down Expand Up @@ -731,7 +741,6 @@ static ucs_config_field_t ucp_config_table[] = {
};
UCS_CONFIG_DECLARE_TABLE(ucp_config_table, "UCP context", NULL, ucp_config_t)


static ucp_tl_alias_t ucp_tl_aliases[] = {
{ "mm", { "posix", "sysv", "xpmem", NULL } }, /* for backward compatibility */
{ "sm", { "posix", "sysv", "xpmem", "knem", "cma", NULL } },
Expand Down Expand Up @@ -1293,22 +1302,12 @@ static int ucp_tl_resource_is_same_device(const uct_tl_resource_desc_t *resource
}

static ucs_status_t
ucp_add_tl_resource_if_enabled(ucp_context_h context, ucp_md_index_t md_index,
const ucp_config_t *config,
const ucs_string_set_t *aux_tls,
const uct_tl_resource_desc_t *resource,
unsigned *num_resources_p,
uint64_t dev_cfg_masks[], uint64_t *tl_cfg_mask)
ucp_add_tl_resource(ucp_context_h context, ucp_md_index_t md_index,
const uct_tl_resource_desc_t *resource, uint8_t rsc_flags)
{
ucp_tl_md_t *md = &context->tl_mds[md_index];
uint8_t rsc_flags;
ucp_rsc_index_t dev_index, i;

if (!ucp_is_resource_enabled(resource, config, aux_tls, &rsc_flags,
dev_cfg_masks, tl_cfg_mask)) {
return UCS_OK;
}

if (context->num_tls >= UCP_MAX_RESOURCES) {
ucs_error("exceeded transports/devices limit (up to %d are supported)",
UCP_MAX_RESOURCES);
Expand Down Expand Up @@ -1347,7 +1346,6 @@ ucp_add_tl_resource_if_enabled(ucp_context_h context, ucp_md_index_t md_index,
}

++context->num_tls;
++(*num_resources_p);

return UCS_OK;
}
Expand All @@ -1358,13 +1356,15 @@ ucp_add_tl_resources(ucp_context_h context, ucp_md_index_t md_index,
const ucs_string_set_t *aux_tls, unsigned *num_resources_p,
ucs_string_set_t avail_devices[],
ucs_string_set_t *avail_tls, uint64_t dev_cfg_masks[],
uint64_t *tl_cfg_mask)
uint64_t *tl_cfg_mask, ucp_tl_info_array_t *all_rscs)
{
ucp_tl_md_t *md = &context->tl_mds[md_index];
ucp_tl_md_t *md = &context->tl_mds[md_index];
ucp_tl_info_entry_t *added_rscs = NULL;
unsigned num_tl_resources, all_rscs_prev_len;
uct_tl_resource_desc_t *tl_resources;
ucp_tl_resource_desc_t *tmp;
unsigned num_tl_resources;
ucs_status_t status;
uint8_t rsc_flags;
unsigned i;

*num_resources_p = 0;
Expand All @@ -1381,6 +1381,25 @@ ucp_add_tl_resources(ucp_context_h context, ucp_md_index_t md_index,
goto free_resources;
}

/* Collect the full (pre-filter) resource list for the transport tables */
if (ucp_context_print_transport_tables_enabled(context)) {
all_rscs_prev_len = ucs_array_length(all_rscs);
if (ucs_array_reserve(all_rscs, all_rscs_prev_len + num_tl_resources) !=
UCS_OK) {
ucs_warn("failed to allocate transport info entries, "
"table will be incomplete");
} else {
ucs_array_set_length(all_rscs,
all_rscs_prev_len + num_tl_resources);
added_rscs = &ucs_array_elem(all_rscs, all_rscs_prev_len);
for (i = 0; i < num_tl_resources; ++i) {
added_rscs[i].rsc = tl_resources[i];
added_rscs[i].cmpt_index = md->cmpt_index;
added_rscs[i].enabled = 0;
}
}
}

tmp = ucs_realloc(context->tl_rscs,
sizeof(*context->tl_rscs) *
(context->num_tls + num_tl_resources),
Expand All @@ -1393,7 +1412,7 @@ ucp_add_tl_resources(ucp_context_h context, ucp_md_index_t md_index,

/* print configuration */
for (i = 0; i < config->tls.array.count; ++i) {
ucs_trace("allowed transport %d : '%s'", i, config->tls.array.names[i]);
ucs_trace("allowed transport %u : '%s'", i, config->tls.array.names[i]);
}

/* copy only the resources enabled by user configuration */
Expand All @@ -1403,13 +1422,23 @@ ucp_add_tl_resources(ucp_context_h context, ucp_md_index_t md_index,
"'%s'(%s)", tl_resources[i].dev_name,
context->tl_cmpts[md->cmpt_index].attr.name);
ucs_string_set_add(avail_tls, tl_resources[i].tl_name);
status = ucp_add_tl_resource_if_enabled(context, md_index, config,
aux_tls, &tl_resources[i],
num_resources_p, dev_cfg_masks,
tl_cfg_mask);

if (!ucp_is_resource_enabled(&tl_resources[i], config, aux_tls,
&rsc_flags, dev_cfg_masks, tl_cfg_mask)) {
continue;
}

status = ucp_add_tl_resource(context, md_index, &tl_resources[i],
rsc_flags);
if (status != UCS_OK) {
goto free_resources;
}

++(*num_resources_p);

if (added_rscs != NULL) {
added_rscs[i].enabled = 1;
}
}

status = UCS_OK;
Expand Down Expand Up @@ -1731,7 +1760,8 @@ ucp_add_component_resources(ucp_context_h context, ucp_rsc_index_t cmpt_index,
ucs_string_set_t *avail_tls,
uint64_t dev_cfg_masks[], uint64_t *tl_cfg_mask,
const ucp_config_t *config,
const ucs_string_set_t *aux_tls)
const ucs_string_set_t *aux_tls,
ucp_tl_info_array_t *all_rscs)
{
const ucp_tl_cmpt_t *tl_cmpt = &context->tl_cmpts[cmpt_index];
size_t avail_mds = config->max_component_mds;
Expand Down Expand Up @@ -1780,7 +1810,8 @@ ucp_add_component_resources(ucp_context_h context, ucp_rsc_index_t cmpt_index,
/* Add communication resources of each MD */
status = ucp_add_tl_resources(context, md_index, config, aux_tls,
&num_tl_resources, avail_devices,
avail_tls, dev_cfg_masks, tl_cfg_mask);
avail_tls, dev_cfg_masks, tl_cfg_mask,
all_rscs);
if (status != UCS_OK) {
uct_md_close(context->tl_mds[md_index].md);
goto out;
Expand Down Expand Up @@ -2106,9 +2137,10 @@ static void ucp_fill_resources_reg_md_map_update(ucp_context_h context)
}
}

static ucs_status_t ucp_fill_resources(ucp_context_h context,
const ucp_config_t *config)
static ucs_status_t
ucp_fill_resources(ucp_context_h context, const ucp_config_t *config)
{
ucp_tl_info_array_t all_rscs = UCS_ARRAY_DYNAMIC_INITIALIZER;
uint64_t dev_cfg_masks[UCT_DEVICE_TYPE_LAST] = {};
uint64_t tl_cfg_mask = 0;
ucs_string_set_t avail_devices[UCT_DEVICE_TYPE_LAST];
Expand Down Expand Up @@ -2214,7 +2246,8 @@ static ucs_status_t ucp_fill_resources(ucp_context_h context,
for (i = 0; i < context->num_cmpts; ++i) {
status = ucp_add_component_resources(context, i, avail_devices,
&avail_tls, dev_cfg_masks,
&tl_cfg_mask, config, &aux_tls);
&tl_cfg_mask, config, &aux_tls,
&all_rscs);
if (status != UCS_OK) {
goto err_free_resources;
}
Expand Down Expand Up @@ -2257,7 +2290,10 @@ static ucs_status_t ucp_fill_resources(ucp_context_h context,

ucp_fill_sockaddr_prio_list(context, config);

ucp_context_log_tl_info(context, &all_rscs);

out_release_components:
ucs_array_cleanup_dynamic(&all_rscs);
uct_release_component_list(uct_components);
out_cleanup_avail_devices:
UCS_STATIC_ASSERT(UCT_DEVICE_TYPE_NET == 0);
Expand Down
17 changes: 15 additions & 2 deletions src/ucp/core/ucp_context.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -259,6 +259,9 @@ typedef struct ucp_context_config {
unsigned long max_hca_per_gpu;
/** Local identificator on a single node */
unsigned long node_local_id;
/** Print transport/device info and lane info tables during context
* and endpoint initialization */
ucs_on_off_auto_value_t print_transport_tables;
} ucp_context_config_t;


Expand Down Expand Up @@ -644,8 +647,6 @@ extern const char *ucp_feature_str[];
void ucp_dump_payload(ucp_context_h context, char *buffer, size_t max,
const void *data, size_t length);

void ucp_context_tag_offload_enable(ucp_context_h context);

void ucp_context_uct_atomic_iface_flags(ucp_context_h context,
ucp_tl_iface_atomic_flags_t *atomic);

Expand Down Expand Up @@ -784,6 +785,18 @@ ucp_context_rndv_is_enabled(ucp_context_h context)
(context->config.ext.rndv_inter_thresh != UCS_MEMUNITS_INF);
}

static UCS_F_ALWAYS_INLINE int
ucp_context_print_transport_tables_enabled(ucp_context_h context)
{
ucs_on_off_auto_value_t value = context->config.ext.print_transport_tables;

if (value == UCS_CONFIG_AUTO) {
return ucs_log_is_enabled(UCS_LOG_LEVEL_DEBUG);
}

return value == UCS_CONFIG_ON;
}

void ucp_context_memaccess_tl_bitmap(ucp_context_h context,
uint64_t mem_type_bitmap,
uint64_t md_reg_flags,
Expand Down
Loading