UCP/TEST: Add cuda_ipc tests in ucp and uct gtests

Author: shasson5

test/gtest/ucp/test_ucp_proto_mock.cc

@@ -948,6 +948,75 @@ UCS_TEST_P(test_ucp_proto_mock_gpu, cuda_managed_ppln_host_frag,
 UCP_INSTANTIATE_TEST_CASE_TLS(test_ucp_proto_mock_gpu, rcx_gpu,
                               "rc_x,cuda,rocm")
 
+class test_ucp_proto_mock_cuda_ipc : public test_ucp_proto_mock {
+public:
+    test_ucp_proto_mock_cuda_ipc()
+    {
+        mock_transport("rc_mlx5");
+    }
+
+    virtual void init() override
+    {
+        if (!mem_buffer::is_mem_type_supported(UCS_MEMORY_TYPE_CUDA)) {
+            UCS_TEST_SKIP_R("CUDA memory is not supported");
+        }
+
+        add_mock_iface("mock", [](uct_iface_attr_t &iface_attr) {
+            iface_attr.bandwidth.shared = 40e9;
+            iface_attr.latency.c        = 400e-9;
+            iface_attr.latency.m        = 1e-9;
+        });
+
+        test_ucp_proto_mock::init();
+    }
+
+    ucp_worker_cfg_index_t find_rkey_cfg_index(ucs_memory_type_t mem_type)
+    {
+        ucp_worker_h worker = sender().worker();
+        ucp_proto_select_key_t sel_key;
+        ucp_rkey_config_t *rkey_config;
+
+        /* Find rkey config with required mem_type for local and remote 
+         * memory */
+        for (auto i = 0; i < ucs_array_length(&worker->rkey_config); ++i) {
+            rkey_config = &ucs_array_elem(&worker->rkey_config, i);
+
+            if (rkey_config->key.mem_type != mem_type) {
+                continue;
+            }
+
+            kh_foreach_key(rkey_config->proto_select.hash, sel_key.u64, {
+                if (sel_key.param.mem_type == mem_type) {
+                    return i;
+                }
+            })
+        }
+
+        return UCP_WORKER_CFG_INDEX_NULL;
+    }
+};
+
+UCS_TEST_P(test_ucp_proto_mock_cuda_ipc, put_cuda, "IB_NUM_PATHS?=1")
+{
+    send_recv_rma_put(UCS_MBYTE, UCS_MEMORY_TYPE_CUDA);
+
+    ucp_proto_select_key_t key = any_key();
+    key.param.op_id_flags      = UCP_OP_ID_PUT;
+    key.param.op_attr          = 0;
+    key.param.mem_type         = UCS_MEMORY_TYPE_CUDA;
+
+    auto rkey_cfg_index = find_rkey_cfg_index(UCS_MEMORY_TYPE_CUDA);
+    ASSERT_NE(rkey_cfg_index, UCP_WORKER_CFG_INDEX_NULL);
+
+    check_rkey_config(sender(), {
+        {0, 0,    "short",   "rc_mlx5/mock"},
+        {1, INF, "zero-copy", "cuda_ipc/cuda"},
+    }, key, rkey_cfg_index);
+}
+
+UCP_INSTANTIATE_TEST_CASE_TLS_GPU_AWARE(test_ucp_proto_mock_cuda_ipc,
+                                        shm_ib_ipc, "shm,ib,cuda_ipc,rocm_ipc")
+
 class test_ucp_proto_mock_rcx_twins : public test_ucp_proto_mock {
 public:
     test_ucp_proto_mock_rcx_twins()

test/gtest/uct/test_uct_ep.cc

@@ -282,4 +282,3 @@ UCS_TEST_SKIP_COND_P(test_uct_ep, destroy_entity_after_send,
 }
 
 UCT_INSTANTIATE_TEST_CASE(test_uct_ep)
-UCT_INSTANTIATE_CUDA_IPC_TEST_CASE(test_uct_ep)

test/gtest/uct/test_uct_iface.cc

@@ -93,4 +93,3 @@ UCS_TEST_P(test_uct_iface, is_reachable)
 }
 
 UCT_INSTANTIATE_TEST_CASE(test_uct_iface)
-UCT_INSTANTIATE_CUDA_IPC_TEST_CASE(test_uct_iface)

test/gtest/uct/uct_p2p_test.cc

@@ -177,7 +177,9 @@ void uct_p2p_test::test_xfer_multi_mem_type(send_func_t send, size_t min_length,
 
     /* Trim at the max allocation available. Divide by 2 for
        2 buffers + 0.5 for spare capacity */
-    max_length = ucs_min(max_length, sender().md_attr().max_alloc / 2.5);
+    if (sender().md_attr().max_alloc > 0) {
+        max_length = ucs_min(max_length, sender().md_attr().max_alloc / 2.5);
+    }
 
     /* Trim at 4.1 GB */
     max_length = ucs_min(max_length, (size_t)(4.1 * (double)UCS_GBYTE));

test/gtest/uct/uct_test.h

@@ -502,7 +502,8 @@ class test_uct_iface_attrs : public uct_test {
 
 #define UCT_TEST_CUDA_MEM_TYPE_TLS \
     cuda_copy,              \
-    gdr_copy
+    gdr_copy,               \
+    cuda_ipc
 
 #define UCT_TEST_ROCM_MEM_TYPE_TLS \
     rocm_copy
@@ -511,18 +512,16 @@ class test_uct_iface_attrs : public uct_test {
     UCT_TEST_NO_SELF_TLS, \
     self
 
-#define UCT_TEST_TLS \
-    UCT_TEST_NO_GPU_MEM_TYPE_TLS, \
-    UCT_TEST_ROCM_MEM_TYPE_TLS, \
-    UCT_TEST_CUDA_MEM_TYPE_TLS
 
 /**
  * Instantiate the parametrized test case for all transports.
  *
  * @param _test_case  Test case class, derived from uct_test.
  */
 #define UCT_INSTANTIATE_TEST_CASE(_test_case) \
-    UCS_PP_FOREACH(_UCT_INSTANTIATE_TEST_CASE, _test_case, UCT_TEST_TLS)
+    UCS_PP_FOREACH(_UCT_INSTANTIATE_TEST_CASE, _test_case, UCT_TEST_NO_GPU_MEM_TYPE_TLS) \
+    UCS_PP_FOREACH(_UCT_INSTANTIATE_TEST_CASE, _test_case, UCT_TEST_ROCM_MEM_TYPE_TLS) \
+    UCS_PP_FOREACH(_UCT_INSTANTIATE_TEST_CASE, _test_case, UCT_TEST_CUDA_MEM_TYPE_TLS)
 #define _UCT_INSTANTIATE_TEST_CASE(_test_case, _tl_name) \
     INSTANTIATE_TEST_SUITE_P(_tl_name, _test_case, \
                             testing::ValuesIn(_test_case::enum_resources(UCS_PP_QUOTE(_tl_name))));
@@ -618,16 +617,6 @@ class test_uct_iface_attrs : public uct_test {
     _UCT_INSTANTIATE_TEST_CASE(_test_case, gga_mlx5)
 
 
-/**
- * Instantiate the parametrized test case for CUDA_IPC.
- * TODO: add cuda_ipc to UCT_INSTANTIATE_TEST_CASE.
- *
- * @param _test_case  Test case class, derived from uct_test.
- */
-#define UCT_INSTANTIATE_CUDA_IPC_TEST_CASE(_test_case) \
-    _UCT_INSTANTIATE_TEST_CASE(_test_case, cuda_ipc)
-
-
 /**
  * Instantiate the parametrized test case for the SRD transports
  *