Perftest: correct rdma_cm header output and enforce --ib-dev

When using --rdma_cm, perftest printed the user-requested device,
GID index, and local GID rather than what rdma_cm actually resolved.
Additionally, --ib-dev was silently ignored since rdma_cm relies on
kernel IP routing to pick the device.

- Bind rdma_cm to a source IP from the --ib-dev device so route
  resolution goes through the correct interface
- After connection, update user_param with the actual device, port,
  and GID index from the resolved cm_id

Signed-off-by: Shmuel Shaul <sshaul@nvidia.com>

Author: sshaulnv

src/perftest_communication.c

@@ -3097,6 +3097,100 @@ int rdma_cm_client_connection(struct pingpong_context *ctx,
 	return error_handler(error_message);
 }
 
+/******************************************************************************
+ * Convert a sockaddr (IPv4 or IPv6) to a 16-byte GID suitable for
+ * comparison against the device's GID table.  Returns 0 on success.
+ ******************************************************************************/
+static int sockaddr_to_gid(const struct sockaddr *addr, union ibv_gid *gid)
+{
+	memset(gid, 0, sizeof(*gid));
+
+	if (addr->sa_family == AF_INET) {
+		const struct sockaddr_in *s4 = (const struct sockaddr_in *)addr;
+		gid->raw[10] = 0xff;
+		gid->raw[11] = 0xff;
+		memcpy(&gid->raw[12], &s4->sin_addr, 4);
+		return 0;
+	}
+
+	if (addr->sa_family == AF_INET6) {
+		const struct sockaddr_in6 *s6 = (const struct sockaddr_in6 *)addr;
+		memcpy(gid->raw, &s6->sin6_addr, 16);
+		return 0;
+	}
+
+	return -1;
+}
+
+/******************************************************************************
+ * Scan the GID table on the given device/port and return the index whose
+ * entry matches 'target'.  Returns the index (>= 0) or -1 if not found.
+ ******************************************************************************/
+static int find_gid_index(struct ibv_context *context, uint8_t port,
+			  const union ibv_gid *target)
+{
+	struct ibv_port_attr port_attr;
+	union ibv_gid entry;
+	int i;
+
+	if (ibv_query_port(context, port, &port_attr))
+		return -1;
+
+	for (i = 0; i < port_attr.gid_tbl_len; i++) {
+		if (ibv_query_gid(context, port, i, &entry))
+			continue;
+		if (!memcmp(entry.raw, target->raw, sizeof(entry.raw)))
+			return i;
+	}
+
+	return -1;
+}
+
+/******************************************************************************
+ * After rdma_cm connection is established, the actual device, port, and GID
+ * may differ from what the user requested via --ib-dev / -x.  Refresh
+ * user_param so that subsequent header prints reflect reality.
+ ******************************************************************************/
+static void update_rdma_cm_params(struct pingpong_context *ctx,
+				  struct perftest_parameters *user_param,
+				  struct perftest_comm *comm)
+{
+	struct rdma_cm_id *cm_id;
+	struct sockaddr *src_addr;
+	union ibv_gid src_gid;
+	int idx;
+
+	cm_id = ctx->cma_master.nodes[0].cma_id;
+	if (!cm_id || !cm_id->verbs)
+		return;
+
+	free(user_param->ib_devname);
+	user_param->ib_devname = strdup(ibv_get_device_name(cm_id->verbs->device));
+
+	/* Similarly, the physical port may differ from what the user assumed. */
+	user_param->ib_port = cm_id->port_num;
+
+	/* Find the GID index that matches the source address rdma_cm actually
+	 * used: convert the resolved local sockaddr to a GID, then look it up
+	 * in the device's GID table. */
+	src_addr = rdma_get_local_addr(cm_id);
+	if (!src_addr)
+		return;
+
+	if (sockaddr_to_gid(src_addr, &src_gid))
+		return;
+
+	idx = find_gid_index(ctx->context, user_param->ib_port, &src_gid);
+	if (idx >= 0) {
+		user_param->gid_index = idx;
+		user_param->use_gid_user = 1;
+		/* comm->rdma_params is a separate copy used by
+		 * ctx_print_pingpong_data() — keep it in sync. */
+		if (comm && comm->rdma_params)
+			comm->rdma_params->gid_index = idx;
+	}
+}
+
 /******************************************************************************
 *
 ******************************************************************************/
@@ -3138,7 +3232,6 @@ int create_rdma_cm_connection(struct pingpong_context *ctx,
 
 	if (rc) {
 		error_message = "Failed to create RDMA CM connection.";
-		free(hints.ai_src_addr);
 		goto destroy_rdma_id;
 	}
 
@@ -3151,6 +3244,8 @@ int create_rdma_cm_connection(struct pingpong_context *ctx,
 
 	free(hints.ai_src_addr);
 
+	update_rdma_cm_params(ctx, user_param, comm);
+
 	return rc;
 
 

src/perftest_resources.c

@@ -6803,6 +6803,65 @@ int run_iter_fs(struct pingpong_context *ctx, struct perftest_parameters *user_p
 }
 
 
+/******************************************************************************
+*
+******************************************************************************/
+static int gid_is_v4mapped(const union ibv_gid *gid)
+{
+	return (gid->raw[10] == 0xff && gid->raw[11] == 0xff &&
+		!gid->raw[0] && !gid->raw[1] && !gid->raw[2] &&
+		!gid->raw[3] && !gid->raw[4] && !gid->raw[5] &&
+		!gid->raw[6] && !gid->raw[7] && !gid->raw[8] &&
+		!gid->raw[9]);
+}
+
+/******************************************************************************
+*
+******************************************************************************/
+static int gid_to_sockaddr(const union ibv_gid *gid, int ai_family,
+			   struct sockaddr **addr, socklen_t *len)
+{
+	if (ai_family == AF_INET && gid_is_v4mapped(gid)) {
+		struct sockaddr_in *s4;
+
+		/* Skip 0.0.0.0 */
+		if (!gid->raw[12] && !gid->raw[13] &&
+		    !gid->raw[14] && !gid->raw[15])
+			return -1;
+
+		s4 = (struct sockaddr_in *)calloc(1, sizeof(*s4));
+		s4->sin_family = AF_INET;
+		memcpy(&s4->sin_addr, &gid->raw[12], 4);
+		*addr = (struct sockaddr *)s4;
+		*len = sizeof(*s4);
+		return 0;
+	}
+
+	if (ai_family == AF_INET6 && !gid_is_v4mapped(gid)) {
+		struct sockaddr_in6 *s6;
+		int j, all_zero = 1;
+
+		for (j = 0; j < 16; j++) {
+			if (gid->raw[j]) { all_zero = 0; break; }
+		}
+		if (all_zero)
+			return -1;
+
+		/* Skip link-local (fe80::/10) — not routable across subnets */
+		if (gid->raw[0] == 0xfe && (gid->raw[1] & 0xc0) == 0x80)
+			return -1;
+
+		s6 = (struct sockaddr_in6 *)calloc(1, sizeof(*s6));
+		s6->sin6_family = AF_INET6;
+		memcpy(&s6->sin6_addr, gid->raw, 16);
+		*addr = (struct sockaddr *)s6;
+		*len = sizeof(*s6);
+		return 0;
+	}
+
+	return -1;
+}
+
 /******************************************************************************
 *
 ******************************************************************************/
@@ -6881,6 +6940,30 @@ int rdma_cm_allocate_nodes(struct pingpong_context *ctx,
 			hints->ai_src_addr = (struct sockaddr *)(source_addr);
 			hints->ai_src_len = sizeof(*source_addr);
 		}
+	} else if (user_param->ib_devname) {
+		/* No --bind_source_ip, but --ib-dev was specified.
+		* rdma_cm ignores --ib-dev when resolving routes, so find an IP
+		* on that device and pass it as the source address to bind the interface. */
+		struct ibv_port_attr port_attr;
+		union ibv_gid gid;
+
+		if (!ibv_query_port(ctx->context, user_param->ib_port,
+				    &port_attr)) {
+			int idx;
+
+			for (idx = 0; idx < port_attr.gid_tbl_len; idx++) {
+				if (ibv_query_gid(ctx->context,
+						  user_param->ib_port,
+						  idx, &gid))
+					continue;
+
+				if (!gid_to_sockaddr(&gid,
+						     user_param->ai_family,
+						     &hints->ai_src_addr,
+						     &hints->ai_src_len))
+					break;
+			}
+		}
 	}
 
 	return rc;