CI/FT: Parse protocol configuration to find active devices

Author: william-gallagher-nv

contrib/test_jenkins.sh

@@ -404,44 +404,55 @@ run_io_demo() {
 }
 
 #
-# Tail and parse ucx_perftest stdout to extract current throughput
-# 
-# Requires ucx_perftest be run with "-v" to generate output in the following form: 
-# [thread 0]273598,21.660,21.940,21.930,45578.82,45599.20,45579,45599
+# Parse the protocol configuration info in ucx_perftest stdout to get active
+# devices 
 #
-get_perftest_throughput() {
+parse_active_devices()
+{
     file=$1
-    timeout_seconds=$2
-    samples=$3
-    sum=0
-    count=0
-    start_time=$(date +%s)
+    config_cnt=$2
 
-    tail -n 0 -F -- "$file" 2>/dev/null |
     while IFS= read -r line;
 	do
-        current_time=$(date +%s)
-        elapsed_time=$((current_time - start_time))
-
-        if (( elapsed_time >= timeout_seconds ));
-		then
-			echo "0"
-            return 0 
+        [[ $line == *"| perftest self cfg#$config_cnt "* ]] || continue
+
+        IFS= read -r line || return 1  # | remote memory write by ucp_put* from host memory to host                                            |
+        IFS= read -r line || return 1  # +--------+-------------+------------------------------------------------------------------------------+
+        IFS= read -r line || return 1  # | Range  | Description |                                    Config                                    |
+        IFS= read -r line || return 1  # +--------+-------------+------------------------------------------------------------------------------+
+        IFS= read -r line || return 1  # | 0..inf | zero-copy   | 34% on rc_mlx5/mlx5_0:1, 34% on rc_mlx5/mlx5_2:1 and 32% on rc_mlx5/mlx5_1:1 |
+        break
+    done < "$file"
+
+    for word in $line; do
+        if [[ $word == */*:* ]]; then
+            device=${word#*/}
+            device=${device%%:*}
+            echo "$device"
         fi
+    done
+}
 
-        [[ $line == \[thread* ]] || continue
+#
+# Verify expected devices match the active device list
+#
+verify_active_devices() {
+    file=$1
+    config_cnt=$2
+    expected=$3
 
-        throughput=$(awk -F',' '{ print int($5) }' <<< "$line")
+    actual_sorted=$(parse_active_devices "$file" "$config_cnt" | sort | paste -sd' ')
+    expected_sorted=$(echo "$expected" | tr ' ' '\n' | sort | paste -sd' ')
 
-        sum=$((sum + throughput))
-        count=$((count + 1))
+    if [[ "$actual_sorted" != "$expected_sorted" ]];
+	then
+        echo "Error: configuration ${config_cnt} device mismatch"
+        echo "  expected: ${expected_sorted}"
+        echo "  actual:   ${actual_sorted}"
+        return 1
+    fi
 
-        if (( count == samples ));
-		then
-            echo $((sum / count))
-            return 0
-        fi
-    done
+    return 0
 }
 
 #
@@ -466,38 +477,31 @@ run_ucx_perftest_fault_tolerance() {
 
 	echo "==== Running UCX Perftest Fault Tolerance Test ===="
 
-	echo "Executor Number is ${EXECUTOR_NUMBER}" || true
-
 	test_exe="${ucx_inst}/bin/ucx_perftest"
-	# Temporary location for drop tool
 	drop_tool="/hpc/scrap/fault_tolerance/ucx_drop_tool"
-	output_file=ucx_perftest_ft.stdout
-	samples=10
-	verify_timeout=20
+	output_file=ucx_perftest.stdout
 	settle_seconds=10
-	drop_ratio=95
 	sl=$(( EXECUTOR_NUMBER + 1 ))
-	echo "Using service level ${sl}"
 	tc=$(( EXECUTOR_NUMBER + 31 ))
-	echo "Using traffic class ${tc}"
 	devices=$(get_active_ib_devnames)
 	device_cnt=$(echo ${devices} | wc -w)
 	background_pids=()
-	block_cnt=0
+	config_cnt=0
 	affinity=$(slice_affinity 0)
-
-	#if [[ ! -x "${drop_tool}" ]];
-	#then
-	#	echo "Error: Could not find drop tool at ${drop_tool}"
-	#	return 1
-	#fi
+	expected_devices="${devices}"
 
 	if [[ "${devices}" == "" ]];
 	then
 		echo "No IB devices detected. Skipping Fault Tolerance test"
 		return 0 
 	fi
 
+	if [[ ! -x "${drop_tool}" ]];
+	then
+		echo "Warning: Could not find drop tool at ${drop_tool}"
+		return 0 
+	fi
+
 	export UCX_IB_SL=${sl}
 	export UCX_IB_TRAFFIC_CLASS=${tc}
 	export UCX_ZCOPY_THRESH=0
@@ -509,90 +513,45 @@ run_ucx_perftest_fault_tolerance() {
 	export UCX_MAX_RMA_RAILS=${device_cnt}
 	export UCX_TLS=rc_x 
 
-	args="-t ucp_put_bw     \
+	args="-t ucp_put_lat    \
 		-n 1000000000000000 \
 		-s 1048576          \
 		-e failover         \
-		-l                  \
-		-v"
+		-l"
 
 	taskset -c ${affinity} ${test_exe} ${args} 2>&1 | tee ${output_file} &
-	perftest_pid=($!)
-	sleep 60
-	kill -9 ${perftest_pid}
-
-	export UCX_NET_DEVICES="mlx5_0"
-	export UCX_MAX_RMA_RAILS=1
-	taskset -c ${affinity} ${test_exe} ${args} 2>&1 | tee ${output_file} &
-	perftest_pid=($!)
-	sleep 60
-	kill -9 ${perftest_pid}
+	background_pids+=($!)
 
-	export UCX_NET_DEVICES="mlx5_1"
-	export UCX_MAX_RMA_RAILS=1
-	taskset -c ${affinity} ${test_exe} ${args} 2>&1 | tee ${output_file} &
-	perftest_pid=($!)
-	sleep 60
-	kill -9 ${perftest_pid}
+	sleep ${settle_seconds} 
 
-	if [[ ! -x "${drop_tool}" ]];
+	if ! verify_active_devices ${output_file} ${config_cnt} "${devices}";
 	then
-		echo "Error: Could not find drop tool at ${drop_tool}"
+		kill_background_processes "${background_pids[@]}"
 		return 1
 	fi
 
-	export UCX_IB_SL=${sl}
-	export UCX_IB_TRAFFIC_CLASS=${tc}
-	export UCX_ZCOPY_THRESH=0
-	export UCX_MIN_RMA_CHUNK_SIZE=64
-	export UCX_PROTO_INFO=y
-	export UCX_RC_MLX5_RETRY_COUNT=2
-	export UCX_IB_NUM_PATHS=1
-	export UCX_NET_DEVICES=$(get_active_ib_devnames | paste -sd,)
-	export UCX_MAX_RMA_RAILS=${device_cnt}
-	export UCX_TLS=rc_x 
-
-	taskset -c ${affinity} ${test_exe} ${args} 2>&1 | tee ${output_file} &
-	background_pids+=($!)
-
-	sleep ${settle_seconds} 
-
-	curr_tput=$(get_perftest_throughput ${output_file} ${verify_timeout} ${samples})
-	echo "Baseline throughput: ${curr_tput}"
-	prev_tput=${curr_tput}
+	echo "Baseline protocol configuration uses devices: ${devices}"
 
 	# Loop through active IB devices. Block each one in turn and verify traffic
-	# continues to flow at reduced rate.
 	for dev in $(get_active_ib_devnames)
 	do
 		echo "Blocking traffic with service level ${sl} or traffic class ${tc} on device ${dev}..."
 		${drop_tool} -d ${dev} -s ${sl} -t ${tc} &
 		background_pids+=($!)
-		block_cnt=$((block_cnt + 1))
+		config_cnt=$((config_cnt + 1))
 
 		sleep ${settle_seconds}
 
- 		threshold=$((prev_tput * drop_ratio / 100))
-   		curr_tput=$(get_perftest_throughput ${output_file} ${verify_timeout} ${samples})
+		expected_devices=$(echo "$expected_devices" | tr ' ' '\n' | grep -vFx "$dev" | paste -sd' ')
 
-		if (( block_cnt < device_cnt ));
+		if ! verify_active_devices ${output_file} ${config_cnt} "${expected_devices}";
 		then
-			if (( curr_tput <= 0 || curr_tput > threshold ));
-			then
-  	     		echo "Error: throughput not in expected range after blocking ${dev}, tput=${curr_tput}, threshold=${threshold}"
-  		      	kill_background_processes "${background_pids[@]}"
-  	   	   		return 1
-    		fi
-    		prev_tput=${curr_tput}
+			kill_background_processes "${background_pids[@]}"
+			return 1
 		fi
-	done
 
-	if (( curr_tput != 0 ));
-	then
-		echo "Error: traffic still flowing when all are devices blocked!"
-		kill_background_processes "${background_pids[@]}"
-		return 1
-	fi
+		echo "Reconfiguration ${config_cnt} uses devices: ${expected_devices}"
+	done
 
 	kill_background_processes "${background_pids[@]}"
 
@@ -1448,12 +1407,6 @@ run_configure_tests() {
 run_tests() {
 	export UCX_PROTO_REQUEST_RESET=y
 
-	# temp debug to find drop tool binary...
-	ls / || true
-	ls /hpc/ || true
-	ls /hpc/scrap/ || true
-	ls /hpc/scrap/fault_tolerance/ || true
-
 	# all are running mpi tests
 	run_mpi_tests