Catch outliers, save raw latency data for debugging

Author: askervin

metrics/scaling/k8s_scale_nc.sh

@@ -22,7 +22,7 @@ nc_req_msg_len=${nc_req_msg_len:-1000}
 nc_port=33101
 # request message
 nc_req_msg=$(head -c $nc_req_msg_len /dev/zero | tr  '\0' 'x')
-nc_percentiles=(5 25 50 75 95)
+nc_percentiles=(0 1 5 25 50 75 95 99 100)
 
 pod_command="[\"nc\", \"-lk\", \"-p\", \"${nc_port}\", \"-e\", \"/bin/cat\"]"
 
@@ -339,6 +339,7 @@ run() {
 		sleep ${settle_time}
 
 		if [[ ${nc_reqs_per_pod} -ge 1 ]]; then
+			local latency_raw_output="$RESULT_DIR/${TEST_NAME// /-}.t_pods_round_ip_latency_fail_rx.raw"
 			pod_ips=$(kubectl get pods --selector ${LABEL}=${LABELVALUE} -o json | jq -r '.items[].status.podIP')
 			if [[ ${reqs} != $(echo $pod_ips | wc -w) ]]; then
 				info "WARNING: pod IP count mismatch expected ${reqs} found $(echo $pod_ips | wc -w)"
@@ -348,12 +349,17 @@ run() {
 			local latency_pod_array=()
 			for latency_round in $(seq ${nc_reqs_per_pod}); do
 				for pod_ip in ${pod_ips}; do
+					local latency_failed=0
 					local latency_pod_start_time=$(date +%s%N)
-					if [[ $(echo ${nc_req_msg} | nc ${pod_ip} ${nc_port}) != "${nc_req_msg}" ]]; then
+					local latency_pod_response=$(echo ${nc_req_msg} | nc ${pod_ip} ${nc_port})
+					local latency_pod_end_time=$(date +%s%N)
+					local latency_response_microseconds=$(( (latency_pod_end_time - latency_pod_start_time) / 1000 ))
+					if [[ "$latency_pod_response" != "${nc_req_msg}" ]]; then
 						latency_failures=$(( latency_failures + 1 ))
+						latency_failed=1
 					fi
-					local latency_pod_end_time=$(date +%s%N)
-					latency_pod_array+=($(( (latency_pod_end_time - latency_pod_start_time) / 1000000 )))
+					latency_pod_array+=($latency_response_microseconds)
+					echo "$latency_pod_start_time $reqs $latency_round $pod_ip $latency_response_microseconds $latency_failed $(echo $latency_pod_response | wc -c)" >> $latency_raw_output
 				done
 			done
 			IFS=$'\n'