CI: Make MAD RTE perftest teardown leak-proof

A failed test run leaks the detached perftest server container:
docker_stop_srv cats the log before docker stop, and when TestLid fails
the GUID log is never created, so the cat aborts the script before the
stop. The leaked server keeps the MAD agent registration on the HCA, so
every following build's server dies with EINVAL and the client hangs
until the 10-minute job timeout - all PRs go red until someone removes
the zombie by hand.

Stop the container before printing logs, sweep stale ucx_perftest_*
containers before starting a new server, verify the detached server is
actually alive so the failure surfaces in Setup Server with the real
error, and bound the client with a timeout.

Author: Alexey-Rivkin

buildlib/pr/mad_tests.yml

@@ -73,7 +73,7 @@ jobs:
       - checkout: none
       - bash: |
           source ./buildlib/tools/test_mad.sh
-          docker_stop_srv LID
+          docker_stop_srv
           docker_run_srv GUID
         displayName: Server Restart
 
@@ -109,5 +109,5 @@ jobs:
       - checkout: none
       - bash: |
           source ./buildlib/tools/test_mad.sh
-          docker_stop_srv GUID
+          docker_stop_srv
         displayName: Server Stop

buildlib/tools/test_mad.sh

@@ -34,8 +34,12 @@ build_ucx_in_docker() {
 
 docker_run_srv() {
     local test_name="$1"
+    local srv_log="${PWD}/ucx_perf_srv_${test_name}_${BUILD_BUILDID}.log"
     HCA=$(detect_active_ib_hca)
     sudo chmod 777 /dev/infiniband/umad*
+    # A stale server from a previous run holds the MAD agent registration
+    # on the HCA and makes every new server fail with EINVAL
+    docker ps -aq --filter "name=ucx_perftest_" | xargs -r docker rm -f
     docker run \
         --rm \
         --detach \
@@ -48,13 +52,20 @@ docker_run_srv() {
         --ulimit memlock=-1:-1 --device=/dev/infiniband/ \
         $IMAGE \
         bash -c "${PWD}/install/bin/ucx_perftest -K ${HCA} > \
-            ${PWD}/ucx_perf_srv_${test_name}_${BUILD_BUILDID}.log 2>&1"
+            ${srv_log} 2>&1"
+    # Fail here with the server error instead of hanging the client job
+    sleep 5
+    if [ -z "$(docker ps -q --filter "name=ucx_perftest_${BUILD_BUILDID}")" ]; then
+        echo "Error: perftest server failed to start"
+        cat "$srv_log" || true
+        exit 1
+    fi
 }
 
 docker_stop_srv() {
-    local test_name="$1"
-    cat "${PWD}/ucx_perf_srv_${test_name}_${BUILD_BUILDID}.log"
-    docker stop ucx_perftest_"$BUILD_BUILDID" || true
+    # Stop first so the server never leaks; print whatever logs exist
+    docker rm -f ucx_perftest_"$BUILD_BUILDID" || true
+    cat "${PWD}"/ucx_perf_srv_*_"${BUILD_BUILDID}".log || true
 }
 
 set_vars() {
@@ -77,7 +88,8 @@ set_vars() {
 run_mad_test() {
     local ib_address="$1"
     sudo chmod 777 /dev/infiniband/umad*
-    "$PWD"/install/bin/ucx_perftest -t tag_bw -e -K "$HCA" -e "$ib_address"
+    # Bounded wait: a dead server must fail the step, not stall the job
+    timeout 300 "$PWD"/install/bin/ucx_perftest -t tag_bw -e -K "$HCA" -e "$ib_address"
 }
 
 detect_active_ib_hca() {
@@ -93,7 +105,7 @@ detect_active_ib_hca() {
     }
     END {
       if (!found) {
-        echo "Error: No active InfiniBand interface found"
+        print "Error: No active InfiniBand interface found" > "/dev/stderr"
         exit 1
       }
     }