CI: Stop MAD perftest server before printing its log

A failed run leaves behind a zombie server container: docker_stop_srv
cats the log before docker stop, and when the log is missing the cat
aborts the script (set -e) before the stop runs. Being a detached
container, the zombie survives the build and keeps the MAD agent
registration on the HCA, so every following build's server fails with
EINVAL and the client hangs until the job timeout - the stage breaks
for all PRs until the zombie is removed by hand.

Stop the container first and sweep zombie ucx_perftest_* containers
before starting a new server, so one bad run cannot wedge the stage.

Author: Alexey-Rivkin

buildlib/tools/test_mad.sh

@@ -36,6 +36,9 @@ docker_run_srv() {
     local test_name="$1"
     HCA=$(detect_active_ib_hca)
     sudo chmod 777 /dev/infiniband/umad*
+    # A stale server from a previous run holds the MAD agent registration
+    # on the HCA and makes every new server fail with EINVAL
+    docker ps -aq --filter "name=ucx_perftest_" | xargs -r docker rm -f
     docker run \
         --rm \
         --detach \
@@ -53,8 +56,9 @@ docker_run_srv() {
 
 docker_stop_srv() {
     local test_name="$1"
-    cat "${PWD}/ucx_perf_srv_${test_name}_${BUILD_BUILDID}.log"
+    # Stop first so a missing log cannot leave a zombie server container
     docker stop ucx_perftest_"$BUILD_BUILDID" || true
+    cat "${PWD}/ucx_perf_srv_${test_name}_${BUILD_BUILDID}.log" || true
 }
 
 set_vars() {