Some scripts improvements (#299)

Author: tellet-q

tools/hetzner/setup_hetzner.sh

@@ -11,7 +11,9 @@ apt-get install -y \
     ca-certificates \
     curl \
     gnupg \
-    lsb-release
+    jq \
+    lsb-release \
+    rsync
 
 mkdir -p /etc/apt/keyrings
 

tools/run_client_script.sh

@@ -34,6 +34,7 @@ GHCR_PASSWORD=${GHCR_PASSWORD:-""}
 GHCR_USERNAME=${GHCR_USERNAME:-""}
 
 if [[ "$EXPERIMENT_MODE" == "snapshot" ]]; then
+  ssh_with_retry -o ServerAliveInterval=10 -o ServerAliveCountMax=10 "${SERVER_USERNAME}@${IP_OF_THE_CLIENT}" "rm -rf ~/configurations ~/datasets.json"
   scp_with_retry "${SCRIPT_PATH}/run_experiment.sh" "${SERVER_USERNAME}@${IP_OF_THE_CLIENT}:~/run_experiment_snapshot.sh"
   scp_with_retry "${SCRIPT_PATH}/../datasets/datasets.json" "${SERVER_USERNAME}@${IP_OF_THE_CLIENT}:~/datasets.json"
   scp_with_retry -r "${SCRIPT_PATH}/../experiments/configurations" "${SERVER_USERNAME}@${IP_OF_THE_CLIENT}:~/configurations"
@@ -51,6 +52,7 @@ if [[ "$EXPERIMENT_MODE" == "snapshot" ]]; then
   ssh_with_retry -tt -o ServerAliveInterval=120 -o ServerAliveCountMax=10 "${SERVER_USERNAME}@${IP_OF_THE_CLIENT}" "${RUN_EXPERIMENT}"
 
 else
+  ssh_with_retry -o ServerAliveInterval=10 -o ServerAliveCountMax=10 "${SERVER_USERNAME}@${IP_OF_THE_CLIENT}" "rm -rf ~/configurations ~/datasets.json"
   scp_with_retry "${SCRIPT_PATH}/run_experiment.sh" "${SERVER_USERNAME}@${IP_OF_THE_CLIENT}:~/run_experiment.sh"
   scp_with_retry "${SCRIPT_PATH}/../datasets/datasets.json" "${SERVER_USERNAME}@${IP_OF_THE_CLIENT}:~/datasets.json"
   scp_with_retry -r "${SCRIPT_PATH}/../experiments/configurations" "${SERVER_USERNAME}@${IP_OF_THE_CLIENT}:~/configurations"

tools/run_experiment.sh

@@ -12,7 +12,11 @@ if [[ -n "${GHCR_PASSWORD}" ]] || [[ "${VECTOR_DB_BENCHMARK_IMAGE}" == ghcr.io/*
     echo "GHCR_PASSWORD and GHCR_USERNAME is required to pull images from ghcr.io"
     exit 1
   fi
-  echo "${GHCR_PASSWORD}" | docker login ghcr.io -u "${GHCR_USERNAME}" --password-stdin
+  for i in 1 2 3; do
+    echo "${GHCR_PASSWORD}" | docker login ghcr.io -u "${GHCR_USERNAME}" --password-stdin && break
+    echo "docker login attempt ${i} failed, retrying in $((i * 10))s..."
+    sleep $((i * 10))
+  done
 fi
 
 ENGINE_NAME=${ENGINE_NAME:-"qdrant-continuous-benchmark"}

tools/run_remote_benchmark.sh

@@ -11,7 +11,8 @@ cleanup() {
   #  bash -x "${SCRIPT_PATH}/tear_down.sh"
 }
 
-trap 'echo signal received!; kill $(jobs -p); wait; cleanup' SIGINT SIGTERM
+# Guarded kill/wait: no-op cleanly when there are no bg jobs (otherwise `set -e` trips on Ctrl-C). EXIT trap below calls cleanup once.
+trap 'echo signal received!; bg=$(jobs -p); [ -n "$bg" ] && kill $bg 2>/dev/null; wait 2>/dev/null || true' SIGINT SIGTERM
 
 CLOUD_NAME=${CLOUD_NAME:-"hetzner"}