test retry configurable

Author: maureeungaro

.github/workflows/sanitize.yml

@@ -5,6 +5,8 @@ permissions:
 
 env:
   FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true
+  # Re-run the test suite up to this many times before declaring failure.
+  GEMC_TEST_MAX_ATTEMPTS: 3
 
 concurrency:
   group: sanitize-${{ github.workflow }}-${{ github.ref }}

.github/workflows/test.yml

@@ -7,6 +7,8 @@ permissions:
 
 env:
   FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true
+  # Re-run the test suite up to this many times before declaring failure.
+  GEMC_TEST_MAX_ATTEMPTS: 3
 
 concurrency:
   group: gemc-test-${{ github.ref }}

ci/env.sh

@@ -89,25 +89,29 @@ function meson_setup_options {
     echo "${args[@]}"
 }
 
-# Run any command, retrying once on failure. Appends output to $test_log.
+# Run any command, retrying on failure. Appends output to $test_log.
+# The number of attempts is GEMC_TEST_MAX_ATTEMPTS (default 3), so a command is
+# re-run up to that many times until it succeeds.
 function run_command_with_retry {
   local label="$1"
   shift
 
-  for attempt in 1 2; do
-    echo " > Running ${label} (attempt ${attempt}/2):" "$@" | tee -a "$test_log"
+  local max_attempts="${GEMC_TEST_MAX_ATTEMPTS:-3}"
+
+  for (( attempt = 1; attempt <= max_attempts; attempt++ )); do
+    echo " > Running ${label} (attempt ${attempt}/${max_attempts}):" "$@" | tee -a "$test_log"
     "$@" >> "$test_log"
     local exit_code=$?
 
     if [ $exit_code -eq 0 ]; then
       return 0
     fi
 
-    if [ $attempt -eq 2 ]; then
+    if [ $attempt -eq $max_attempts ]; then
       return $exit_code
     fi
 
-    echo " > ${label} failed; retrying once" | tee -a "$test_log"
+    echo " > ${label} failed; retrying (attempt $((attempt + 1))/${max_attempts})" | tee -a "$test_log"
   done
 }