Retry network fetches across CI so a transient blip does not fail a job

Author: aidangarske

.github/scripts/git-clone-retry.sh

@@ -0,0 +1,21 @@
+#!/bin/sh
+# git clone with retry; removes the target dir (last arg) between tries so a partial clone cannot block the retry.
+set -u
+target=""
+for target in "$@"; do :; done
+case "$target" in http*|git@*|ssh://*|"") target="" ;; esac
+tries="${RETRY_TRIES:-5}"
+delay="${RETRY_DELAY:-10}"
+i=1
+while true; do
+    [ -n "$target" ] && rm -rf "$target"
+    git clone "$@" && exit 0
+    rc=$?
+    if [ "$i" -ge "$tries" ]; then
+        echo "git clone failed after $tries attempts (rc=$rc)" >&2
+        exit "$rc"
+    fi
+    echo "git clone attempt $i/$tries failed (rc=$rc); retrying in ${delay}s" >&2
+    i=$((i + 1))
+    sleep "$delay"
+done

.github/scripts/retry.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+# Retry a re-runnable command (downloads, docker build) up to RETRY_TRIES times.
+set -u
+tries="${RETRY_TRIES:-5}"
+delay="${RETRY_DELAY:-10}"
+i=1
+while true; do
+    "$@" && exit 0
+    rc=$?
+    if [ "$i" -ge "$tries" ]; then
+        echo "retry: '$*' failed after $tries attempts (rc=$rc)" >&2
+        exit "$rc"
+    fi
+    echo "retry: '$*' failed (rc=$rc); attempt $i/$tries, retrying in ${delay}s" >&2
+    i=$((i + 1))
+    sleep "$delay"
+done

.github/scripts/se050-run.sh

@@ -12,18 +12,19 @@ mkdir -p "$WORK"
 cd "$WORK"
 
 echo "::group::SE050Sim (Rust TCP server)"
-git clone -q --depth 1 https://github.com/wolfSSL/simulators sims
+"$ROOT/.github/scripts/git-clone-retry.sh" -q --depth 1 https://github.com/wolfSSL/simulators sims
 # se050-sim depends on the se050 crate at ../nxp-se050/se050 (its own repo)
-git clone -q --branch sim-compat --depth 1 \
+"$ROOT/.github/scripts/git-clone-retry.sh" -q --branch sim-compat --depth 1 \
   https://github.com/LinuxJedi/nxp-se050.git sims/SE050Sim/nxp-se050
 ( cd sims/SE050Sim/se050-sim && cargo build --release --bin tcp_server )
 SIM="$WORK/sims/SE050Sim/wolfcrypt-test"
 echo "::endgroup::"
 
 echo "::group::plug-and-trust SDK + wolfSSL HostCrypto patch + sim overlays"
-git clone -q --depth 1 --branch v04.07.01 https://github.com/NXP/plug-and-trust simw-top
-curl -fsSL https://raw.githubusercontent.com/wolfSSL/osp/master/nxp-se05x-middleware/simw-top-v040701.patch \
-  | ( cd simw-top && patch -p1 -l --forward --fuzz=3 )
+"$ROOT/.github/scripts/git-clone-retry.sh" -q --depth 1 --branch v04.07.01 https://github.com/NXP/plug-and-trust simw-top
+"$ROOT/.github/scripts/retry.sh" curl -fsSL -o "$WORK/osp-se050.patch" \
+  https://raw.githubusercontent.com/wolfSSL/osp/master/nxp-se05x-middleware/simw-top-v040701.patch
+( cd simw-top && patch -p1 -l --forward --fuzz=3 < "$WORK/osp-se050.patch" )
 test -f simw-top/sss/src/wolfssl/fsl_sss_wolfssl_apis.c
 cp "$SIM/i2c_a7.c"      simw-top/hostlib/hostLib/platform/linux/i2c_a7.c
 cp "$SIM/se05x_reset.c" simw-top/hostlib/hostLib/platform/rsp/se05x_reset.c
@@ -32,7 +33,7 @@ cp "$SIM/CMakeLists.txt" simw-top/CMakeLists.txt
 echo "::endgroup::"
 
 echo "::group::wolfSSL pass A (no se050, so the SDK can link -lwolfssl)"
-git clone -q --depth 1 https://github.com/wolfSSL/wolfssl
+"$ROOT/.github/scripts/git-clone-retry.sh" -q --depth 1 https://github.com/wolfSSL/wolfssl wolfssl
 ( cd wolfssl && ./autogen.sh >/dev/null 2>&1 && \
   ./configure --enable-keygen --enable-cmac \
     CFLAGS="-DWOLFSSL_SE050_NO_TRNG -DSIZEOF_LONG_LONG=8" >/dev/null && \

.github/workflows/analysis.yml

@@ -68,7 +68,7 @@ jobs:
       - name: Build wolfSSL for ${{ matrix.mode }}
         run: |
           set -euo pipefail
-          git clone -q --depth 1 --branch master https://github.com/wolfSSL/wolfssl /tmp/wolfssl
+          bash "$GITHUB_WORKSPACE/.github/scripts/git-clone-retry.sh" -q --depth 1 --branch master https://github.com/wolfSSL/wolfssl /tmp/wolfssl
           cd /tmp/wolfssl
           ./autogen.sh >/dev/null
           if [ "${{ matrix.mode }}" = asan ]; then

.github/workflows/android.yml

@@ -62,7 +62,7 @@ jobs:
           test -e wolfssl/.git || { echo "FAIL: wolfssl submodule not checked out"; exit 1; }
           # Replace the tree: fetching into the shallow submodule races on .git/modules/*/shallow
           rm -rf wolfssl
-          git clone -q --depth 1 --branch '${{ matrix.wolfssl_ref }}' \
+          bash "$GITHUB_WORKSPACE/.github/scripts/git-clone-retry.sh" -q --depth 1 --branch '${{ matrix.wolfssl_ref }}' \
             https://github.com/wolfSSL/wolfssl.git wolfssl
           echo "wolfssl now at ${{ matrix.wolfssl_ref }}: $(git -C wolfssl rev-parse --short HEAD)"
 

.github/workflows/bsdkm.yml

@@ -59,13 +59,23 @@ jobs:
             if [ ! -f /usr/src/sys/conf/kmod.mk ]; then
               # keyed off the running kernel, so this cannot drift from the VM
               rel=$(uname -r | sed 's/-p[0-9]*$//')
-              fetch -o /tmp/src.txz "https://download.freebsd.org/releases/amd64/${rel}/src.txz"
+              f=1; while :; do
+                fetch -o /tmp/src.txz "https://download.freebsd.org/releases/amd64/${rel}/src.txz" && break
+                [ "$f" -ge 5 ] && { echo "FAIL: src.txz fetch failed"; exit 1; }
+                f=$((f + 1)); sleep 10
+              done
               tar -C / -xf /tmp/src.txz
             fi
             test -f /usr/src/sys/conf/kmod.mk
 
-            git clone -q --depth 1 --branch '${{ matrix.wolfssl_ref }}' \
-              https://github.com/wolfSSL/wolfssl /tmp/wolfssl
+            # FreeBSD VM has no bash, so retry the clone with a POSIX sh loop
+            n=1; while :; do
+              rm -rf /tmp/wolfssl
+              git clone -q --depth 1 --branch '${{ matrix.wolfssl_ref }}' \
+                https://github.com/wolfSSL/wolfssl /tmp/wolfssl && break
+              [ "$n" -ge 5 ] && { echo "FAIL: wolfssl clone failed"; exit 1; }
+              n=$((n + 1)); sleep 10
+            done
             cd /tmp/wolfssl
             ./autogen.sh
             ./configure --enable-freebsdkm --enable-cryptonly \

.github/workflows/cmake.yml

@@ -46,7 +46,7 @@ jobs:
       - name: Fetch the wolfSSL ref under test
         run: |
           set -euo pipefail
-          git clone -q --depth 1 --branch '${{ matrix.wolfssl_ref }}' \
+          bash "$GITHUB_WORKSPACE/.github/scripts/git-clone-retry.sh" -q --depth 1 --branch '${{ matrix.wolfssl_ref }}' \
             https://github.com/wolfSSL/wolfssl cmake/wolfssl
           git -C cmake/wolfssl log -1 --format='wolfssl at ${{ matrix.wolfssl_ref }}: %h %s'
 

.github/workflows/config-matrix.yml

@@ -72,7 +72,7 @@ jobs:
       - name: Build wolfSSL (${{ matrix.name }})
         run: |
           set -euo pipefail
-          git clone -q --depth 1 --branch master https://github.com/wolfSSL/wolfssl /tmp/wolfssl
+          bash "$GITHUB_WORKSPACE/.github/scripts/git-clone-retry.sh" -q --depth 1 --branch master https://github.com/wolfSSL/wolfssl /tmp/wolfssl
           cd /tmp/wolfssl
           ./autogen.sh >/dev/null
           ./configure --enable-ecc --enable-ed25519 --enable-aesgcm --enable-keygen ${{ matrix.opts }} \

.github/workflows/cross-arch.yml

@@ -69,7 +69,7 @@ jobs:
       - name: Cross-build wolfSSL (${{ matrix.name }})
         run: |
           set -euo pipefail
-          git clone -q --depth 1 --branch master https://github.com/wolfSSL/wolfssl /tmp/wolfssl
+          bash "$GITHUB_WORKSPACE/.github/scripts/git-clone-retry.sh" -q --depth 1 --branch master https://github.com/wolfSSL/wolfssl /tmp/wolfssl
           cd /tmp/wolfssl
           ./autogen.sh >/dev/null
           host=""

.github/workflows/csharp.yml

@@ -55,7 +55,7 @@ jobs:
       - name: Build wolfSSL with the C# wrapper's user_settings.h
         run: |
           set -euo pipefail
-          git clone -q --depth 1 --branch '${{ matrix.wolfssl_ref }}' \
+          bash "$GITHUB_WORKSPACE/.github/scripts/git-clone-retry.sh" -q --depth 1 --branch '${{ matrix.wolfssl_ref }}' \
             https://github.com/wolfSSL/wolfssl /tmp/wolfssl
           cd /tmp/wolfssl
           # wrapper/CSharp/user_settings.h already enables ML-KEM, ML-DSA and