fix(ci): pre-clone nargo external git deps with retry to survive DNS flakes

Author: AztecBot

noir-projects/aztec-nr/bootstrap.sh

@@ -13,6 +13,10 @@ else
 fi
 
 function build {
+  # Pre-clone external nargo git deps with retry, to survive transient DNS
+  # hiccups that otherwise fail a single merge-queue-heavy shard.
+  ../scripts/prefetch_nargo_git_deps.sh
+
   # Being a library, aztec-nr does not technically need to be built. But we can still run nargo check to find any type
   # errors and prevent warnings
   echo_stderr "Checking aztec-nr for warnings..."

noir-projects/bootstrap.sh

@@ -9,6 +9,7 @@ function build {
   # Also doubles up as our formatting check.
   function prep {
     set -eu
+    ./scripts/prefetch_nargo_git_deps.sh
     (cd noir-protocol-circuits && yarn && node ./scripts/generate_variants.js)
     for dir in noir-contracts noir-protocol-circuits mock-protocol-circuits aztec-nr; do
       (cd $dir && ../../noir/noir-repo/target/release/nargo fmt --check)

noir-projects/noir-contracts/bootstrap.sh

@@ -137,6 +137,10 @@ export -f compile
 # If given an argument, it's the contract to compile.
 # Otherwise parse out all relevant contracts from the root Nargo.toml and process them in parallel.
 function build {
+  # Pre-clone external nargo git deps with retry, to survive transient DNS
+  # hiccups that otherwise fail a single merge-queue-heavy shard.
+  ../scripts/prefetch_nargo_git_deps.sh
+
   echo_stderr "Compiling contracts (bb-hash: $BB_HASH)..."
   local folder_name
   if [ -n "${DOCS_WORKING_DIR:-}" ]; then

noir-projects/noir-protocol-circuits/bootstrap.sh

@@ -149,6 +149,10 @@ export -f hex_to_fields_json compile
 function build {
   set -eu
 
+  # Pre-clone external nargo git deps with retry, to survive transient DNS
+  # hiccups that otherwise fail a single merge-queue-heavy shard.
+  ../scripts/prefetch_nargo_git_deps.sh
+
   # If pinned-build.tar.gz exists, use it instead of compiling.
   if [ -f pinned-build.tar.gz ]; then
     echo_stderr "Using pinned-build.tar.gz instead of compiling."

noir-projects/scripts/prefetch_nargo_git_deps.sh

@@ -0,0 +1,63 @@
+#!/usr/bin/env bash
+# Pre-clone external nargo git dependencies referenced in Nargo.toml files under
+# noir-projects/ with bounded retry. Works around transient DNS / network failures
+# during nargo's on-demand clone (e.g. "Could not resolve host: github.com"),
+# which on merge-queue-heavy shards halts the entire run via parallel --halt fail=1.
+#
+# Idempotent: skips deps already cached. Safe under concurrent invocation across
+# parallel make targets via a per-dep flock.
+
+set -euo pipefail
+
+NARGO_HOME="${NARGO_HOME:-$HOME/nargo}"
+ROOT="$(git rev-parse --show-toplevel)"
+
+mapfile -t deps < <(
+  find "$ROOT/noir-projects" -name Nargo.toml -print0 \
+  | xargs -0 grep -hE 'git\s*=\s*"https://github\.com/[^"]+"' \
+  | sed -nE 's/.*git\s*=\s*"https:\/\/github\.com\/([^"]+)".*tag\s*=\s*"([^"]+)".*/\1|\2/p;
+             t end;
+             s/.*tag\s*=\s*"([^"]+)".*git\s*=\s*"https:\/\/github\.com\/([^"]+)".*/\2|\1/p;
+             :end' \
+  | sort -u
+)
+
+clone_one() (
+  set -euo pipefail
+  local org_repo=$1
+  local tag=$2
+  local dest="$NARGO_HOME/github.com/$org_repo/$tag"
+  if [ -f "$dest/Nargo.toml" ]; then
+    return 0
+  fi
+  mkdir -p "$(dirname "$dest")"
+  local lock="$NARGO_HOME/github.com/$org_repo/.$tag.lock"
+  exec 9>"$lock"
+  flock 9
+  if [ -f "$dest/Nargo.toml" ]; then
+    return 0
+  fi
+  rm -rf "$dest"
+  local attempt
+  for attempt in 1 2 3; do
+    if git -c advice.detachedHead=false clone --quiet --depth 1 --branch "$tag" "https://github.com/$org_repo" "$dest"; then
+      return 0
+    fi
+    rm -rf "$dest"
+    if [ "$attempt" -lt 3 ]; then
+      sleep $((attempt * 2))
+    fi
+  done
+  echo "ERROR: failed to clone https://github.com/$org_repo @ $tag after 3 attempts" >&2
+  return 1
+)
+
+failed=0
+for dep in "${deps[@]}"; do
+  IFS='|' read -r org_repo tag <<< "$dep"
+  if ! clone_one "$org_repo" "$tag"; then
+    failed=1
+  fi
+done
+
+exit $failed