ENH: Mirror dockcross images to GHCR for reliable Python wheel builds

Python wheel CI builds pull large dockcross/manylinux container images
(~1-2 GB) from Docker Hub and Quay.io on every run. These pulls
intermittently fail with "unexpected EOF" or Docker Hub rate limiting
(401/429), causing wheel builds to fail across all remote modules.

Add a scheduled workflow that mirrors the pinned dockcross images to
GHCR (ghcr.io/insightsoftwareconsortium/), which has much better
connectivity from GitHub Actions runners (same network).

Add pre-pull steps to the Linux x64 and ARM build jobs that:
1. Try pulling from the GHCR mirror first
2. Fall back to the original Docker Hub / Quay.io source
3. Retry up to 3 times with backoff
4. Tag the GHCR image with the original name so the downstream
   ITKPythonPackage build scripts find it already cached

This eliminates transient Docker image pull failures that have been
causing sporadic Python wheel build failures across ~55 remote modules.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

Author: hjmjohnson

.github/workflows/build-test-package-python.yml

@@ -84,6 +84,38 @@ jobs:
         large-packages: "false"  # Takes too long to remove apt-get packages
         mandb:          "true"  # Speeds up future apt-get installs (disables man page generation), this CI does not use apt-get
 
+    - name: 'Pre-pull dockcross image'
+      run: |
+        MANYLINUX_PLATFORM=${{ matrix.manylinux-platform }}
+        MANYLINUX_VERSION=$(echo ${MANYLINUX_PLATFORM} | cut -d '-' -f 1)
+        TARGET_ARCH=$(echo ${MANYLINUX_PLATFORM} | cut -d '-' -f 2)
+        if [[ ${MANYLINUX_VERSION} == _2_28 && ${TARGET_ARCH} == x64 ]]; then
+          IMAGE_TAG="20240304-9e57d2b"
+        elif [[ ${MANYLINUX_VERSION} == 2014 ]]; then
+          IMAGE_TAG="20240304-9e57d2b"
+        else
+          echo "Unknown manylinux platform ${MANYLINUX_PLATFORM}, skipping pre-pull"
+          exit 0
+        fi
+        IMAGE="dockcross/manylinux${MANYLINUX_VERSION}-${TARGET_ARCH}:${IMAGE_TAG}"
+        GHCR_IMAGE="ghcr.io/insightsoftwareconsortium/dockcross-manylinux${MANYLINUX_VERSION}-${TARGET_ARCH}:${IMAGE_TAG}"
+        echo "Pre-pulling ${GHCR_IMAGE} (mirror of docker.io/${IMAGE})"
+        for attempt in 1 2 3; do
+          if docker pull "${GHCR_IMAGE}" 2>/dev/null; then
+            docker tag "${GHCR_IMAGE}" "docker.io/${IMAGE}"
+            echo "Successfully pulled from GHCR mirror"
+            exit 0
+          fi
+          echo "GHCR pull attempt ${attempt} failed, trying Docker Hub..."
+          if docker pull "docker.io/${IMAGE}"; then
+            echo "Successfully pulled from Docker Hub"
+            exit 0
+          fi
+          echo "Docker Hub pull attempt ${attempt} failed, retrying in 15s..."
+          sleep 15
+        done
+        echo "WARNING: All pull attempts failed, build script will retry"
+
     - name: 'Fetch build script'
       run: |
         IPP_DOWNLOAD_GIT_TAG=${{ inputs.itk-python-package-tag }}
@@ -180,6 +212,28 @@ jobs:
         large-packages: "false"  # Takes too long to remove apt-get packages
         mandb:          "true"  # Speeds up future apt-get installs (disables man page generation), this CI does not use apt-get
 
+    - name: 'Pre-pull manylinux aarch64 image'
+      run: |
+        IMAGE_TAG="2024-03-25-9206bd9"
+        IMAGE="quay.io/pypa/manylinux_2_28_aarch64:${IMAGE_TAG}"
+        GHCR_IMAGE="ghcr.io/insightsoftwareconsortium/dockcross-manylinux_2_28-aarch64:${IMAGE_TAG}"
+        echo "Pre-pulling ${GHCR_IMAGE} (mirror of ${IMAGE})"
+        for attempt in 1 2 3; do
+          if docker pull "${GHCR_IMAGE}" 2>/dev/null; then
+            docker tag "${GHCR_IMAGE}" "${IMAGE}"
+            echo "Successfully pulled from GHCR mirror"
+            exit 0
+          fi
+          echo "GHCR pull attempt ${attempt} failed, trying upstream..."
+          if docker pull "${IMAGE}"; then
+            echo "Successfully pulled from upstream"
+            exit 0
+          fi
+          echo "Upstream pull attempt ${attempt} failed, retrying in 15s..."
+          sleep 15
+        done
+        echo "WARNING: All pull attempts failed, build script will retry"
+
     - name: 'Fetch build script'
       run: |
         IPP_DOWNLOAD_GIT_TAG=${{ inputs.itk-python-package-tag }}

.github/workflows/mirror-dockcross-images.yml

@@ -0,0 +1,46 @@
+name: Mirror dockcross images to GHCR
+
+on:
+  schedule:
+    # Run weekly on Monday at 06:00 UTC
+    - cron: '0 6 * * 1'
+  workflow_dispatch:
+
+env:
+  GHCR_ORG: ghcr.io/insightsoftwareconsortium
+
+jobs:
+  mirror:
+    runs-on: ubuntu-22.04
+    permissions:
+      packages: write
+    strategy:
+      matrix:
+        include:
+          - source: docker.io/dockcross/manylinux_2_28-x64:20240304-9e57d2b
+            target: dockcross-manylinux_2_28-x64:20240304-9e57d2b
+          - source: docker.io/dockcross/manylinux2014-x64:20240304-9e57d2b
+            target: dockcross-manylinux2014-x64:20240304-9e57d2b
+          - source: quay.io/pypa/manylinux_2_28_aarch64:2024-03-25-9206bd9
+            target: dockcross-manylinux_2_28-aarch64:2024-03-25-9206bd9
+
+    steps:
+    - name: Log in to GHCR
+      uses: docker/login-action@v3
+      with:
+        registry: ghcr.io
+        username: ${{ github.actor }}
+        password: ${{ secrets.GITHUB_TOKEN }}
+
+    - name: Pull source image
+      run: |
+        for attempt in 1 2 3; do
+          docker pull ${{ matrix.source }} && break
+          echo "Pull attempt $attempt failed, retrying in 15s..."
+          sleep 15
+        done
+
+    - name: Tag and push to GHCR
+      run: |
+        docker tag ${{ matrix.source }} ${{ env.GHCR_ORG }}/${{ matrix.target }}
+        docker push ${{ env.GHCR_ORG }}/${{ matrix.target }}