ci: Split up network fetches and retry with Justfile targets

Our CI is long and flakes are painful, especially with merge queues.
Let's take a heavy hammer and refactor all network-fetched resources
into a clearly separate stage, and then we have a fully enforced
retry loop for those that has even longer timeouts in CI.

Assisted-by: OpenCode (claude-sonnet-4-6@default)
Signed-off-by: Colin Walters <walters@verbum.org>

Author: cgwalters

.github/workflows/ci.yml

@@ -27,6 +27,9 @@ env:
   # as the default runner user doesn't have access
   LIBVIRT_DEFAULT_URI: "qemu:///session"
   DEV_IMAGE: ghcr.io/bootc-dev/dev-bootc
+  # Retry parameters for `just build-fetch` (transient Koji/Copr/quay.io failures)
+  BOOTC_CI_RETRIES: "10"
+  BOOTC_CI_DELAY: "60"
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
@@ -108,6 +111,9 @@ jobs:
         run: sudo tune2fs -O verity $(findmnt -vno SOURCE /)
       - name: Install utils
         run: sudo apt -y install fsverity just
+      - name: Fetch external dependencies (with retry)
+        run: just build-fetch
+
       - name: Integration tests
         run: |
           set -xeu
@@ -271,6 +277,9 @@ jobs:
           name: packages-${{ matrix.test_os }}
           path: target/packages/
 
+      - name: Fetch external dependencies (with retry)
+        run: BOOTC_SKIP_PACKAGE=1 just build-fetch
+
       - name: Build container
         run: |
           BOOTC_SKIP_PACKAGE=1 just bootloader=$BOOTC_bootloader build
@@ -355,6 +364,9 @@ jobs:
           name: packages-${{ matrix.test_os }}
           path: target/packages/
 
+      - name: Fetch external dependencies (with retry)
+        run: just build-fetch
+
       - name: Run upgrade test
         run: just test-upgrade
 

Dockerfile

@@ -53,18 +53,20 @@ RUN --mount=type=tmpfs,target=/run --mount=type=tmpfs,target=/tmp \
     /run/packaging/enable-compose-repos
 RUN --mount=type=tmpfs,target=/run --mount=type=tmpfs,target=/tmp /usr/libexec/bootc-base-imagectl build-rootfs --manifest=standard /target-rootfs
 
-FROM scratch as base
+FROM scratch as fetch
 COPY --from=target-base /target-rootfs/ /
 # SKIP_CONFIGS=1 skips LBIs, test kargs, and install configs (for FCOS testing)
 ARG SKIP_CONFIGS
 ARG boot_type
 ARG seal_state
-# Use tmpfs for /run and /tmp with bind mounts inside to avoid leaking mount stubs into the image
+# All network-fetching operations: package installs from distro repos, Copr, Koji.
+# Separated so `just build-fetch --target=fetch` can be retried independently on
+# transient network failures without re-running the configuration phase.
 RUN --mount=type=tmpfs,target=/run --mount=type=tmpfs,target=/tmp \
     --mount=type=bind,from=src,src=/src/hack,target=/run/hack <<-EOF
     set -ex
 
-    cd /run/hack/ && SKIP_CONFIGS="${SKIP_CONFIGS}" ./provision-derived.sh
+    cd /run/hack/ && SKIP_CONFIGS="${SKIP_CONFIGS}" ./provision-fetch.sh
 
     pkgs_to_install=()
     if [[ "${seal_state}" == "sealed" ]]; then
@@ -106,7 +108,7 @@ CMD ["/sbin/init"]
 
 # This layer contains things which aren't in the default image and may
 # be used for sealing images in particular.
-FROM base as tools
+FROM fetch as tools
 RUN --mount=type=tmpfs,target=/run --mount=type=tmpfs,target=/tmp \
     --mount=type=bind,from=packaging,src=/,target=/run/packaging \
     /run/packaging/initialize-sealing-tools
@@ -118,6 +120,14 @@ RUN --mount=type=tmpfs,target=/run --mount=type=tmpfs,target=/tmp \
 # This is verified in `cargo xtask check-buildsys`.
 # -------------
 
+FROM fetch as base
+ARG SKIP_CONFIGS
+# Local configuration only — no network access required or permitted.
+# Sits after the cutoff so the linter enforces --network=none automatically.
+RUN --network=none --mount=type=tmpfs,target=/run --mount=type=tmpfs,target=/tmp \
+    --mount=type=bind,from=src,src=/src/hack,target=/run/hack \
+    sh -c 'cd /run/hack/ && SKIP_CONFIGS="${SKIP_CONFIGS}" ./provision-configure.sh'
+
 FROM buildroot as build
 # Version for RPM build (optional, computed from git in Justfile)
 ARG pkgversion

Justfile

@@ -86,6 +86,56 @@ build: package _keygen && _pull-lbi-images
     eval $(just _git-build-vars)
     podman build {{_nocache_arg}} --build-arg=image_version=${VERSION} --build-context "packages=${pkg_path}" -t {{base_img}} {{buildargs}} .
 
+# Fetch all external dependencies with a retry loop.
+#
+# This runs `podman build --target=fetch` for both the main image and the
+# upgrade-source image, retrying on transient network failures (Koji 503s,
+# Copr outages, quay.io blips, etc.).  In CI this runs as its own step
+# before `just build` / `just test-upgrade` so that flakes don't require
+# re-queueing the entire PR.
+#
+# The retry parameters can be overridden via environment variables:
+#   BOOTC_CI_RETRIES=10 BOOTC_CI_DELAY=60 just build-fetch
+[group('core')]
+build-fetch: _keygen
+    #!/bin/bash
+    set -euo pipefail
+    retries=${BOOTC_CI_RETRIES:-3}
+    delay=${BOOTC_CI_DELAY:-30}
+    retry() {
+        local attempt
+        for attempt in $(seq 1 "$retries"); do
+            echo "--- Attempt ${attempt}/${retries}: $*"
+            if "$@"; then
+                return 0
+            fi
+            if [ "$attempt" -lt "$retries" ]; then
+                echo "--- Attempt ${attempt} failed, retrying in ${delay}s..."
+                sleep "$delay"
+            fi
+        done
+        echo "--- All ${retries} attempts failed: $*" >&2
+        return 1
+    }
+    # Pull the base images explicitly so failures are retried cleanly
+    # before we even start the container build.
+    retry podman pull -q {{base}}
+    retry podman pull -q {{buildroot_base}}
+    # Pull LBI images (also fetched later by _pull-lbi-images, but doing it
+    # here means a failure is retried rather than aborting the full build).
+    for img in {{lbi_images}}; do
+        retry podman pull -q "$img"
+    done
+    # Build the network-heavy fetch stage of the main image.  If this
+    # succeeds, `just build` will get a cache hit on the fetch layer and
+    # run entirely offline.
+    # Note: buildargs (not base_buildargs) is needed here because the
+    # target-base stage requires --cap-add/--security-opt for bwrap.
+    retry podman build {{_nocache_arg}} --target=fetch {{buildargs}} .
+    # Same for the upgrade-source image used by test-upgrade.
+    retry podman build {{_nocache_arg}} --build-arg=base={{base}} \
+        --target=fetch -f tmt/tests/Dockerfile.upgrade-source .
+
 # Show available build variants and current configuration
 [group('core')]
 list-variants:

hack/provision-configure.sh

@@ -0,0 +1,121 @@
+#!/bin/bash
+# All local filesystem configuration for a derived test image.
+# No network access required; this runs after provision-fetch.sh has
+# installed all packages. See also: Dockerfile fetch/base stage split.
+set -xeu
+
+cloudinit=0
+case ${1:-} in
+  cloudinit) cloudinit=1 ;;
+  "") ;;
+  *) echo "Unhandled flag: ${1:-}" 1>&2; exit 1 ;;
+esac
+
+# Clean root's homedir (provision-fetch.sh may have left cargo/dnf state).
+rm -rf /var/roothome/.config
+mkdir -p -m 0700 /var/roothome
+
+# Nushell config for root: store the files under /usr so they are covered by
+# the OS image, then use tmpfiles.d 'C' to copy them into /var/roothome at
+# first boot.  Writing directly to /var would require tmpfiles entries anyway
+# and would fail `bootc container lint --fatal-warnings`.
+mkdir -p /usr/share/bootc-test/nushell-skel
+echo '$env.config = { show_banner: false, }' > /usr/share/bootc-test/nushell-skel/config.nu
+touch /usr/share/bootc-test/nushell-skel/env.nu
+cat >/usr/lib/tmpfiles.d/bootc-test-nushell.conf <<'EOF'
+d  /var/roothome/.config          0700 root root - -
+d  /var/roothome/.config/nushell  0700 root root - -
+C+ /var/roothome/.config/nushell/config.nu 0600 root root - /usr/share/bootc-test/nushell-skel/config.nu
+C+ /var/roothome/.config/nushell/env.nu    0600 root root - /usr/share/bootc-test/nushell-skel/env.nu
+EOF
+
+# kargs for serial console
+cat <<KARGEOF >> /usr/lib/bootc/kargs.d/20-console.toml
+kargs = ["console=ttyS0,115200n8"]
+KARGEOF
+
+if test $cloudinit = 1; then
+  ln -s ../cloud-init.target /usr/lib/systemd/system/default.target.wants
+  # Allow root SSH login for testing with bcvk/tmt
+  mkdir -p /etc/cloud/cloud.cfg.d
+  cat > /etc/cloud/cloud.cfg.d/80-enable-root.cfg <<'CLOUDEOF'
+# Enable root login for testing
+disable_root: false
+
+# In image mode, the host root filesystem is mounted at /sysroot, not /
+# That is the one we should attempt to resize, not what is mounted at /
+growpart:
+  mode: auto
+  devices: ["/sysroot"]
+resize_rootfs: false
+CLOUDEOF
+fi
+
+cat >/usr/lib/tmpfiles.d/bootc-cloud-init.conf <<'EOF'
+d /var/lib/cloud 0755 root root - -
+EOF
+
+# Fast track tmpfiles.d content from the base image, xref
+# https://gitlab.com/fedora/bootc/base-images/-/merge_requests/92
+if test '!' -f /usr/lib/tmpfiles.d/bootc-base-rpmstate.conf; then
+  cat >/usr/lib/tmpfiles.d/bootc-base-rpmstate.conf <<'EOF'
+# Workaround for https://bugzilla.redhat.com/show_bug.cgi?id=771713
+d /var/lib/rpm-state 0755 - - -
+EOF
+fi
+if ! grep -q -r var/roothome/buildinfo /usr/lib/tmpfiles.d; then
+  cat > /usr/lib/tmpfiles.d/bootc-contentsets.conf <<'EOF'
+# Workaround for https://github.com/konflux-ci/build-tasks-dockerfiles/pull/243
+d /var/roothome/buildinfo 0755 - - -
+d /var/roothome/buildinfo/content_manifests 0755 - - -
+# Note we don't actually try to recreate the content; this just makes the linter ignore it
+f /var/roothome/buildinfo/content_manifests/content-sets.json 0644 - - -
+EOF
+fi
+
+# And add missing sysusers.d entries
+if ! grep -q -r sudo /usr/lib/sysusers.d; then
+  cat >/usr/lib/sysusers.d/bootc-sudo-workaround.conf <<'EOF'
+g sudo 16
+EOF
+fi
+
+# dhcpcd
+if rpm -q dhcpcd &>/dev/null; then
+if ! grep -q -r dhcpcd /usr/lib/sysusers.d; then
+  cat >/usr/lib/sysusers.d/bootc-dhcpcd-workaround.conf <<'EOF'
+u dhcpcd - 'Minimalistic DHCP client' /var/lib/dhcpcd
+EOF
+fi
+cat >/usr/lib/tmpfiles.d/bootc-dhcpd.conf <<'EOF'
+d /var/lib/dhcpcd 0755 root dhcpcd - -
+EOF
+  rm -rf /var/lib/dhcpcd
+fi
+# dhclient
+if test -d /var/lib/dhclient; then
+  cat >/usr/lib/tmpfiles.d/bootc-dhclient.conf <<'EOF'
+d /var/lib/dhclient 0755 root root - -
+EOF
+  rm -rf /var/lib/dhclient
+fi
+
+# The following configs are skipped when SKIP_CONFIGS=1, which is used
+# for testing bootc install on Fedora CoreOS where these would conflict.
+if test -z "${SKIP_CONFIGS:-}"; then
+  # For test-22-logically-bound-install
+  install -D -m 0644 -t /usr/share/containers/systemd/ lbi/*
+  for x in curl.container curl-base.image podman.image; do
+      ln -s /usr/share/containers/systemd/$x /usr/lib/bootc/bound-images.d/$x
+  done
+
+  # Add some testing kargs into our dev builds
+  install -D -t /usr/lib/bootc/kargs.d test-kargs/*
+  # Also copy in some default install configs we use for testing
+  install -D -t /usr/lib/bootc/install/ install-test-configs/*
+
+  # Install os-image-map.json for tests that need to select OS-matched images
+  install -D -m 0644 os-image-map.json /usr/share/bootc/os-image-map.json
+else
+  echo "SKIP_CONFIGS is set, skipping LBIs, test kargs, and install configs"
+fi