Add apt retries, optional mirror for debootstrap

We've observed many issues over the years with ubuntu's apt servers,
including lots of temporary 503's.
This commit will
1) Use an in-gcp mirror for debootstrap, which should be fast (it
   doesn't run on ubuntu's infrastructure at all)
2) Add retries for all apt-get invocations during the build of both the
   os-image, and the iaas-specific builds - details below

Acquire::Retries=10 If a download fails (including transient HTTP errors
like 503), apt retries that fetch up to 10 times before giving up on
that URL. Without this, one bad response often aborts the whole apt-get
run.

Acquire::Retries::Delay=true Between retries, apt waits with increasing
delay (backoff) instead of hammering the server immediately. On Jammy
this is the “delay between retries” behavior (apt ≥ 2.0). Cumulatively
with Retries=10, you get many attempts spread over time, which helps
when snapshot.ubuntu.com is briefly overloaded rather than hard-down.

Acquire::http::Timeout=120 and Acquire::https::Timeout=120 Each
individual HTTP/HTTPS connection apt opens to a mirror can block for at
most 120 seconds before apt treats it as stuck and fails that attempt
(which can then trigger a retry if retries remain).

ai-assisted=yes
[TNZ-88995]

Author: selzoc

bosh-stemcell/lib/bosh/stemcell/builder_options.rb

@@ -61,7 +61,7 @@ def ovf_options
     def environment_variables
       {
         "UBUNTU_ISO" => environment["UBUNTU_ISO"],
-        "UBUNTU_MIRROR" => environment["UBUNTU_MIRROR"],
+        "UBUNTU_DEBOOTSTRAP_MIRROR" => environment["UBUNTU_DEBOOTSTRAP_MIRROR"],
         "UBUNTU_ADVANTAGE_TOKEN" => environment["UBUNTU_ADVANTAGE_TOKEN"],
         "UBUNTU_FIPS_USE_IAAS_KERNEL" => environment["UBUNTU_FIPS_USE_IAAS_KERNEL"]
       }

bosh-stemcell/spec/bosh/stemcell/builder_options_spec.rb

@@ -64,7 +64,7 @@ def self.it_sets_correct_environment_variables
           let(:env) do
             {
               "UBUNTU_ISO" => "fake_ubuntu_iso",
-              "UBUNTU_MIRROR" => "fake_ubuntu_mirror",
+              "UBUNTU_DEBOOTSTRAP_MIRROR" => "fake_ubuntu_mirror",
               "RUBY_BIN" => "fake_ruby_bin"
             }
           end
@@ -76,7 +76,7 @@ def self.it_sets_correct_environment_variables
             expect(result["stemcell_infrastructure"]).to eq(infrastructure.name)
             expect(result["stemcell_hypervisor"]).to eq(infrastructure.hypervisor)
             expect(result["UBUNTU_ISO"]).to eq("fake_ubuntu_iso")
-            expect(result["UBUNTU_MIRROR"]).to eq("fake_ubuntu_mirror")
+            expect(result["UBUNTU_DEBOOTSTRAP_MIRROR"]).to eq("fake_ubuntu_mirror")
             expect(result["ruby_bin"]).to eq("fake_ruby_bin")
             expect(result["image_create_disk_size"]).to eq(default_disk_size)
             expect(result["os_image_tgz"]).to eq("fake/os_image.tgz")

ci/pipelines/builder.yml

@@ -248,6 +248,7 @@ jobs:
     params:
       OPERATING_SYSTEM_NAME: ubuntu
       OPERATING_SYSTEM_VERSION: (@= data.values.stemcell_details.os_short_name @)
+      UBUNTU_DEBOOTSTRAP_MIRROR: http://gce.archive.ubuntu.com/ubuntu # Ubuntu's GCP-internal mirror, must change if we change Concourse IaaS!
     privileged: true
     vars:
       image_os_tag: (@= data.values.stemcell_details.os_short_name @)

ci/tasks/os-images/build.yml

@@ -19,3 +19,4 @@ params:
   OPERATING_SYSTEM_VERSION:   replace-me
   ESM_TOKEN:
   UBUNTU_ADVANTAGE_TOKEN:
+  UBUNTU_DEBOOTSTRAP_MIRROR:

stemcell_builder/lib/prelude_apply.bash

@@ -22,16 +22,23 @@ fi
 # Mark /opt/bosh as a safe git repo to avoid "fatal: unsafe repository ('/opt/bosh' is owned by someone else)"
 git config --global --add safe.directory /opt/bosh
 
+# Apt retry / timeout options applied to every apt-get invocation during the
+# build. Passed via -o so nothing leaks into the resulting OS image, and so
+# that flaky upstream mirrors (notably snapshot.ubuntu.com) don't take down
+# multi-hour builds on a single transient 503. Acquire::Retries::Delay=true
+# enables exponential backoff (apt >= 2.0).
+APT_RETRY_OPTS='-o Acquire::Retries=10 -o Acquire::Retries::Delay=true -o Acquire::http::Timeout=120 -o Acquire::https::Timeout=120'
+
 function pkg_mgr {
-  run_in_chroot $chroot "apt-get update"
-  run_in_chroot $chroot "export DEBIAN_FRONTEND=noninteractive; apt-get --fix-broken --no-install-recommends --assume-yes $*"
+  run_in_chroot $chroot "apt-get $APT_RETRY_OPTS update"
+  run_in_chroot $chroot "export DEBIAN_FRONTEND=noninteractive; apt-get $APT_RETRY_OPTS --fix-broken --no-install-recommends --assume-yes $*"
   run_in_chroot $chroot "apt-get clean"
 }
 
 # checks if an OS package with the given name exists in the current database of available packages.
 # returns 0 if package exists (whether or not is is installed); 1 otherwise
 function pkg_exists {
-  run_in_chroot $chroot "apt-get update"
+  run_in_chroot $chroot "apt-get $APT_RETRY_OPTS update"
   result=`run_in_chroot $chroot "if apt-cache show $1 2>/dev/null >/dev/null; then echo exists; else echo does not exist; fi"`
   if [[ "$result" == *"exists"* ]]; then
     return 0

stemcell_builder/lib/prelude_fips.bash

@@ -22,7 +22,7 @@ fi
 function ua_attach() {
     echo "Setting up Ubuntu Advantage ..."
 
-    DEBIAN_FRONTEND=noninteractive run_in_chroot ${chroot} "apt-get install --assume-yes ubuntu-pro-client"
+    DEBIAN_FRONTEND=noninteractive run_in_chroot ${chroot} "apt-get $APT_RETRY_OPTS install --assume-yes ubuntu-pro-client"
 
     run_in_chroot ${chroot} "ua attach --no-auto-enable ${UBUNTU_ADVANTAGE_TOKEN}"
 }
@@ -117,7 +117,7 @@ PSUEDO_GRUB_PROBE
 
 function mock_grub_probe() {
     # make sure /usr/sbin/grub-probe is installed in the chroot
-    DEBIAN_FRONTEND=noninteractive run_in_chroot ${chroot} "apt-get install --assume-yes grub-common"
+    DEBIAN_FRONTEND=noninteractive run_in_chroot ${chroot} "apt-get $APT_RETRY_OPTS install --assume-yes grub-common"
     gprobe="${chroot}/usr/sbin/grub-probe"
     if [ -f "${gprobe}" ]; then
         mv "${gprobe}" "${gprobe}.dist"

stemcell_builder/stages/base_debootstrap/apply.sh

@@ -76,7 +76,7 @@ cleanup_debootstrap() {
 }
 trap cleanup_debootstrap EXIT
 
-debootstrap --arch="$base_debootstrap_arch" "$base_debootstrap_suite" "$chroot" ""
+debootstrap --arch="$base_debootstrap_arch" "$base_debootstrap_suite" "$chroot" "${UBUNTU_DEBOOTSTRAP_MIRROR:-}"
 
 cleanup_debootstrap
 trap - EXIT

stemcell_builder/stages/base_debootstrap/config.sh

@@ -13,9 +13,9 @@ then
   persist UBUNTU_ISO
 fi
 
-if [ ! -z "${UBUNTU_MIRROR:-}" ]
+if [ ! -z "${UBUNTU_DEBOOTSTRAP_MIRROR:-}" ]
 then
-  persist UBUNTU_MIRROR
+  persist UBUNTU_DEBOOTSTRAP_MIRROR
 fi
 
 base_debootstrap_arch=amd64