Merge pull request #587 from cloudfoundry/add-ubuntu-server-resilience

Add apt retries, optional mirror for debootstrap

Author: selzoc

bosh-stemcell/lib/bosh/stemcell/builder_options.rb

@@ -61,7 +61,7 @@ def ovf_options
     def environment_variables
       {
         "UBUNTU_ISO" => environment["UBUNTU_ISO"],
-        "UBUNTU_MIRROR" => environment["UBUNTU_MIRROR"],
+        "UBUNTU_DEBOOTSTRAP_MIRROR" => environment["UBUNTU_DEBOOTSTRAP_MIRROR"],
         "UBUNTU_ADVANTAGE_TOKEN" => environment["UBUNTU_ADVANTAGE_TOKEN"],
         "UBUNTU_FIPS_USE_IAAS_KERNEL" => environment["UBUNTU_FIPS_USE_IAAS_KERNEL"]
       }

bosh-stemcell/spec/bosh/stemcell/builder_options_spec.rb

@@ -64,7 +64,7 @@ def self.it_sets_correct_environment_variables
           let(:env) do
             {
               "UBUNTU_ISO" => "fake_ubuntu_iso",
-              "UBUNTU_MIRROR" => "fake_ubuntu_mirror",
+              "UBUNTU_DEBOOTSTRAP_MIRROR" => "fake_ubuntu_mirror",
               "RUBY_BIN" => "fake_ruby_bin"
             }
           end
@@ -76,7 +76,7 @@ def self.it_sets_correct_environment_variables
             expect(result["stemcell_infrastructure"]).to eq(infrastructure.name)
             expect(result["stemcell_hypervisor"]).to eq(infrastructure.hypervisor)
             expect(result["UBUNTU_ISO"]).to eq("fake_ubuntu_iso")
-            expect(result["UBUNTU_MIRROR"]).to eq("fake_ubuntu_mirror")
+            expect(result["UBUNTU_DEBOOTSTRAP_MIRROR"]).to eq("fake_ubuntu_mirror")
             expect(result["ruby_bin"]).to eq("fake_ruby_bin")
             expect(result["image_create_disk_size"]).to eq(default_disk_size)
             expect(result["os_image_tgz"]).to eq("fake/os_image.tgz")

ci/pipelines/builder.yml

@@ -248,6 +248,7 @@ jobs:
     params:
       OPERATING_SYSTEM_NAME: ubuntu
       OPERATING_SYSTEM_VERSION: (@= data.values.stemcell_details.os_short_name @)
+      UBUNTU_DEBOOTSTRAP_MIRROR: http://gce.archive.ubuntu.com/ubuntu # Ubuntu's GCP-internal mirror, must change if we change Concourse IaaS!
     privileged: true
     vars:
       image_os_tag: (@= data.values.stemcell_details.os_short_name @)

ci/tasks/os-images/build.yml

@@ -19,3 +19,4 @@ params:
   OPERATING_SYSTEM_VERSION:   replace-me
   ESM_TOKEN:
   UBUNTU_ADVANTAGE_TOKEN:
+  UBUNTU_DEBOOTSTRAP_MIRROR:

stemcell_builder/lib/prelude_apply.bash

@@ -22,16 +22,23 @@ fi
 # Mark /opt/bosh as a safe git repo to avoid "fatal: unsafe repository ('/opt/bosh' is owned by someone else)"
 git config --global --add safe.directory /opt/bosh
 
+# Apt retry / timeout options applied to every apt-get invocation during the
+# build. Passed via -o so nothing leaks into the resulting OS image, and so
+# that flaky upstream mirrors (notably snapshot.ubuntu.com) don't take down
+# multi-hour builds on a single transient 503. Acquire::Retries::Delay=true
+# enables exponential backoff (apt >= 2.0).
+APT_RETRY_OPTS='-o Acquire::Retries=10 -o Acquire::Retries::Delay=true -o Acquire::http::Timeout=120 -o Acquire::https::Timeout=120'
+
 function pkg_mgr {
-  run_in_chroot $chroot "apt-get update"
-  run_in_chroot $chroot "export DEBIAN_FRONTEND=noninteractive; apt-get --fix-broken --no-install-recommends --assume-yes $*"
+  run_in_chroot $chroot "apt-get $APT_RETRY_OPTS update"
+  run_in_chroot $chroot "export DEBIAN_FRONTEND=noninteractive; apt-get $APT_RETRY_OPTS --fix-broken --no-install-recommends --assume-yes $*"
   run_in_chroot $chroot "apt-get clean"
 }
 
 # checks if an OS package with the given name exists in the current database of available packages.
 # returns 0 if package exists (whether or not is is installed); 1 otherwise
 function pkg_exists {
-  run_in_chroot $chroot "apt-get update"
+  run_in_chroot $chroot "apt-get $APT_RETRY_OPTS update"
   result=`run_in_chroot $chroot "if apt-cache show $1 2>/dev/null >/dev/null; then echo exists; else echo does not exist; fi"`
   if [[ "$result" == *"exists"* ]]; then
     return 0

stemcell_builder/lib/prelude_fips.bash

@@ -22,7 +22,7 @@ fi
 function ua_attach() {
     echo "Setting up Ubuntu Advantage ..."
 
-    DEBIAN_FRONTEND=noninteractive run_in_chroot ${chroot} "apt-get install --assume-yes ubuntu-pro-client"
+    DEBIAN_FRONTEND=noninteractive run_in_chroot ${chroot} "apt-get $APT_RETRY_OPTS install --assume-yes ubuntu-pro-client"
 
     run_in_chroot ${chroot} "ua attach --no-auto-enable ${UBUNTU_ADVANTAGE_TOKEN}"
 }
@@ -117,7 +117,7 @@ PSUEDO_GRUB_PROBE
 
 function mock_grub_probe() {
     # make sure /usr/sbin/grub-probe is installed in the chroot
-    DEBIAN_FRONTEND=noninteractive run_in_chroot ${chroot} "apt-get install --assume-yes grub-common"
+    DEBIAN_FRONTEND=noninteractive run_in_chroot ${chroot} "apt-get $APT_RETRY_OPTS install --assume-yes grub-common"
     gprobe="${chroot}/usr/sbin/grub-probe"
     if [ -f "${gprobe}" ]; then
         mv "${gprobe}" "${gprobe}.dist"

stemcell_builder/stages/base_debootstrap/apply.sh

@@ -76,7 +76,7 @@ cleanup_debootstrap() {
 }
 trap cleanup_debootstrap EXIT
 
-debootstrap --arch="$base_debootstrap_arch" "$base_debootstrap_suite" "$chroot" ""
+debootstrap --arch="$base_debootstrap_arch" "$base_debootstrap_suite" "$chroot" "${UBUNTU_DEBOOTSTRAP_MIRROR:-}"
 
 cleanup_debootstrap
 trap - EXIT

stemcell_builder/stages/base_debootstrap/config.sh

@@ -13,9 +13,9 @@ then
   persist UBUNTU_ISO
 fi
 
-if [ ! -z "${UBUNTU_MIRROR:-}" ]
+if [ ! -z "${UBUNTU_DEBOOTSTRAP_MIRROR:-}" ]
 then
-  persist UBUNTU_MIRROR
+  persist UBUNTU_DEBOOTSTRAP_MIRROR
 fi
 
 base_debootstrap_arch=amd64