Fix OCI build

Author: jvstme

scripts/packer/oci-image-cuda.json

@@ -27,10 +27,6 @@
       "type": "shell",
       "inline": ["cloud-init status --long --wait"]
     },
-    {
-      "type": "shell",
-      "script": "provisioners/wait-for-dpkg-lock.sh"
-    },
     {
       "type": "shell",
       "scripts": [

scripts/packer/oci-image.json

@@ -26,10 +26,6 @@
       "type": "shell",
       "inline": ["cloud-init status --long --wait"]
     },
-    {
-      "type": "shell",
-      "script": "provisioners/wait-for-dpkg-lock.sh"
-    },
     {
       "type": "shell",
       "scripts": [

scripts/packer/provisioners/kernel/apt-upgrade.sh

@@ -5,5 +5,39 @@
 
 set -e
 
-sudo apt-get update
+#
+# Run apt-get update, but retry in case the lock is held by another process.
+#
+# A better way of handling apt races is the `-o DPkg::Lock::Timeout=X` option,
+# but it does not work with `apt-get update`.
+#
+# This function was added specifically for the `oci` backend, where our build
+# process conflicts with OCI's instance agent.
+#
+apt_update_with_retry() {
+    local MAX_RETRIES=10
+    local RETRY_DELAY=3
+    local COUNT=0
+    local LOGFILE=$(mktemp)
+
+    while [ $COUNT -lt $MAX_RETRIES ]; do
+        set +e
+        sudo apt-get update 2>&1 | tee "$LOGFILE"
+        local EXIT_CODE=${PIPESTATUS[0]}
+        set -e
+
+        if grep -q "Could not get lock" "$LOGFILE"; then
+            echo "apt lock file is held by another process. Retrying in $RETRY_DELAY seconds..."
+            COUNT=$((COUNT + 1))
+            sleep $RETRY_DELAY
+        else
+            return $EXIT_CODE
+        fi
+    done
+
+    echo "apt-get update failed due to lock being held after $MAX_RETRIES attempts."
+    return 1
+}
+
+apt_update_with_retry
 sudo DEBIAN_FRONTEND=noninteractive apt-get -o DPkg::Lock::Timeout=60 dist-upgrade -y -q