Merge pull request #10701 from julek-wolfssl/ci-drop-apt-deps-cache

CI: install all apt deps from ghcr bundles, drop actions/cache apt-deps layer

Author: dgarske

.github/actions/install-apt-deps/action.yml

@@ -1,5 +1,5 @@
 name: 'Install apt dependencies'
-description: 'Install apt packages with retry logic and caching'
+description: 'Install apt packages with retry logic and an optional offline ghcr bundle'
 inputs:
   packages:
     description: 'Space-separated list of apt packages to install'
@@ -16,16 +16,12 @@ inputs:
     description: 'Pass --no-install-recommends to apt-get install'
     required: false
     default: 'false'
-  cache:
-    description: 'Cache apt archives (disable for dynamic package names)'
-    required: false
-    default: 'true'
   ghcr-debs-tag:
     description: >
       Tag of a prebuilt .deb bundle published to
       ghcr.io/<owner>/wolfssl-ci-debs by the ci-deps-image workflow
       (e.g. "ubuntu-24.04-minimal"). When set, the packages are installed
-      offline from that bundle and the apt cache path below is skipped; on
+      offline from that bundle and the apt path below is skipped; on
       that happy path the apt mirror is not contacted. The offline install
       is all-or-nothing (a single --no-download install of the whole set),
       so any failure - bundle missing, not public, or not covering every
@@ -39,7 +35,7 @@ runs:
     # Preferred path: install from a prebuilt .deb bundle pulled from ghcr,
     # entirely offline (--no-download), so a flaky/timing-out apt mirror
     # cannot break the build. Best-effort: on any failure we leave
-    # "satisfied" unset and the apt steps below run unchanged. The bundle
+    # "satisfied" unset and the apt step below runs unchanged. The bundle
     # image must be PUBLIC so anonymous `docker pull` works (including from
     # fork PRs whose GITHUB_TOKEN cannot read private packages).
     - name: Install from ghcr .deb bundle (offline)
@@ -77,40 +73,9 @@ runs:
           echo "::notice::offline install incomplete for $IMG; using apt"
         fi
 
-    - name: Compute cache key
-      if: inputs.cache == 'true' && steps.ghcr.outputs.satisfied != 'true'
-      id: cache-key
-      shell: bash
-      run: |
-        SORTED_PKGS=$(echo "${{ inputs.packages }}" | tr ' ' '\n' | sort -u | tr '\n' ' ')
-        PKG_HASH=$(echo "$SORTED_PKGS" | sha256sum | cut -d' ' -f1 | head -c 16)
-        OS_VERSION=$(lsb_release -rs 2>/dev/null || echo "unknown")
-        echo "key=apt-deps-${{ runner.os }}-${{ runner.arch }}-${OS_VERSION}-${PKG_HASH}" >> $GITHUB_OUTPUT
-        echo "restore-key=apt-deps-${{ runner.os }}-${{ runner.arch }}-${OS_VERSION}-" >> $GITHUB_OUTPUT
-
-    - name: Restore apt cache
-      if: inputs.cache == 'true' && steps.ghcr.outputs.satisfied != 'true'
-      id: apt-cache
-      uses: actions/cache/restore@v5
-      with:
-        path: ~/apt-cache
-        key: ${{ steps.cache-key.outputs.key }}
-        restore-keys: ${{ steps.cache-key.outputs.restore-key }}
-
-    - name: Pre-seed apt archives from cache
-      if: inputs.cache == 'true' && steps.apt-cache.outputs.cache-hit == 'true' && steps.ghcr.outputs.satisfied != 'true'
-      shell: bash
-      run: |
-        if [ -d ~/apt-cache ] && ls ~/apt-cache/*.deb >/dev/null 2>&1; then
-          sudo cp ~/apt-cache/*.deb /var/cache/apt/archives/
-          echo "Restored $(ls ~/apt-cache/*.deb | wc -l) cached .deb files"
-        fi
-
     - name: Install packages
       if: steps.ghcr.outputs.satisfied != 'true'
       shell: bash
-      env:
-        APT_CACHE_HIT: ${{ steps.apt-cache.outputs.cache-hit }}
       run: |
         export DEBIAN_FRONTEND=noninteractive
         RETRIES=${{ inputs.retries }}
@@ -120,17 +85,6 @@ runs:
           NO_REC="--no-install-recommends"
         fi
 
-        # Fast path: on cache hit the .debs are already pre-seeded into
-        # /var/cache/apt/archives. Try installing directly first; if that
-        # fails (e.g. the cached .debs were superseded in the index) fall
-        # through to the regular update + install path.
-        if [ "$APT_CACHE_HIT" = "true" ]; then
-          if sudo apt-get install -y $NO_REC ${{ inputs.packages }}; then
-            exit 0
-          fi
-          echo "::warning::install from cached .debs failed, falling back to apt-get update"
-        fi
-
         for i in $(seq 1 $RETRIES); do
           if sudo apt-get update -q && \
              sudo apt-get install -y $NO_REC ${{ inputs.packages }}; then
@@ -144,21 +98,3 @@ runs:
           sleep $DELAY
           DELAY=$((DELAY * 2))
         done
-
-    # PR runs never write the apt cache (no churn); only push/schedule runs
-    # refresh it. The make-check family does not need it anyway - it installs
-    # from the ghcr bundle above.
-    - name: Collect .deb files for cache
-      if: inputs.cache == 'true' && github.event_name != 'pull_request' && steps.apt-cache.outputs.cache-hit != 'true' && steps.ghcr.outputs.satisfied != 'true'
-      shell: bash
-      run: |
-        mkdir -p ~/apt-cache
-        cp /var/cache/apt/archives/*.deb ~/apt-cache/ 2>/dev/null || true
-        echo "Cached $(ls ~/apt-cache/*.deb 2>/dev/null | wc -l) .deb files"
-
-    - name: Save apt cache
-      if: inputs.cache == 'true' && github.event_name != 'pull_request' && steps.apt-cache.outputs.cache-hit != 'true' && steps.ghcr.outputs.satisfied != 'true'
-      uses: actions/cache/save@v5
-      with:
-        path: ~/apt-cache
-        key: ${{ steps.cache-key.outputs.key }}

.github/actions/install-arduino-core/action.yml

@@ -0,0 +1,81 @@
+name: 'Install Arduino core'
+description: >
+  Make an Arduino core (and the shared CI libraries) available, preferring a
+  prebuilt bundle pulled from ghcr (published by the arduino-cores-image
+  workflow) and falling back to `arduino-cli core install` when the bundle is
+  unavailable or stale. Assumes arduino-cli is already on PATH.
+inputs:
+  core-id:
+    description: 'vendor:arch core to make available, e.g. esp32:esp32'
+    required: true
+  board-manager-url:
+    description: >
+      Optional third-party board_manager index URL, used only on the
+      online-install fallback (the ghcr bundle already carries its own).
+    required: false
+    default: ''
+  libs:
+    description: 'Space-separated Arduino libraries to ensure are present'
+    required: false
+    default: 'ArduinoJson WiFiNINA Ethernet Bridge'
+runs:
+  using: 'composite'
+  steps:
+    # Preferred path: restore ~/.arduino15 (the core + toolchain) and the
+    # shared libraries from a prebuilt tarball pulled from ghcr, so the flaky
+    # board_manager / toolchain downloads are off the PR critical path. The
+    # bundle is published only under the wolfssl org (gated below), so fork PRs
+    # read the public upstream image too. Best-effort: any failure leaves
+    # "satisfied" unset and the online install below runs unchanged.
+    - name: Restore Arduino core from ghcr bundle
+      id: ghcr
+      shell: bash
+      run: |
+        set -u
+        command -v docker >/dev/null 2>&1 || { echo "::notice::docker unavailable; installing core online"; exit 0; }
+        command -v arduino-cli >/dev/null 2>&1 || { echo "::notice::arduino-cli not on PATH; installing core online"; exit 0; }
+        CORE_ID='${{ inputs.core-id }}'
+        TAG=$(echo "$CORE_ID" | tr ':' '-')
+        IMG="ghcr.io/wolfssl/wolfssl-ci-arduino:$TAG"
+        if ! docker pull -q "$IMG" >/dev/null 2>&1; then
+          echo "::notice::ghcr bundle $IMG unavailable; installing core online"
+          exit 0
+        fi
+        cid=$(docker create "$IMG" 2>/dev/null) || { echo "::notice::cannot open bundle; installing core online"; exit 0; }
+        rm -f "$RUNNER_TEMP/arduino-core.tar"
+        docker cp "$cid:/arduino-core.tar" "$RUNNER_TEMP/arduino-core.tar" >/dev/null 2>&1 || true
+        docker rm "$cid" >/dev/null 2>&1 || true
+        test -f "$RUNNER_TEMP/arduino-core.tar" || { echo "::notice::bundle had no tarball; installing core online"; exit 0; }
+        # Entries are stored relative to $HOME (.arduino15/..., Arduino/libraries/...).
+        tar -C "$HOME" -xf "$RUNNER_TEMP/arduino-core.tar" || { echo "::notice::could not unpack bundle; installing core online"; exit 0; }
+        rm -f "$RUNNER_TEMP/arduino-core.tar"
+        if arduino-cli core list 2>/dev/null | awk 'NR>1 {print $1}' | grep -Fxq "$CORE_ID"; then
+          echo "satisfied=true" >> "$GITHUB_OUTPUT"
+          echo "Restored $CORE_ID from $IMG"
+        else
+          echo "::notice::bundle did not yield $CORE_ID; installing core online"
+        fi
+
+    - name: Install Arduino core online
+      if: steps.ghcr.outputs.satisfied != 'true'
+      shell: bash
+      run: |
+        set -euo pipefail
+        CORE_ID='${{ inputs.core-id }}'
+        BM_URL='${{ inputs.board-manager-url }}'
+        retry() { local i; for i in 1 2 3 4 5; do "$@" && return 0; sleep $((2**i)); done; "$@"; }
+
+        arduino-cli config init --overwrite
+        # Wait up to 10 minutes for the big toolchain downloads.
+        arduino-cli config set network.connection_timeout 600s
+        # Scope third-party indexes to the one core that needs them: arduino-cli
+        # re-reads every configured index on each call and fails if any is
+        # unreachable, so an unconditional URL makes all jobs depend on it.
+        if [ -n "$BM_URL" ]; then
+          arduino-cli config add board_manager.additional_urls "$BM_URL"
+        fi
+        retry arduino-cli core update-index
+        retry arduino-cli core install "$CORE_ID"
+        for lib in ${{ inputs.libs }}; do
+          retry arduino-cli lib install "$lib"
+        done

.github/ci-deps/packages-ubuntu-22.04-full.txt

@@ -4,6 +4,7 @@
 # Keep sorted; add a package when an interop workflow adds one.
 autoconf
 automake
+binutils-dev
 bison
 bridge-utils
 build-essential
@@ -17,6 +18,7 @@ crossbuild-essential-arm64
 crossbuild-essential-armel
 crossbuild-essential-armhf
 crossbuild-essential-riscv64
+curl
 device-tree-compiler
 dfu-util
 diffstat
@@ -39,12 +41,19 @@ help2man
 iproute2
 lcov
 libcairo2-dev
+libcurl4-openssl-dev
+libdbus-1-dev
 libglib2.0-dev
 libgtk2.0-0
+libiberty-dev
 liblocale-gettext-perl
 libmagic1
 libncurses5-dev
+libnl-3-dev
+libnl-genl-3-dev
+libnl-route-3-dev
 libpcap-dev
+libpcap0.8
 libpopt0
 libsdl1.2-dev
 libsdl2-dev
@@ -63,6 +72,7 @@ python-is-python3
 python3-dev
 python3-pip
 python3-ply
+python3-pycryptodome
 python3-setuptools
 python3-tk
 python3-wheel
@@ -73,6 +83,7 @@ socat
 srecord
 sudo
 texinfo
+tshark
 uml-utilities
 unzip
 wget

.github/ci-deps/packages-ubuntu-24.04-embedded.txt

@@ -0,0 +1,15 @@
+# membrowse embedded-target apt packages for ubuntu-24.04 (the
+# '-embedded' bundle: ghcr.io/<owner>/wolfssl-ci-debs:ubuntu-24.04-embedded).
+# Kept separate from -full because the ARM cross-toolchain is large (~0.5 GB)
+# and unrelated to the interop workflows that pull -full. Keep sorted.
+build-essential
+ca-certificates
+cmake
+gcc-arm-none-eabi
+git
+libnewlib-arm-none-eabi
+libstdc++-arm-none-eabi-newlib
+ninja-build
+python3
+unzip
+wget

.github/ci-deps/packages-ubuntu-24.04-full.txt

@@ -8,6 +8,7 @@ autoconf
 autoconf-archive
 automake
 autopoint
+bc
 bubblewrap
 build-essential
 ccache
@@ -51,6 +52,8 @@ libidn2-dev
 libio-socket-ssl-perl
 libjansson-dev
 libkrb5-dev
+libldb-dev
+libldb2
 liblz4-dev
 liblzma-dev
 liblzo2-dev
@@ -87,6 +90,7 @@ pkgconf
 psmisc
 python3-docutils
 python3-impacket
+python3-ldb
 python3-psutil
 shellcheck
 uuid-dev