perf(airgap): per-version cache, gzip cleanup, staged workflow

EliShteinman · claude · EliShteinman · commit c5c82df21dee · 2026-05-13T18:03:55.000+03:00
Three coupled changes to reduce build time and surface progress:

1. airgap-multibuild.sh: per-version output cache keyed by content hash.
   For each (product, version) build, hashes the version's content +
   shared layouts + config + script. On hit, skips Hugo entirely and
   restores the cached output. On miss, builds and saves to cache.
   Backed by a BuildKit cache mount on /var/cache/airgap-versions, so
   the cache persists across builds (and across workflow runs via
   cache-to=type=gha,mode=max). Expected savings: ~30 min on a typical
   merge that only changes one version's content.

2. Dockerfile: split builder into three named stages (deps, components,
   builder) so the workflow can target each independently and the run
   summary shows progress between them. Removes .json from the gzip
   pre-compression list — nginx serves .json with sub_filter for the
   __DOCS_BASE_URL__ placeholder, which requires gzip_static OFF, so
   pre-compressing those ~12k files is wasted CPU.

3. Workflow: replace the single docker/build-push-action call with five
   sequential steps (deps → components → builder → push priv →
   push unpriv). After each step, append a checkmark line to the run
   summary so you see progress between stages in real time. The build
   plan now includes a progress table seeded with ⏳ pending entries.

Co-Authored-By: Claude Opus 4.7 (1M context) &lt;noreply@anthropic.com&gt;
diff --git a/.github/workflows/airgap-build.yml b/.github/workflows/airgap-build.yml
@@ -55,8 +55,7 @@ jobs:
         uses: actions/checkout@v4
 
       # ----------------------------------------------------------------------
-      # Step 1: Compute build parameters. Inputs are passed via env: to avoid
-      # any shell-injection risk from the free-text tag_override field.
+      # Setup
       # ----------------------------------------------------------------------
       - name: Compute build params
         id: meta
@@ -120,29 +119,37 @@ jobs:
             echo "|---------|-------|"
             echo "| Commit | \`$HASH\` |"
             echo "| Branch | \`$BRANCH\` |"
-            echo "| Variant input | \`$INPUT_VARIANT\` |"
+            echo "| Variant | \`$INPUT_VARIANT\` |"
             echo "| Platforms | \`$PLATFORMS\` |"
             echo "| Tag override | \`$OVERRIDE_DISPLAY\` |"
             echo ""
 
             if [ "$INPUT_VARIANT" != "unprivileged" ]; then
-              echo "### Privileged tags about to be pushed"
+              echo "### Privileged tags to push"
               echo '```'
               echo "$PRIV_TAGS"
               echo '```'
             fi
 
             if [ "$INPUT_VARIANT" != "privileged" ]; then
-              echo "### Unprivileged tags about to be pushed"
+              echo "### Unprivileged tags to push"
               echo '```'
               echo "$UNPRIV_TAGS"
               echo '```'
             fi
+
+            echo ""
+            echo "## Progress"
+            echo ""
+            echo "| Stage | Status |"
+            echo "|-------|--------|"
+            echo "| 1/5: Install dependencies (apt + hugo + npm + pip) | ⏳ pending |"
+            echo "| 2/5: Clone external components (make components) | ⏳ pending |"
+            echo "| 3/5: Hugo multi-version build + gzip | ⏳ pending |"
+            echo "| 4/5: Build & push privileged | ⏳ pending |"
+            echo "| 5/5: Build & push unprivileged | ⏳ pending |"
           } >> "$GITHUB_STEP_SUMMARY"
 
-      # ----------------------------------------------------------------------
-      # Step 2: Cross-arch setup (skipped when building amd64-only)
-      # ----------------------------------------------------------------------
       - name: Set up QEMU (for arm64 runtime layer)
         if: inputs.platforms != 'amd64'
         uses: docker/setup-qemu-action@v4
@@ -159,12 +166,78 @@ jobs:
           password: ${{ secrets.DOCKERHUB_TOKEN }}
 
       # ----------------------------------------------------------------------
-      # Step 3: privileged. The Dockerfile pins builder to $BUILDPLATFORM
-      # (amd64 native on this runner), so the heavy work runs ONCE.
-      # `cache-to: type=gha,mode=max` writes intermediates to GitHub Actions
-      # cache → next workflow run starts with a warm cache.
+      # Stage 1/5: Install build dependencies
       # ----------------------------------------------------------------------
-      - name: Build & push privileged
+      - name: "Stage 1/5: Install build dependencies"
+        id: stage1
+        uses: docker/build-push-action@v7
+        with:
+          context: .
+          target: deps
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+          push: false
+      - name: "Stage 1/5: summary"
+        if: always() && steps.stage1.conclusion != 'skipped'
+        env:
+          STAGE_OUTCOME: ${{ steps.stage1.outcome }}
+        run: |
+          ICON="✅"
+          [ "$STAGE_OUTCOME" != "success" ] && ICON="❌"
+          echo "- $ICON **Stage 1/5**: Dependencies installed ($STAGE_OUTCOME)" >> "$GITHUB_STEP_SUMMARY"
+
+      # ----------------------------------------------------------------------
+      # Stage 2/5: Clone external example repos
+      # ----------------------------------------------------------------------
+      - name: "Stage 2/5: Clone external components"
+        id: stage2
+        uses: docker/build-push-action@v7
+        with:
+          context: .
+          target: components
+          secrets: |
+            PRIVATE_ACCESS_TOKEN=${{ secrets.PRIVATE_ACCESS_TOKEN }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+          push: false
+      - name: "Stage 2/5: summary"
+        if: always() && steps.stage2.conclusion != 'skipped'
+        env:
+          STAGE_OUTCOME: ${{ steps.stage2.outcome }}
+        run: |
+          ICON="✅"
+          [ "$STAGE_OUTCOME" != "success" ] && ICON="❌"
+          echo "- $ICON **Stage 2/5**: External components cloned ($STAGE_OUTCOME)" >> "$GITHUB_STEP_SUMMARY"
+
+      # ----------------------------------------------------------------------
+      # Stage 3/5: Hugo multi-version build (the long one, ~30 min on GHA
+      # for a fully-cold build, much less when the per-version cache mount
+      # has entries from previous runs).
+      # ----------------------------------------------------------------------
+      - name: "Stage 3/5: Hugo multi-version build + gzip"
+        id: stage3
+        uses: docker/build-push-action@v7
+        with:
+          context: .
+          target: builder
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+          push: false
+      - name: "Stage 3/5: summary"
+        if: always() && steps.stage3.conclusion != 'skipped'
+        env:
+          STAGE_OUTCOME: ${{ steps.stage3.outcome }}
+        run: |
+          ICON="✅"
+          [ "$STAGE_OUTCOME" != "success" ] && ICON="❌"
+          echo "- $ICON **Stage 3/5**: Hugo multi-build + gzip complete ($STAGE_OUTCOME)" >> "$GITHUB_STEP_SUMMARY"
+
+      # ----------------------------------------------------------------------
+      # Stage 4/5: Build & push privileged. Builder layers come from cache;
+      # only the runtime stage (COPY + LABEL) actually runs, then push.
+      # ----------------------------------------------------------------------
+      - name: "Stage 4/5: Build & push privileged"
+        id: stage4
         if: inputs.variant != 'unprivileged'
         uses: docker/build-push-action@v7
         with:
@@ -179,12 +252,20 @@ jobs:
           cache-to: type=gha,mode=max
           tags: ${{ steps.meta.outputs.priv_tags }}
           push: true
+      - name: "Stage 4/5: summary"
+        if: always() && steps.stage4.conclusion != 'skipped'
+        env:
+          STAGE_OUTCOME: ${{ steps.stage4.outcome }}
+        run: |
+          ICON="✅"
+          [ "$STAGE_OUTCOME" != "success" ] && ICON="❌"
+          echo "- $ICON **Stage 4/5**: Privileged pushed to DockerHub ($STAGE_OUTCOME)" >> "$GITHUB_STEP_SUMMARY"
 
       # ----------------------------------------------------------------------
-      # Step 4: unprivileged. The builder stage is CACHED from step 3 (same
-      # runner, in-memory BuildKit). Only the runtime stage actually runs.
+      # Stage 5/5: Build & push unprivileged
       # ----------------------------------------------------------------------
-      - name: Build & push unprivileged
+      - name: "Stage 5/5: Build & push unprivileged"
+        id: stage5
         if: inputs.variant != 'privileged'
         uses: docker/build-push-action@v7
         with:
@@ -199,10 +280,18 @@ jobs:
           cache-to: type=gha,mode=max
           tags: ${{ steps.meta.outputs.unpriv_tags }}
           push: true
+      - name: "Stage 5/5: summary"
+        if: always() && steps.stage5.conclusion != 'skipped'
+        env:
+          STAGE_OUTCOME: ${{ steps.stage5.outcome }}
+        run: |
+          ICON="✅"
+          [ "$STAGE_OUTCOME" != "success" ] && ICON="❌"
+          echo "- $ICON **Stage 5/5**: Unprivileged pushed to DockerHub ($STAGE_OUTCOME)" >> "$GITHUB_STEP_SUMMARY"
 
       # ----------------------------------------------------------------------
-      # Step 5: final summary with pull commands (runs even if build failed,
-      # so you see what was attempted)
+      # Final summary with pull/cleanup commands. Runs even on failure so
+      # you see what was attempted.
       # ----------------------------------------------------------------------
       - name: Write final summary with pull / cleanup commands
         if: always()
diff --git a/Dockerfile b/Dockerfile
@@ -3,19 +3,16 @@
 ARG VARIANT=privileged
 
 # ============================================================
-# Builder stage (shared by both variants AND both target arches)
+# Stage: deps (apt + hugo + npm + pip)
 # ============================================================
-# Force the builder to run on the host's native platform — the output is
-# static HTML/CSS/JS that's identical regardless of target arch, so there's
-# no point running Hugo + npm + pip twice (once natively and once under
-# QEMU emulation) for a multi-arch build. Each target's runtime stage
-# COPYs the same /site/public out of this single builder.
-FROM --platform=$BUILDPLATFORM node:24-trixie AS builder
+# Force builder-side stages to run on the host's native platform — the output
+# is static HTML/CSS/JS that's identical regardless of target arch, so there's
+# no point running Hugo + npm + pip twice under QEMU emulation. Each target's
+# runtime stage COPYs the same /site/public out of the final builder stage.
+FROM --platform=$BUILDPLATFORM node:24-trixie AS deps
 
 ARG HUGO_VERSION=0.143.1
 ARG BUILDARCH
-ARG GIT_COMMIT=unknown
-ARG BUILD_DATE=unknown
 
 RUN apt-get update && apt-get install -y --no-install-recommends \
     python3 \
@@ -40,28 +37,45 @@ RUN npm install
 COPY requirements.txt ./
 RUN python3 -m venv /venv && /venv/bin/pip install -r requirements.txt
 
+# ============================================================
+# Stage: components (COPY workspace + make components)
+# ============================================================
+FROM deps AS components
+
 COPY . .
 
 ENV PATH="/venv/bin:$PATH"
 
 RUN sed -i 's#baseURL = "https://redis.io"#baseURL = "/"#g' config.toml
 # Hugo per-partial timeout: upstream sets 75s, which fits CI but not multi-platform
-# Docker builds where dynacache is constantly evicted under memory pressure and
-# QEMU-emulated amd64 plus parallel arm64 share a single host's resources.
+# Docker builds where dynacache is constantly evicted under memory pressure.
 RUN sed -i 's/timeout="75"/timeout="600"/' config.toml
 
-# Fetch external client repos (clones into examples/). Cached unless deps or
-# build/make.py change. Cannot move into the multi-build below because each
-# version build resets the workspace; we want examples/ in the snapshot.
+# Fetch external client repos (clones into examples/). Cannot move into the
+# multi-build below because each version build resets the workspace; we want
+# examples/ in the snapshot.
 RUN --mount=type=secret,id=PRIVATE_ACCESS_TOKEN,env=PRIVATE_ACCESS_TOKEN \
     make components
 
+# ============================================================
+# Stage: builder (multi-version Hugo build + gzip pre-compression)
+# ============================================================
+FROM components AS builder
+
 # Multi-build pipeline: latest + one Hugo invocation per (product, version),
 # then merged into a single public/ tree. See airgap-multibuild.sh.
-# Mirrors .github/workflows/main.yml's parallel matrix as a sequential loop.
-RUN bash airgap-multibuild.sh
-
-RUN find /site/public -type f \( -name "*.html" -o -name "*.css" -o -name "*.js" -o -name "*.json" -o -name "*.xml" -o -name "*.svg" -o -name "*.txt" \) \
+#
+# The cache mount preserves per-version Hugo outputs across builds. The script
+# computes a content-hash per version and reuses cached outputs when the hash
+# matches — so a merge that only touches one version's content rebuilds only
+# that version, not the other 27.
+RUN --mount=type=cache,target=/var/cache/airgap-versions \
+    bash airgap-multibuild.sh
+
+# Pre-compress static assets that nginx serves via gzip_static. Skips .md and
+# .json because nginx runs sub_filter on those at request time (gzip_static is
+# OFF for those locations — pre-compressing them would be wasted CPU).
+RUN find /site/public -type f \( -name "*.html" -o -name "*.css" -o -name "*.js" -o -name "*.xml" -o -name "*.svg" -o -name "*.txt" \) \
     -exec gzip -9 -k {} \;
 
 # ============================================================
diff --git a/airgap-multibuild.sh b/airgap-multibuild.sh
@@ -17,6 +17,31 @@ set -euo pipefail
 readonly SITE=/site
 readonly SNAPSHOT=/tmp/site-snapshot
 readonly FINAL=/tmp/public-final
+# Per-version output cache. The Dockerfile mounts this path via
+# `RUN --mount=type=cache,target=/var/cache/airgap-versions`, so it persists
+# across builds (and across workflow runs when cache-to=type=gha,mode=max).
+readonly CACHE_ROOT=/var/cache/airgap-versions
+mkdir -p "$CACHE_ROOT"
+
+# Hash everything that affects rendering for a given (product, version):
+# the version's own content, shared layouts, data, top-level configs, and
+# the script itself. Identical inputs → identical key → cache reuse.
+compute_version_cache_key() {
+  local product_path=$1
+  local version=$2
+  {
+    [ -d "$SNAPSHOT/content/$product_path/$version" ] && \
+      find "$SNAPSHOT/content/$product_path/$version" -type f -print0 | sort -z | xargs -0 sha256sum
+    find "$SNAPSHOT/layouts" -type f -print0 | sort -z | xargs -0 sha256sum
+    [ -d "$SNAPSHOT/data" ] && \
+      find "$SNAPSHOT/data" -type f -print0 | sort -z | xargs -0 sha256sum
+    for f in config.toml postcss.config.js tailwind.config.js package-lock.json; do
+      [ -f "$SNAPSHOT/$f" ] && sha256sum "$SNAPSHOT/$f"
+    done
+    sha256sum "$SITE/airgap-multibuild.sh"
+    printf 'product=%s version=%s\n' "$product_path" "$version"
+  } 2>/dev/null | sha256sum | cut -c1-16
+}
 
 cd "$SITE"
 
@@ -116,7 +141,26 @@ build_version() {
   local version=$4
   local all_versions=$5     # newline-separated version list for this product
 
-  echo ">>> Building ${product_path} v${version}"
+  local cache_key
+  cache_key=$(compute_version_cache_key "$product_path" "$version")
+  local pp_slug="${product_path//\//-}"
+  local cache_dir="$CACHE_ROOT/${pp_slug}-v${version}-${cache_key}"
+
+  if [ -d "$cache_dir/version" ]; then
+    echo ">>> [CACHE HIT] ${product_path} v${version} (key=${cache_key})"
+    mkdir -p "$FINAL/$product_path"
+    rm -rf "$FINAL/$product_path/$version"
+    cp -a "$cache_dir/version" "$FINAL/$product_path/$version"
+    for d in css scss; do
+      if [ -d "$cache_dir/$d" ]; then
+        mkdir -p "$FINAL/$d"
+        cp -an "$cache_dir/$d/." "$FINAL/$d/"
+      fi
+    done
+    return
+  fi
+
+  echo ">>> [CACHE MISS] Building ${product_path} v${version} (key=${cache_key})"
   reset_workspace
   cd "$SITE"
 
@@ -190,6 +234,21 @@ build_version() {
     fi
   done
 
+  # Save the fresh build to the per-version cache so the next build with the
+  # same inputs is a cache hit. We use a temp dir + atomic rename so a build
+  # killed mid-write doesn't poison the cache.
+  local cache_tmp="${cache_dir}.tmp.$$"
+  rm -rf "$cache_tmp"
+  mkdir -p "$cache_tmp"
+  cp -a "$SITE/public/$product_path/$version" "$cache_tmp/version"
+  for d in css scss; do
+    if [ -d "$SITE/public/$d" ]; then
+      cp -a "$SITE/public/$d" "$cache_tmp/$d"
+    fi
+  done
+  rm -rf "$cache_dir"
+  mv "$cache_tmp" "$cache_dir"
+
   rm -rf "$SITE/public"
 }
 
