IONOS(ci): introduce prepare-matrix + build-apps parallel jobs

printminion-co · printminion-co · commit 13b745d66f14 · 2026-05-12T17:38:50.000+02:00
Adds the cached parallel matrix infrastructure to hidrive-next-build.yml as
two new jobs. The legacy monolithic hidrive-next-build job that builds
everything inline is preserved unchanged in this commit — the next commit
rewires it to consume the per-app artifacts produced here.

prepare-matrix
  - Computes effective_cache_version from CACHE_VERSION + optional
    workflow_dispatch input cache_version_suffix
  - Generates the build matrix by invoking make -f IONOS/Makefile
    generate_apps_matrix_json (a 7-entry sorted JSON, one per IONOS app)
  - Probes each app's per-SHA cache via .github/scripts/detect-app-cache.sh
    (JFrog first, GitHub Actions cache as fallback)
  - Emits apps_to_build / apps_to_restore / apps_matrix outputs

build-apps
  - Parallel matrix (max-parallel: 7) keyed on apps_to_build entries
  - Setup PHP + Node per-app, run make -f IONOS/Makefile
    &lt;app&gt;-target&gt;, tar the result and upload to
    JFrog under \${ARTIFACTORY_REPOSITORY_SNAPSHOT}/apps/\${VERSION}/...,
    and save to GitHub Actions cache as a fallback restore source

Adds CACHE_VERSION env (v1.2) and three workflow_dispatch inputs
(force_rebuild, cache_version_suffix, apps_to_rebuild) needed by the new
jobs. At this commit the new jobs run alongside the inline build and
upload their artifacts but do not yet feed the final image — both paths
produce the same outputs in parallel until the next commit retires the
inline path.

Signed-off-by: Misha M.-Kupriyanov &lt;kupriyanov@strato.de&gt;
diff --git a/.github/workflows/hidrive-next-build.yml b/.github/workflows/hidrive-next-build.yml
@@ -4,9 +4,10 @@ name: HiDrive Next Build
 # SPDX-FileCopyrightText: 2024 STRATO AG
 # SPDX-License-Identifier: AGPL-3.0-or-later
 
-# The HiDrive Next source is packaged as a container image.
-# This is a workaround because releases can not be created without tags
-# and we want to be able to create snapshots from branches.
+# Cached parallel matrix build pipeline:
+# Stage 1: prepare-matrix — compute app build matrix from IONOS/Makefile and probe per-app caches
+# Stage 2: build-apps — parallel matrix per app, per-app SHA caching in JFrog + GitHub Actions cache
+# Final:   hidrive-next-build — restore all per-app artifacts and build Nextcloud core
 
 on:
   pull_request:
@@ -31,17 +32,277 @@ on:
     branches:
       - ionos-dev
       - ionos-stable
+  workflow_dispatch:
+    inputs:
+      force_rebuild:
+        description: 'Force rebuild all apps and dependencies (bypass ALL caches)'
+        required: false
+        type: boolean
+        default: false
+      cache_version_suffix:
+        description: 'Optional cache version suffix (e.g., "test", "debug") - creates separate cache namespace'
+        required: false
+        type: string
+        default: ''
+      apps_to_rebuild:
+        description: 'Comma-separated list of specific apps to rebuild (e.g., "simplesettings,viewer")'
+        required: false
+        type: string
+        default: ''
 
 env:
   TARGET_PACKAGE_NAME: hidrive-next.zip
   REGISTRY: ghcr.io
   IMAGE_NAME: ${{ github.repository }}
   ARTIFACTORY_REPOSITORY_SNAPSHOT: ionos-productivity-hdnext-snapshot
+  # Cache version - increment to invalidate all caches when build tooling changes
+  # Format: v<major>.<minor> (e.g., v1.0, v1.1, v2.0)
+  CACHE_VERSION: v1.2
 
 permissions:
   contents: read
 
 jobs:
+  prepare-matrix:
+    runs-on: ubuntu-latest
+    outputs:
+      apps_to_build: ${{ steps.detect.outputs.apps_to_build }}
+      apps_to_restore: ${{ steps.detect.outputs.apps_to_restore }}
+      apps_matrix: ${{ steps.set_matrix.outputs.matrix }}
+      apps_sha_map: ${{ steps.detect.outputs.apps_sha_map }}
+      has_apps_to_build: ${{ steps.detect.outputs.has_apps_to_build }}
+      has_apps_to_restore: ${{ steps.detect.outputs.has_apps_to_restore }}
+      effective_cache_version: ${{ steps.compute_cache_version.outputs.effective_cache_version }}
+
+    permissions:
+      contents: read
+      actions: read
+
+    steps:
+      - name: Compute effective cache version
+        id: compute_cache_version
+        run: |
+          EFFECTIVE_VERSION="${{ env.CACHE_VERSION }}${{ github.event.inputs.cache_version_suffix && format('-{0}', github.event.inputs.cache_version_suffix) || '' }}"
+          echo "effective_cache_version=$EFFECTIVE_VERSION" >> "$GITHUB_OUTPUT"
+          echo "Effective cache version: $EFFECTIVE_VERSION"
+
+      - name: Checkout repository
+        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 #v4.1.7
+        with:
+          submodules: true
+          fetch-depth: 1
+
+      - name: Install dependencies
+        run: sudo apt-get update && sudo apt-get install -y make jq
+
+      - name: Check JFrog credentials
+        id: jfrog-available
+        if: github.event.inputs.force_rebuild != 'true'
+        env:
+          JF_URL: ${{ secrets.JF_ARTIFACTORY_URL }}
+          JF_USER: ${{ secrets.JF_ARTIFACTORY_USER }}
+          JF_ACCESS_TOKEN: ${{ secrets.JF_ACCESS_TOKEN }}
+        run: |
+          if [ -n "$JF_URL" ] && [ -n "$JF_USER" ] && [ -n "$JF_ACCESS_TOKEN" ]; then
+            echo "available=true" >> $GITHUB_OUTPUT
+          else
+            echo "available=false" >> $GITHUB_OUTPUT
+            echo "⚠ Full JFrog credential set not available — skipping JFrog checks"
+          fi
+
+      - name: Setup JFrog CLI
+        if: steps.jfrog-available.outputs.available == 'true'
+        uses: jfrog/setup-jfrog-cli@7c95feb32008765e1b4e626b078dfd897c4340ad # v4.4.1
+        env:
+          JF_URL: ${{ secrets.JF_ARTIFACTORY_URL }}
+          JF_USER: ${{ secrets.JF_ARTIFACTORY_USER }}
+          JF_ACCESS_TOKEN: ${{ secrets.JF_ACCESS_TOKEN }}
+
+      - name: Generate apps matrix from Makefile
+        id: set_matrix
+        run: |
+          echo "Generating apps matrix from Makefile..."
+          matrix_output=$(make -f IONOS/Makefile generate_apps_matrix_json 2>&1)
+          if echo "$matrix_output" | grep -q '^\[i\]'; then
+            matrix=$(echo "$matrix_output" | grep -v '^\[i\]')
+          else
+            matrix="$matrix_output"
+          fi
+          if ! echo "$matrix" | jq empty 2>/dev/null; then
+            echo "Error: Generated matrix is not valid JSON"
+            echo "Output: $matrix_output"
+            exit 1
+          fi
+          echo "matrix=$(echo "$matrix" | jq -c '.')" >> $GITHUB_OUTPUT
+          echo "Matrix generated with $(echo "$matrix" | jq 'length') apps"
+
+      - name: Collect apps SHA and check cache status
+        id: detect
+        env:
+          GH_TOKEN: ${{ github.token }}
+          CACHE_VERSION: ${{ steps.compute_cache_version.outputs.effective_cache_version }}
+          FORCE_REBUILD: ${{ github.event.inputs.force_rebuild || 'false' }}
+          APPS_TO_REBUILD: ${{ github.event.inputs.apps_to_rebuild || '' }}
+          JF_URL: ${{ secrets.JF_ARTIFACTORY_URL }}
+          JF_USER: ${{ secrets.JF_ARTIFACTORY_USER }}
+          JF_ACCESS_TOKEN: ${{ secrets.JF_ACCESS_TOKEN }}
+          ARTIFACTORY_REPOSITORY_SNAPSHOT: ${{ env.ARTIFACTORY_REPOSITORY_SNAPSHOT }}
+          GITHUB_REF: ${{ github.ref }}
+        run: |
+          bash .github/scripts/detect-app-cache.sh '${{ steps.set_matrix.outputs.matrix }}'
+
+  build-apps:
+    runs-on: ubuntu-latest
+    needs: prepare-matrix
+    if: |
+      always() &&
+      needs.prepare-matrix.result == 'success' &&
+      needs.prepare-matrix.outputs.has_apps_to_build == 'true'
+
+    permissions:
+      contents: read
+      actions: write
+
+    name: build-apps
+    strategy:
+      max-parallel: 7
+      matrix:
+        app_info: ${{ fromJson(needs.prepare-matrix.outputs.apps_to_build) }}
+
+    steps:
+      - name: Get app configuration from full matrix
+        id: app-config
+        run: |
+          FULL_MATRIX='${{ needs.prepare-matrix.outputs.apps_matrix }}'
+          APP_NAME='${{ matrix.app_info.name }}'
+
+          APP_CONFIG=$(echo "$FULL_MATRIX" | jq -c --arg name "$APP_NAME" '.[] | select(.name == $name)')
+          if [ -z "$APP_CONFIG" ]; then
+            echo "ERROR: Could not find configuration for $APP_NAME in matrix"
+            exit 1
+          fi
+
+          echo "path=$(echo "$APP_CONFIG" | jq -r '.path')" >> $GITHUB_OUTPUT
+          echo "has-npm=$(echo "$APP_CONFIG" | jq -r '.has_npm')" >> $GITHUB_OUTPUT
+          echo "has-composer=$(echo "$APP_CONFIG" | jq -r '.has_composer')" >> $GITHUB_OUTPUT
+          echo "npm-lock-path=$(echo "$APP_CONFIG" | jq -r '.npm_lock_path')" >> $GITHUB_OUTPUT
+          echo "makefile-target=$(echo "$APP_CONFIG" | jq -r '.makefile_target')" >> $GITHUB_OUTPUT
+          echo "Building $APP_NAME (SHA: ${{ matrix.app_info.sha }})"
+
+      - name: Checkout repository
+        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 #v4.1.7
+        with:
+          submodules: true
+          fetch-depth: 1
+
+      - name: Check JFrog credentials
+        id: jfrog-creds
+        env:
+          JF_URL: ${{ secrets.JF_ARTIFACTORY_URL }}
+          JF_USER: ${{ secrets.JF_ARTIFACTORY_USER }}
+          JF_ACCESS_TOKEN: ${{ secrets.JF_ACCESS_TOKEN }}
+        run: |
+          if [ -n "$JF_URL" ] && [ -n "$JF_USER" ] && [ -n "$JF_ACCESS_TOKEN" ]; then
+            echo "available=true" >> $GITHUB_OUTPUT
+          else
+            echo "available=false" >> $GITHUB_OUTPUT
+            echo "⚠ Full JFrog credential set not available — using GitHub cache path where possible"
+          fi
+
+      - name: Setup JFrog CLI
+        if: steps.jfrog-creds.outputs.available == 'true'
+        uses: jfrog/setup-jfrog-cli@7c95feb32008765e1b4e626b078dfd897c4340ad # v4.4.1
+        env:
+          JF_URL: ${{ secrets.JF_ARTIFACTORY_URL }}
+          JF_USER: ${{ secrets.JF_ARTIFACTORY_USER }}
+          JF_ACCESS_TOKEN: ${{ secrets.JF_ACCESS_TOKEN }}
+
+      - name: Set up node
+        if: steps.app-config.outputs.has-npm == 'true'
+        uses: actions/setup-node@60edb5dd545a775178f52524783378180af0d1f8 # v4.0.2
+        with:
+          node-version-file: "package.json"
+          cache: 'npm'
+          cache-dependency-path: ${{ steps.app-config.outputs.npm-lock-path }}
+
+      - name: Setup PHP
+        if: steps.app-config.outputs.has-composer == 'true'
+        uses: shivammathur/setup-php@c541c155eee45413f5b09a52248675b1a2575231 #v2.31.1
+        with:
+          tools: composer:v2
+          extensions: gd, zip, curl, xml, xmlrpc, mbstring, sqlite, xdebug, pgsql, intl, imagick, gmp, apcu, bcmath, redis, soap, imap, opcache
+        env:
+          runner: ubuntu-latest
+
+      - name: Cache Composer dependencies for ${{ matrix.app_info.name }}
+        if: steps.app-config.outputs.has-composer == 'true' && github.event.inputs.force_rebuild != 'true'
+        uses: actions/cache@v4
+        with:
+          path: ${{ steps.app-config.outputs.path }}/vendor
+          key: ${{ runner.os }}-composer-${{ matrix.app_info.name }}-${{ hashFiles(format('{0}/composer.lock', steps.app-config.outputs.path)) }}
+          restore-keys: |
+            ${{ runner.os }}-composer-${{ matrix.app_info.name }}-
+
+      - name: Build ${{ matrix.app_info.name }}
+        env:
+          CYPRESS_INSTALL_BINARY: 0
+          PUPPETEER_SKIP_DOWNLOAD: true
+        run: make -f IONOS/Makefile ${{ steps.app-config.outputs.makefile-target }}
+
+      - name: Compute app cache key
+        id: app-cache-key
+        run: |
+          APP_SHA="${{ matrix.app_info.sha }}"
+          APP_NAME="${{ matrix.app_info.name }}"
+          EFFECTIVE_VERSION="${{ needs.prepare-matrix.outputs.effective_cache_version }}"
+          SHORT_SHA="${APP_SHA:0:8}"
+
+          echo "cache_key=${EFFECTIVE_VERSION}-app-build-${APP_NAME}-${SHORT_SHA}" >> $GITHUB_OUTPUT
+          echo "archive_name=${APP_NAME}-${SHORT_SHA}.tar.gz" >> $GITHUB_OUTPUT
+
+      - name: Upload ${{ matrix.app_info.name }} to JFrog
+        if: steps.jfrog-creds.outputs.available == 'true'
+        run: |
+          APP_NAME="${{ matrix.app_info.name }}"
+          APP_SHA="${{ matrix.app_info.sha }}"
+          APP_PATH="${{ steps.app-config.outputs.path }}"
+          EFFECTIVE_VERSION="${{ needs.prepare-matrix.outputs.effective_cache_version }}"
+          ARCHIVE_NAME="${{ steps.app-cache-key.outputs.archive_name }}"
+          JFROG_PATH="${{ env.ARTIFACTORY_REPOSITORY_SNAPSHOT }}/apps/${EFFECTIVE_VERSION}/${APP_NAME}/${ARCHIVE_NAME}"
+
+          echo "Packaging $APP_NAME..."
+          tar -czf "$ARCHIVE_NAME" \
+            --exclude="node_modules" \
+            --exclude=".git" \
+            --exclude="*.log" \
+            -C "$(dirname "$APP_PATH")" \
+            "$(basename "$APP_PATH")"
+
+          echo "Archive size: $(ls -lh "$ARCHIVE_NAME" | awk '{print $5}')"
+          echo "Uploading to: $JFROG_PATH"
+
+          if jf rt upload "$ARCHIVE_NAME" "$JFROG_PATH" \
+            --target-props "app.name=${APP_NAME};app.sha=${APP_SHA};vcs.branch=${{ github.ref_name }};vcs.revision=${{ github.sha }}"; then
+            echo "✅ Uploaded $APP_NAME to JFrog"
+          else
+            echo "❌ JFrog upload failed"
+            exit 1
+          fi
+          rm -f "$ARCHIVE_NAME"
+
+      - name: Save build to GitHub Actions cache
+        uses: actions/cache/save@v4
+        with:
+          path: ${{ steps.app-config.outputs.path }}
+          key: ${{ steps.app-cache-key.outputs.cache_key }}
+
+      - name: Show changes on failure
+        if: failure()
+        run: |
+          git status
+          git --no-pager diff
+          exit 1
+
   hidrive-next-build:
     runs-on: ubuntu-latest
 
