Merge pull request #277 from WyriHaximusNet/move-towards-uniform-image-builder-tooling-phase-I

Move towards uniform image building tooling: Phase I

Author: WyriHaximus

.github/workflows/ci.yml

@@ -29,17 +29,16 @@ jobs:
         uses: wyrihaximus/github-action-supported-php-versions@v1
         with:
           upcomingReleases: true
-  supported-arch-matrix:
-    name: Supported processor architectures
+  supported-platform-matrix:
+    name: Supported platforms
     runs-on: ubuntu-latest
     outputs:
-      arch: ${{ steps.supported-arch-matrix.outputs.arch }}
+      platform: ${{ steps.supported-platform-matrix.outputs.platform }}
     steps:
-      - uses: actions/checkout@v4
-      - id: supported-arch-matrix
-        name: Generate Arch
+      - id: supported-platform-matrix
+        name: Generate Platform list
         run: |
-          echo "arch=[\\\"amd64\\\",\\\"arm64\\\"]" >> $GITHUB_OUTPUT
+          echo "platform=[\"linux/amd64\",\"linux/arm64\"]" >> $GITHUB_OUTPUT
   image-type-matrix:
     name: Create Image Type Matrix
     runs-on: ubuntu-latest
@@ -53,18 +52,6 @@ jobs:
           ls Dockerfile-* | jq -csR '. | rtrimstr("\n") | split("\n")' | php -r "echo str_replace('Dockerfile-', '', stream_get_contents(STDIN));" > types.list
           cat types.list
           echo "type=$(cat types.list)" >> $GITHUB_OUTPUT
-  exclude-matrix:
-    name: Create Exclude Matrix
-    runs-on: ubuntu-latest
-    outputs:
-      exclude: ${{ steps.exclude-matrix.outputs.exclude }}
-      exclude-push: ${{ steps.exclude-matrix.outputs.exclude-push }}
-    steps:
-      - uses: actions/checkout@v4
-      - id: exclude-matrix
-        name: Generate Exclude
-        run: |
-          php utils/exclude-list.php
   image-matrix:
     name: Create Image Matrix
     runs-on: ubuntu-latest
@@ -132,98 +119,115 @@ jobs:
           entrypoint: hadolint
           args: Dockerfile-${{ matrix.type }}
   build:
-    name: Building "${{ matrix.image }}"
+    name: Building "${{ matrix.image }}" on ${{ matrix.platform }}
     needs:
       - lint
       - image-matrix
-      - supported-arch-matrix
-      - exclude-matrix
-    runs-on: ubuntu-latest
+      - supported-platform-matrix
+    runs-on: ${{ contains(matrix.platform, 'arm') && 'ubuntu-24.04-arm' || 'ubuntu-24.04' }}
     strategy:
       fail-fast: false
       matrix:
         image: ${{ fromJson(needs.image-matrix.outputs.image) }}
-        exclude: ${{ fromJson(needs.exclude-matrix.outputs.exclude) }}
+        platform: ${{ fromJson(needs.supported-platform-matrix.outputs.platform) }}
     steps:
+      - name: Prepare
+        run: |
+          platform=${{ matrix.platform }}
+          echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV
+          printf "PLATFORM_ARCH=%s" $(echo "${{ matrix.platform }}" | cut -d "/" -f 2) >> $GITHUB_ENV
+      - name: Runner Arch
+        run: echo "${{ runner.arch }}"
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v3
+        if: ${{ !contains(matrix.platform, 'arm') && !contains(matrix.platform, 'amd64') }}
       - uses: actions/checkout@v4
       - run: mkdir ./docker-image/
       - uses: nick-invision/retry@v3
         with:
           timeout_minutes: 120
           retry_wait_seconds: 30
           max_attempts: 50
-          command: (echo "${{ needs.supported-arch-matrix.outputs.arch }}" | jq -r '.[]') | xargs -I % ./build-php.sh $(echo "${{ matrix.image }}" | tr '-' ' ') %
-      - run: cat ./docker-image/image.tags | xargs -I % docker inspect --format='%={{.Id}}:{{index .Config.Env 7}}' %
-      - run: docker save "${DOCKER_IMAGE}" | gzip -9 > ./docker-image/image.tar
+          command: ./build-php.sh $(echo "${{ matrix.image }}" | tr '-' ' ') $(echo "${{ matrix.platform }}" | cut -d "/" -f 2)
+      - run: mv ./docker-image/image.tags ./docker-image/image-${{ matrix.image }}-${{ env.PLATFORM_PAIR }}.tags
+      - run: cat ./docker-image/image-${{ matrix.image }}-${{ env.PLATFORM_PAIR }}.tags | xargs -I % docker inspect --format='%={{.Id}}:{{index .Config.Env 7}}' %
+      - run: docker save "${DOCKER_IMAGE}" -o ./docker-image/docker_image-${{ env.PLATFORM_PAIR }}.tar
       - run: docker images
       - name: Upload Images
         uses: actions/upload-artifact@v4
         with:
-          name: docker-image-${{ matrix.image }}
+          name: docker-image-${{ matrix.image }}-${{ env.PLATFORM_PAIR }}
           path: ./docker-image
   scan-vulnerability:
-    name: Scanning "${{ matrix.image }}" for vulnerabilities
+    name: Scanning "${{ matrix.image }}" on ${{ matrix.platform }} for vulnerabilities
     needs:
       - build
       - image-matrix
-      - supported-arch-matrix
-      - exclude-matrix
-    runs-on: ubuntu-latest
+      - supported-platform-matrix
+    runs-on: ${{ contains(matrix.platform, 'arm') && 'ubuntu-24.04-arm' || 'ubuntu-24.04' }}
     strategy:
       fail-fast: false
       matrix:
         image: ${{ fromJson(needs.image-matrix.outputs.image) }}
-        exclude: ${{ fromJson(needs.exclude-matrix.outputs.exclude) }}
+        platform: ${{ fromJson(needs.supported-platform-matrix.outputs.platform) }}
     steps:
+      - name: Prepare
+        run: |
+          platform=${{ matrix.platform }}
+          echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV
+          printf "PLATFORM_ARCH=%s" $(echo "${{ matrix.platform }}" | cut -d "/" -f 2) >> $GITHUB_ENV
+      - name: Runner Arch
+        run: echo "${{ runner.arch }}"
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v3
-        if: contains(matrix.image, 'alpine')
+        if: ${{ !contains(matrix.platform, 'arm') && !contains(matrix.platform, 'amd64') }}
       - uses: actions/checkout@v4
-        if: contains(matrix.image, 'alpine')
       - name: Download Images
-        if: contains(matrix.image, 'alpine')
         uses: actions/download-artifact@v4
         with:
-          name: docker-image-${{ matrix.image }}
+          name: docker-image-${{ matrix.image }}-${{ env.PLATFORM_PAIR }}
           path: ./docker-image
-      - run: docker load --input ./docker-image/image.tar
-        if: contains(matrix.image, 'alpine')
+      - run: ls -lasth ./docker-image/
+      - run: mv ./docker-image/image-${{ matrix.image }}-${{ env.PLATFORM_PAIR }}.tags ./docker-image/image.tags
+      - run: ls -lasth ./docker-image/
+      - run: docker load --input ./docker-image/docker_image-${{ env.PLATFORM_PAIR }}.tar
       - run: make ci-scan-vulnerability
-        if: contains(matrix.image, 'alpine')
   test:
-    name: Testing "${{ matrix.image }}"
+    name: Testing "${{ matrix.image }}" on ${{ matrix.platform }}
     needs:
       - build
       - image-matrix
-      - supported-arch-matrix
-      - exclude-matrix
-    runs-on: ubuntu-latest
+      - supported-platform-matrix
+    runs-on: ${{ contains(matrix.platform, 'arm') && 'ubuntu-24.04-arm' || 'ubuntu-24.04' }}
     strategy:
       fail-fast: false
       matrix:
         image: ${{ fromJson(needs.image-matrix.outputs.image) }}
-        exclude: ${{ fromJson(needs.exclude-matrix.outputs.exclude) }}
+        platform: ${{ fromJson(needs.supported-platform-matrix.outputs.platform) }}
     steps:
+      - name: Prepare
+        run: |
+          platform=${{ matrix.platform }}
+          echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV
+          printf "PLATFORM_ARCH=%s" $(echo "${{ matrix.platform }}" | cut -d "/" -f 2) >> $GITHUB_ENV
+      - name: Runner Arch
+        run: echo "${{ runner.arch }}"
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v3
+#        if: ${{ !contains(matrix.platform, 'arm') && !contains(matrix.platform, 'amd64') }}
       - uses: actions/checkout@v4
       - name: Download Images
         uses: actions/download-artifact@v4
         with:
-          name: docker-image-${{ matrix.image }}
+          name: docker-image-${{ matrix.image }}-${{ env.PLATFORM_PAIR }}
           path: ./docker-image
       - run: ls -lasth ./docker-image/
-      - run: docker load --input ./docker-image/image.tar
-      - uses: nick-invision/retry@v3
-        with:
-          timeout_minutes: 120
-          retry_wait_seconds: 30
-          max_attempts: 50
-          command: |
-            export IMAGE_BASE_VERSION=$(php -r 'echo explode("-", "${{ matrix.image }}")[2];') 
-            (echo "${{ needs.supported-arch-matrix.outputs.arch }}" | jq -r '.[]') | xargs -I % make $(php -r 'echo "test-", explode("-", str_replace(["zts-zts", "cli-nts"], ["zts", "nts"], "${{ matrix.image }}"))[0];') IMAGE_ARCH=%
+      - run: mv ./docker-image/image-${{ matrix.image }}-${{ env.PLATFORM_PAIR }}.tags ./docker-image/image.tags
+      - run: ls -lasth ./docker-image/
+      - run: docker load --input ./docker-image/docker_image-${{ env.PLATFORM_PAIR }}.tar
+      - run: |
+          export IMAGE_BASE_VERSION=$(php -r 'echo explode("-", "${{ matrix.image }}")[2];')
+          make $(php -r 'echo "test-", explode("-", str_replace(["zts-zts", "cli-nts"], ["zts", "nts"], "${{ matrix.image }}"))[0];') IMAGE_ARCH=$(echo "${PLATFORM_ARCH}")
       - run: rm -Rf ./docker-image/
   check-mark:
     name: ✔️
@@ -241,14 +245,17 @@ jobs:
     needs:
       - check-mark
       - image-matrix
-      - supported-arch-matrix
-      - exclude-matrix
+      - supported-platform-matrix
     runs-on: ubuntu-latest
+    services:
+      registry:
+        image: registry:3
+        ports:
+          - 5000:5000
     strategy:
       fail-fast: false
       matrix:
         image: ${{ fromJson(needs.image-matrix.outputs.image) }}
-        exclude: ${{ fromJson(needs.exclude-matrix.outputs.exclude) }}
     steps:
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v3
@@ -257,6 +264,7 @@ jobs:
         uses: docker/setup-buildx-action@v3
         with:
           install: true
+          driver-opts: network=host
       - name: Inspect builder
         run: |
           echo "Name:      ${{ steps.buildx.outputs.name }}"
@@ -265,16 +273,22 @@ jobs:
           echo "Flags:     ${{ steps.buildx.outputs.flags }}"
           echo "Platforms: ${{ steps.buildx.outputs.platforms }}"
       - uses: actions/checkout@v4
-      - name: Download Images
-        uses: actions/download-artifact@v4
+      - uses: actions/download-artifact@v4
         with:
-          name: docker-image-${{ matrix.image }}
-          path: ./docker-image
-      - run: ls -lasthR ./docker-image
-      - run: find ./docker-image/ | grep "image.tar"
-      - run: find ./docker-image/ | grep "image.tags"
+          pattern: docker-image-${{ matrix.image }}-*
+          path: /tmp/docker-image
+          merge-multiple: true
+      - run: ls -lasthR /tmp/docker-image/
+      - run: cat /tmp/docker-image/image-*.tags > /tmp/docker-image/image.tags
+      - run: ls -lasthR /tmp/docker-image/
+      - run: |
+          for f in /tmp/docker-image/docker_image-*.tar; do
+            docker load --input $f
+          done
       - run: docker images
-      - run: find ./docker-image/ | grep "image.tar" | xargs -I % sh -c 'docker load --input % && rm %'
+      - run: |
+          xargs -I % docker tag "%" "localhost:5000/%" < /tmp/docker-image/image.tags
+          xargs -I % docker push "localhost:5000/%" < /tmp/docker-image/image.tags
       - run: docker images
       - name: Login to container registries
         uses: nick-invision/retry@v3
@@ -290,29 +304,17 @@ jobs:
           HUB_PASSCODE: ${{ secrets.HUB_PASSCODE }}
       - name: Docker info
         run: docker info
-      - name: Retag
-        uses: nick-invision/retry@v3
-        with:
-          timeout_minutes: 120
-          retry_wait_seconds: 30
-          max_attempts: 5
-          command: |
-            (jq -r 'to_entries | map("(find ./docker-image/ | grep image.tags | xargs -I $ cat $) | xargs -I % docker tag % " + .key + "/%") | .[]' <<<"$DOCKER_IMAGE_REGISTRIES_SECRET_MAPPING") | sh
-            (jq -r 'to_entries | map("(find ./docker-image/ | grep image.tags | xargs -I $ cat $) | xargs -I % docker push " + .key + "/%") | .[]' <<<"$DOCKER_IMAGE_REGISTRIES_SECRET_MAPPING") | sh
-      - run: docker images
-      - name: Push all images to registries
-        uses: nick-invision/retry@v3
-        with:
-          timeout_minutes: 120
-          retry_wait_seconds: 30
-          max_attempts: 5
-          command: |
-            find ./docker-image/ | grep image.tags | xargs -I $ cat $ > tags-to-push.list
-            touch command.sh
-            (jq -r 'to_entries | map("php utils/create-manifest-command.php " + .key) | .[]' <<<"$DOCKER_IMAGE_REGISTRIES_SECRET_MAPPING") | sh
-            chmod +x command.sh
-            cat ./command.sh
-            ./command.sh
+      - name: Collect non-Arch suffixed tags
+        run: |
+          php utils/strip-arch-and-deduplicate.php ${{ env.DOCKER_IMAGE }} > /tmp/tags-to-push.list
+          cat /tmp/tags-to-push.list
         env:
-          TARGET_ARCHS: ${{ needs.supported-arch-matrix.outputs.arch }}
-      - run: docker images
+          ARCHS: ${{ join(fromJson(needs.supported-platform-matrix.outputs.platform), ',') }}
+      - name: Create merge Dockerfiles
+        run: |
+          cat /tmp/tags-to-push.list | xargs -I % sh -c 'echo "FROM localhost:5000/${{ env.DOCKER_IMAGE }}:%-\${TARGETARCH}" >> docker-file-%'
+          cat docker-file-*
+          ls -lasth docker-file-*
+      - name: Push all images to registries
+        run: |
+          (jq -r 'to_entries | map("cat /tmp/tags-to-push.list | xargs -I % docker buildx build -f docker-file-% -t " + .key + "/${{ env.DOCKER_IMAGE }}:% --platform=${{ join(fromJson(needs.supported-platform-matrix.outputs.platform), ',') }} --push  .") | .[]' <<<"$DOCKER_IMAGE_REGISTRIES_SECRET_MAPPING") | sh