Merge pull request #168 from WyriHaximusNet/use-qemu-static-for-multi-arch-images

WyriHaximus · web-flow · commit 94c330748b54 · 2021-09-14T08:06:39.000+02:00
Use qemu static for multi arch images
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -41,19 +41,17 @@ jobs:
         uses: wyrihaximus/github-action-supported-php-versions@v1
         with:
           upcomingReleases: true
-  registry-matrix:
-    name: Extract registries from registry secret mapping
-    if: (github.event_name == 'push' || github.event_name == 'schedule') && github.ref == 'refs/heads/master'
+  supported-arch-matrix:
+    name: Supported processor architectures
     runs-on: ubuntu-latest
-    needs:
-      - check-mark
     outputs:
-      registry: ${{ steps.registry-matrix.outputs.registry }}
+      arch: ${{ steps.supported-arch-matrix.outputs.arch }}
     steps:
-      - id: registry-matrix
-        name: Extract registries from registry secret mapping
+      - uses: actions/checkout@v1
+      - id: supported-arch-matrix
+        name: Generate Arch
         run: |
-          echo "::set-output name=registry::$(printenv DOCKER_IMAGE_REGISTRIES_SECRET_MAPPING | jq -c 'keys')"
+          echo "::set-output name=arch::[\"amd64\",\"arm64\",\"arm\"]"
   image-type-matrix:
     name: Create Image Type Matrix
     runs-on: ubuntu-latest
@@ -121,45 +119,52 @@ jobs:
           entrypoint: hadolint
           args: Dockerfile-${{ matrix.type }}
   build:
-    name: Building "${{ matrix.image }}"
+    name: Building "${{ matrix.image }}" on ${{ matrix.arch }}
     needs:
       - lint
       - image-matrix
+      - supported-arch-matrix
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
         image: ${{ fromJson(needs.image-matrix.outputs.image) }}
+        arch: ${{ fromJson(needs.supported-arch-matrix.outputs.arch) }}
     steps:
       - uses: actions/checkout@v2
+      - uses: dbhi/qus/action@main
       - run: mkdir ./docker-image/
-      - run: ./build-php.sh $(echo "${{ matrix.image }}" | tr '-' ' ')
+      - run: ./build-php.sh $(echo "${{ matrix.image }}" | tr '-' ' ') ${{ matrix.arch }}
       - run: cat ./docker-image/image.tags | xargs -I % docker inspect --format='%={{.Id}}:{{index .Config.Env 7}}' %
       - run: docker save "${DOCKER_IMAGE}" | gzip -9 > ./docker-image/image.tar
       - run: docker images
       - name: Upload Images
         uses: actions/upload-artifact@v2
         with:
-          name: docker-image-${{ matrix.image }}
+          name: docker-image-${{ matrix.image }}-${{ matrix.arch }}
           path: ./docker-image
   scan-vulnerability:
-    name: Scanning "${{ matrix.image }}" for vulnerabilities
+    name: Scanning "${{ matrix.image }}" for vulnerabilities on ${{ matrix.arch }}
     needs:
       - build
       - image-matrix
+      - supported-arch-matrix
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
         image: ${{ fromJson(needs.image-matrix.outputs.image) }}
+        arch: ${{ fromJson(needs.supported-arch-matrix.outputs.arch) }}
     steps:
       - uses: actions/checkout@v2
         if: contains(matrix.image, 'alpine')
+      - uses: dbhi/qus/action@main
+        if: contains(matrix.image, 'alpine')
       - name: Download Images
         if: contains(matrix.image, 'alpine')
         uses: actions/download-artifact@v2
         with:
-          name: docker-image-${{ matrix.image }}
+          name: docker-image-${{ matrix.image }}-${{ matrix.arch }}
           path: ./docker-image
       - run: docker load --input ./docker-image/image.tar
         if: contains(matrix.image, 'alpine')
@@ -168,25 +173,30 @@ jobs:
       - run: make ci-scan-vulnerability
         if: contains(matrix.image, 'alpine')
   test:
-    name: Testing "${{ matrix.image }}"
+    name: Testing "${{ matrix.image }}" on ${{ matrix.arch }}
     needs:
       - build
       - image-matrix
+      - supported-arch-matrix
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
         image: ${{ fromJson(needs.image-matrix.outputs.image) }}
+        arch: ${{ fromJson(needs.supported-arch-matrix.outputs.arch) }}
     steps:
       - uses: actions/checkout@v2
+      - uses: dbhi/qus/action@main
       - name: Download Images
         uses: actions/download-artifact@v2
         with:
-          name: docker-image-${{ matrix.image }}
+          name: docker-image-${{ matrix.image }}-${{ matrix.arch }}
           path: ./docker-image
       - run: ls -lasth ./docker-image
       - run: docker load --input ./docker-image/image.tar
       - run: IMAGE_BASE_VERSION=$(php -r 'echo explode("-", "${{ matrix.image }}")[2];') make $(php -r 'echo "test-", explode("-", str_replace(["zts-zts", "cli-nts"], ["zts", "nts"], "${{ matrix.image }}"))[0];')
+        env:
+          IMAGE_ARCH: ${{ matrix.arch }}
       - run: rm -Rf ./docker-image/
   check-mark:
     name: ✔️
@@ -198,56 +208,75 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - run: echo "✔️"
-#  push:
-#    name: Pushing "${{ matrix.image }}" to ${{ matrix.registry }}
-#    if: (github.event_name == 'push' || github.event_name == 'schedule') && github.ref == 'refs/heads/master'
-#    needs:
-#      - check-mark
-#      - registry-matrix
-#      - image-matrix
-#    runs-on: ubuntu-latest
-#    strategy:
-#      fail-fast: false
-#      matrix:
-#        image: ${{ fromJson(needs.image-matrix.outputs.image) }}
-#        registry: ${{ fromJson(needs.registry-matrix.outputs.registry) }}
-#    steps:
-#      - uses: actions/checkout@v2
-#      - name: Download Images
-#        uses: actions/download-artifact@v2
-#        with:
-#          name: docker-image-${{ matrix.image }}
-#          path: ./docker-image
-#      - run: docker load --input ./docker-image/image.tar
-#      - run: cat ./docker-image/image.tags | xargs -I % docker tag % ${{ matrix.registry }}/%
-#      - run: make ci-push
-#        env:
-#          DOCKER_USER: ${{ secrets.HUB_USERNAME }}
-#          DOCKER_PASSWORD: ${{ secrets[fromJson(env.DOCKER_IMAGE_REGISTRIES_SECRET_MAPPING)[matrix.registry]] }}
-#          DOCKER_REGISTRY: ${{ matrix.registry }}
-  build-and-push-all-archs:
-    name: Building and Pushing "${{ matrix.image }}" to ${{ matrix.registry }} for all additional archs
+  push:
+    name: Pushing "${{ matrix.image }}" for ${{ matrix.arch }}
     if: (github.event_name == 'push' || github.event_name == 'schedule') && github.ref == 'refs/heads/master'
     needs:
       - check-mark
-      - registry-matrix
       - image-matrix
+      - supported-arch-matrix
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
         image: ${{ fromJson(needs.image-matrix.outputs.image) }}
-        registry: ${{ fromJson(needs.registry-matrix.outputs.registry) }}
+        arch: ${{ fromJson(needs.supported-arch-matrix.outputs.arch) }}
     steps:
       - uses: actions/checkout@v2
-      - name: Set up Docker Buildx
-        uses: crazy-max/ghaction-docker-buildx@v3
+      - uses: dbhi/qus/action@main
+      - name: Download Images
+        uses: actions/download-artifact@v2
         with:
-          buildx-version: latest
-          qemu-version: latest
-      - run: make ci-docker-login
+          name: docker-image-${{ matrix.image }}-${{ matrix.arch }}
+          path: ./docker-image
+      - run: docker load --input ./docker-image/image.tar
+      - name: Login to container registries
+        run: |
+          (jq -r 'to_entries | map("echo \"$" + .value + "\" | docker login " + .key + " --username \"${{ env.DOCKER_USER }}\" --password-stdin") | .[]' <<<"$DOCKER_IMAGE_REGISTRIES_SECRET_MAPPING") | sh
         env:
           DOCKER_USER: ${{ secrets.HUB_USERNAME }}
-          DOCKER_PASSWORD: ${{ secrets[fromJson(env.DOCKER_IMAGE_REGISTRIES_SECRET_MAPPING)[matrix.registry]] }}
-          DOCKER_REGISTRY: ${{ matrix.registry }}
-      - run: ./buildx-php.sh $(echo "${{ matrix.image }}" | tr '-' ' ') ${{ matrix.registry }}
+          GHCR_TOKEN: ${{ secrets.GHCR_TOKEN }}
+          HUB_PASSCODE: ${{ secrets.HUB_PASSCODE }}
+      - name: Retag
+        run: |
+          (jq -r 'to_entries | map("cat ./docker-image/image.tags | xargs -I % docker tag % " + .key + "/%") | .[]' <<<"$DOCKER_IMAGE_REGISTRIES_SECRET_MAPPING") | sh
+      - name: Push tags
+        run: |
+          (jq -r 'to_entries | map("cat ./docker-image/image.tags | xargs -I % docker push " + .key + "/%") | .[]' <<<"$DOCKER_IMAGE_REGISTRIES_SECRET_MAPPING") | sh
+  push-manifest:
+    name: Push ${{ matrix.image }} manifest
+    if: (github.event_name == 'push' || github.event_name == 'schedule') && github.ref == 'refs/heads/master'
+    strategy:
+      fail-fast: false
+      matrix:
+        image: ${{ fromJson(needs.image-matrix.outputs.image) }}
+    needs:
+      - push
+      - image-matrix
+      - supported-arch-matrix
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: dbhi/qus/action@main
+      - name: Download Images
+        uses: actions/download-artifact@v2
+        with:
+          name: docker-image-${{ matrix.image }}-amd64
+          path: ./docker-image
+      - name: Login to container registries
+        run: |
+          (jq -r 'to_entries | map("echo \"$" + .value + "\" | docker login " + .key + " --username \"${{ env.DOCKER_USER }}\" --password-stdin") | .[]' <<<"$DOCKER_IMAGE_REGISTRIES_SECRET_MAPPING") | sh
+        env:
+          DOCKER_USER: ${{ secrets.HUB_USERNAME }}
+          GHCR_TOKEN: ${{ secrets.GHCR_TOKEN }}
+          HUB_PASSCODE: ${{ secrets.HUB_PASSCODE }}
+      - name: Docker info
+        run: docker info
+      - name: Push manifest to registries
+        run: |
+          touch command.sh
+          (jq -r 'to_entries | map("cat ./docker-image/image.tags | xargs -I % php utils/create-manifest-command.php " + .key + " %") | .[]' <<<"$DOCKER_IMAGE_REGISTRIES_SECRET_MAPPING") | sh
+          chmod +x command.sh
+          ./command.sh
+        env:
+          TARGET_ARCHS: ${{ needs.supported-arch-matrix.outputs.arch }}
diff --git a/Makefile b/Makefile
@@ -1,7 +1,6 @@
 qa: lint build test scan-vulnerability
 build: clean-tags build-all
 push: build push
-ci-push: ci-docker-login push-from-tags
 
 mkfile_path := $(abspath $(lastword $(MAKEFILE_LIST)))
 current_dir := $(abspath $(patsubst %/,%,$(dir $(mkfile_path))))
@@ -14,14 +13,6 @@ BUILDINGIMAGE=*
 clean-tags:
 	rm ${current_dir}/docker-image/build.tags || true
 
-# Docker images push
-push-from-tags:
-	cat ./docker-image/image.tags | xargs -I % docker push $$DOCKER_REGISTRY/%
-
-# CI dependencies
-ci-docker-login:
-	docker login $$DOCKER_REGISTRY --username $$DOCKER_USER --password $$DOCKER_PASSWORD
-
 lint:
 	docker run -v ${current_dir}:/project:ro --workdir=/project --rm -it hadolint/hadolint:latest-debian hadolint /project/Dockerfile-*
 
diff --git a/build-php.sh b/build-php.sh
@@ -18,6 +18,8 @@ declare -r VERSION_OS_TAG=$7
 
 declare -r VERSION_OS_FROM=$8
 
+declare -r TARGET_ARCH=$9
+
 # I could create a placeholder like php:x.y-image-alpinex.y in the Dockerfile itself,
 # but I think it wouldn't be a good experience if you try to build the image yourself
 # thus that's the way I opted to have dynamic base images
@@ -50,6 +52,6 @@ docker pull "php:${IMAGE_TAG}"
 
 for buildTarget in "${target[@]}"
 do
-  sed -E "s/${IMAGE_ORIGINAL_TAG}/${IMAGE_TAG}/g" "Dockerfile-${DST_IMAGE}-${OS}" | docker build --label org.label-schema.build-date=`date -u +"%Y-%m-%dT%H:%M:%SZ"` --label org.label-schema.vcs-ref=`git rev-parse --short HEAD` -t "${WYRIHAXIMUSNET_TAG}${buildTarget}" --target="${DST_IMAGE}${buildTarget}" -f - .
-  echo "${WYRIHAXIMUSNET_TAG}${buildTarget}" >> "$TAG_FILE"
+  sed -E "s/${IMAGE_ORIGINAL_TAG}/${IMAGE_TAG}/g" "Dockerfile-${DST_IMAGE}-${OS}" | docker build --platform ${TARGET_ARCH} --label org.label-schema.build-date=`date -u +"%Y-%m-%dT%H:%M:%SZ"` --label org.label-schema.vcs-ref=`git rev-parse --short HEAD` -t "${WYRIHAXIMUSNET_TAG}${buildTarget}-${TARGET_ARCH}" --target="${DST_IMAGE}${buildTarget}" -f - .
+  echo "${WYRIHAXIMUSNET_TAG}${buildTarget}-${TARGET_ARCH}" >> "$TAG_FILE"
 done
diff --git a/buildx-php.sh b/buildx-php.sh
diff --git a/test-nts.sh b/test-nts.sh
@@ -13,16 +13,18 @@ declare -r DOCKER_TAG="$1"
 
 declare TEST_SUITE
 
+TEST_SUITE="php_$IMAGE_ARCH"
+
 if [[ $DOCKER_TAG == *"-dev"* && $IMAGE_BASE_VERSION != *"alpha"* && $IMAGE_BASE_VERSION != *"beta"* && $IMAGE_BASE_VERSION != *"rc"* ]]; then
     TEST_SUITE="php_nts or php_dev"
 else
     TEST_SUITE="php_nts or php_no_dev and not php_dev"
 fi
 
 if [[ $DOCKER_TAG == *"-slim"* ]]; then
-    TEST_SUITE="php_slim or $TEST_SUITE"
+    TEST_SUITE="php_slim or php_slim_$IMAGE_ARCH or $TEST_SUITE"
 else
-    TEST_SUITE="php_not_slim or $TEST_SUITE"
+    TEST_SUITE="php_not_slim or php_not_slim_$IMAGE_ARCH or $TEST_SUITE"
 fi
 
 if [[ $DOCKER_TAG == *"-root"* ]]; then
diff --git a/test-zts.sh b/test-zts.sh
@@ -13,16 +13,18 @@ declare -r DOCKER_TAG="$1"
 
 declare TEST_SUITE
 
+TEST_SUITE="php_$IMAGE_ARCH"
+
 if [[ $DOCKER_TAG == *"-dev"* && $IMAGE_BASE_VERSION != *"alpha"* && $IMAGE_BASE_VERSION != *"beta"* && $IMAGE_BASE_VERSION != *"rc"* ]]; then
     TEST_SUITE="php_zts or php_dev"
 else
     TEST_SUITE="php_zts or php_no_dev and not php_dev"
 fi
 
 if [[ $DOCKER_TAG == *"-slim"* ]]; then
-    TEST_SUITE="php_slim or $TEST_SUITE"
+    TEST_SUITE="php_slim or php_slim_$IMAGE_ARCH or $TEST_SUITE"
 else
-    TEST_SUITE="php_not_slim or $TEST_SUITE"
+    TEST_SUITE="php_not_slim or php_not_slim_$IMAGE_ARCH or $TEST_SUITE"
 fi
 
 if [[ $DOCKER_TAG == *"-root"* ]]; then
diff --git a/test/container/test_php.py b/test/container/test_php.py
@@ -64,7 +64,7 @@ def test_php_ext_uv_is_functional(host):
     assert output.stdout == '0123finished'
     assert output.rc == 0
 
-@pytest.mark.php_not_slim
+@pytest.mark.php_not_slim_amd64
 def test_php_ext_vips_is_enabled(host):
     output = host.run('php -r "exit(function_exists(\'vips_version\') ? 0 : 255);"')
     assert output.rc == 0
diff --git a/test/container/test_php_ext.py b/test/container/test_php_ext.py
@@ -44,7 +44,7 @@ def test_pgsql_is_loaded(host):
 def test_uv_is_loaded(host):
     assert 'uv' in host.run('php -m').stdout
 
-@pytest.mark.php_not_slim
+@pytest.mark.php_not_slim_amd64
 def test_vips_is_loaded(host):
     assert 'vips' in host.run('php -m').stdout
 
diff --git a/utils/create-manifest-command.php b/utils/create-manifest-command.php
