WIP: Add aarch64 docker and musl outputs

.github/workflows/release-binaries.yml

@@ -12,11 +12,11 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Install Nix with good defaults
-        uses: input-output-hk/install-nix-action@v31
+        uses: input-output-hk/install-nix-action@v31.10.6
         with:
           extra_nix_config: |
-            trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= hydra.iohk.io:f/Ea+s+dFdN+3Y/G+FDgSq+a5NEWhJGzdjvKNGv0/EQ=
-            substituters = https://cache.iog.io/ https://cache.nixos.org/
+            extra-trusted-public-keys = hydra.iohk.io:f/Ea+s+dFdN+3Y/G+FDgSq+a5NEWhJGzdjvKNGv0/EQ=
+            extra-substituters = https://cache.nixos.org/
             experimental-features = nix-command flakes fetch-closure
             allow-import-from-derivation = true
             accept-flake-config = true
@@ -25,23 +25,53 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@v3
 
-      - name: Download archive from cache
+      - name: Download archives from cache
         run: |
           nix build \
             --builders "" \
             --max-jobs 0 \
-            --out-link result-linux \
-            .#cardano-db-sync-linux
+            --out-link result-amd64 \
+            .#packages.x86_64-linux.cardano-db-sync-linux
+          nix build \
+            --builders "" \
+            --max-jobs 0 \
+            --out-link result-arm64 \
+            .#packages.aarch64-linux.cardano-db-sync-linux
           nix build \
             --builders "" \
             --max-jobs 0 \
             --out-link result-macos \
-            .#packages.x86_64-darwin.cardano-db-sync-macos
+            .#packages.aarch64-darwin.cardano-db-sync-macos
+
+      - name: Prepare release assets
+        run: |
+          mkdir -p assets
+
+          # The linux tarballs are named "cardano-db-sync-<ver>-<platform>.tar.gz". This
+          # should be the same on any architecture, so we just choose one of them.
+          linux_base="$(basename result-amd64/cardano-db-sync-*-linux.tar.gz .tar.gz)"
+
+          # Copy the renamed linux archives
+          cp "result-amd64/${linux_base}.tar.gz" "assets/${linux_base}-amd64.tar.gz"
+          cp "result-arm64/${linux_base}.tar.gz" "assets/${linux_base}-arm64.tar.gz"
+
+          # Do the same for macOS
+          macos_base="$(basename result-macos/cardano-db-sync-*-macos.tar.gz .tar.gz)"
+          cp "result-macos/${macos_base}.tar.gz" "assets/${macos_base}-arm64.tar.gz"
+
+      - name: Upload build artifacts
+        if: github.event_name == 'workflow_dispatch'
+        uses: actions/upload-artifact@v4
+        with:
+          name: binary-dist
+          path: assets/*.tar.gz
 
       - name: Release
+        if: github.event_name == 'release'
         uses: input-output-hk/action-gh-release@v1
         with:
           draft: true
           files: |
-            result-linux/cardano-db-sync-*-linux.tar.gz
-            result-macos/cardano-db-sync-*-macos.tar.gz
+            assets/cardano-db-sync-*-linux-amd64.tar.gz
+            assets/cardano-db-sync-*-linux-arm64.tar.gz
+            assets/cardano-db-sync-*-macos-arm64.tar.gz

.github/workflows/release-ghcr.yml

@@ -9,20 +9,36 @@ on:
       - '**'
   workflow_dispatch: {}
 
+permissions:
+  contents: read
+  packages: write
+
 env:
   REGISTRY: ghcr.io
 
 jobs:
   build:
-    name: "Upload to ghcr.io"
+    name: "Build ${{ matrix.image.name }} (${{ matrix.arch.name }})"
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        arch:
+          - name: amd64
+            system: x86_64-linux
+          - name: arm64
+            system: aarch64-linux
+        image:
+          - name: cardano-db-sync
+            nix_key: cardano-db-sync-docker
+          - name: cardano-smash-server
+            nix_key: cardano-smash-server-docker
     steps:
       - name: Install Nix with good defaults
-        uses: input-output-hk/install-nix-action@v31
+        uses: input-output-hk/install-nix-action@v31.10.6
         with:
           extra_nix_config: |
-            trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= hydra.iohk.io:f/Ea+s+dFdN+3Y/G+FDgSq+a5NEWhJGzdjvKNGv0/EQ=
-            substituters = https://cache.iog.io/ https://cache.nixos.org/
+            extra-trusted-public-keys = hydra.iohk.io:f/Ea+s+dFdN+3Y/G+FDgSq+a5NEWhJGzdjvKNGv0/EQ=
+            extra-substituters = https://cache.nixos.org/
             experimental-features = nix-command flakes fetch-closure
             allow-import-from-derivation = true
             accept-flake-config = true
@@ -48,36 +64,93 @@ jobs:
           echo "REPO_OWNER=${REPO_OWNER}" >> "$GITHUB_ENV"
           echo "IMAGE_TAG=${IMAGE_TAG}" >> "$GITHUB_ENV"
 
-      - name: Upload ${{ github.actor }}/cardano-db-sync
+      - name: Upload ${{ matrix.image.name }} (${{ matrix.arch.name }})
+        env:
+          IMAGE_NAME: ${{ matrix.image.name }}
+          NIX_KEY: ${{ matrix.image.nix_key }}
+          ARCH_NAME: ${{ matrix.arch.name }}
+          ARCH_SYSTEM: ${{ matrix.arch.system }}
         run: |
-          # Download the image from the nix binary cachhe
-          nix build --builders "" --max-jobs 0 .#cardano-db-sync-docker
+          # Download the image from the nix binary cache
+          nix build \
+            --builders "" \
+            --max-jobs 0 \
+            --out-link "result-${IMAGE_NAME}-${ARCH_NAME}" \
+            ".#packages.${ARCH_SYSTEM}.${NIX_KEY}"
 
-          # Push the image
+          # Push the arch-suffixed image
           skopeo copy \
-            docker-archive:./result \
-            docker://ghcr.io/${REPO_OWNER}/cardano-db-sync:$IMAGE_TAG
+            "docker-archive:./result-${IMAGE_NAME}-${ARCH_NAME}" \
+            "docker://ghcr.io/${REPO_OWNER}/${IMAGE_NAME}:${IMAGE_TAG}-${ARCH_NAME}"
+
+  create-manifest:
+    name: "Create multi-arch manifest"
+    needs: [build]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Install Nix with good defaults
+        uses: input-output-hk/install-nix-action@v31.10.6
+        with:
+          extra_nix_config: |
+            extra-trusted-public-keys = hydra.iohk.io:f/Ea+s+dFdN+3Y/G+FDgSq+a5NEWhJGzdjvKNGv0/EQ=
+            extra-substituters = https://cache.nixos.org/
+            experimental-features = nix-command flakes fetch-closure
+            allow-import-from-derivation = true
+            accept-flake-config = true
+          nix_path: nixpkgs=channel:nixos-unstable
+
+      - name: Install regctl # simplifies obtaining multi-arch digests
+        run: nix profile install nixpkgs#regctl
+
+      - name: Setup Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Log in to ghcr.io
+        uses: docker/login-action@v2.1.0
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Set environment variables
+        run: |
+          REPO_OWNER="$(echo ${{ github.repository_owner }} | tr '[:upper:]' '[:lower:]')"
+          IMAGE_TAG="$(echo $GITHUB_REF_NAME | tr '/' '_')"
 
-          # If it's a tag build, also tag it as latest
-          if [[ "$GITHUB_REF_TYPE" = "tag" ]]; then
-            skopeo copy \
-              docker-archive:./result \
-              docker://ghcr.io/${REPO_OWNER}/cardano-db-sync:latest
-          fi
+          echo "REPO_OWNER=${REPO_OWNER}" >> "$GITHUB_ENV"
+          echo "IMAGE_TAG=${IMAGE_TAG}" >> "$GITHUB_ENV"
 
-      - name: Upload ${{ github.actor }}/cardano-smash-server
+      - name: Create and verify manifests
         run: |
-          # Download the image from the nix binary cachhe
-          nix build --builders "" --max-jobs 0 .#cardano-smash-server-docker
+          IMAGES=(cardano-db-sync cardano-smash-server)
+          ARCHES=(amd64 arm64)
 
-          # Push the image
-          skopeo copy \
-            docker-archive:./result \
-            docker://ghcr.io/${REPO_OWNER}/cardano-smash-server:$IMAGE_TAG
-
-          # If it's a tag build, also tag it as latest
-          if [[ "$GITHUB_REF_TYPE" = "tag" ]]; then
-            skopeo copy \
-              docker-archive:./result \
-              docker://ghcr.io/${REPO_OWNER}/cardano-smash-server:latest
-          fi
+          for IMAGE in "${IMAGES[@]}"; do
+            IMAGE_REPO="ghcr.io/${REPO_OWNER}/${IMAGE}"
+            DIGESTS=()
+
+            echo "::group::Fetching digests for ${IMAGE}"
+            for ARCH in "${ARCHES[@]}"; do
+              DIGEST=$(skopeo inspect --no-tags "docker://${IMAGE_REPO}:${IMAGE_TAG}-${ARCH}" | jq -r .Digest)
+              echo "${IMAGE} ${ARCH} digest: ${DIGEST}"
+              DIGESTS+=("${IMAGE_REPO}@${DIGEST}")
+            done
+            echo "::endgroup::"
+
+            echo "::group::Creating manifest for ${IMAGE_REPO}:${IMAGE_TAG}"
+            docker buildx imagetools create --tag "${IMAGE_REPO}:${IMAGE_TAG}" "${DIGESTS[@]}"
+            echo "::endgroup::"
+
+            # If it's a tag build, also tag the multi-arch manifest as latest
+            if [[ "$GITHUB_REF_TYPE" = "tag" ]]; then
+              MANIFEST_DIGEST=$(regctl manifest head "${IMAGE_REPO}:${IMAGE_TAG}")
+              echo "::group::Tagging ${IMAGE_REPO}:latest"
+              docker buildx imagetools create --tag "${IMAGE_REPO}:latest" "${IMAGE_REPO}@${MANIFEST_DIGEST}"
+              echo "::endgroup::"
+            fi
+
+            echo "::group::Verifying ${IMAGE_REPO}:${IMAGE_TAG}"
+            regctl manifest head "${IMAGE_REPO}:${IMAGE_TAG}"
+            skopeo inspect --raw "docker://${IMAGE_REPO}:${IMAGE_TAG}" | jq
+            echo "::endgroup::"
+          done

flake.nix

@@ -532,7 +532,12 @@
                     echo "file binary-dist $out/$dist_file" > $out/nix-support/hydra-build-products
                   '';
 
-            cardano-db-sync-linux = mkDist "linux" project.projectCross.musl64;
+            cardano-db-sync-linux = 
+              if system == "aarch64-linux" then
+                mkDist "linux" project.projectCross.aarch64-multiplatform-musl
+              else
+                mkDist "linux" project.projectCross.musl64;
+
             cardano-db-sync-macos = mkDist "macos" project;
             cardano-smash-server-no-basic-auth = (project.appendModule {
               modules = [{packages.cardano-smash-server.flags.disable-basic-auth = true;}];
@@ -586,34 +591,36 @@
             hydraJobs = callPackages inputs.iohkNix.utils.ciJobsAggregates {
               ciJobs = flake.hydraJobs;
               nonRequiredPaths = map lib.hasPrefix nonRequiredPaths;
-            } // lib.optionalAttrs (system == "x86_64-linux") {
+            } // lib.optionalAttrs hostPlatform.isLinux {
               inherit
                 cardano-db-sync-linux
                 cardano-db-sync-docker
                 cardano-smash-server-docker;
-
+            } // lib.optionalAttrs (system == "x86_64-linux") {
               # No need to run static checks on all architectures
               checks = staticChecks;
-            } // lib.optionalAttrs (system == "x86_64-darwin") {
+            } // lib.optionalAttrs (system == "aarch64-darwin") {
               inherit cardano-db-sync-macos;
             } // {
               inherit cardano-smash-server-no-basic-auth profiled;
             };
 
             legacyPackages = pkgs;
 
-            packages = lib.optionalAttrs (system == "x86_64-linux") {
+            packages = lib.optionalAttrs hostPlatform.isLinux {
               inherit
                 cardano-db-sync-linux
                 cardano-db-sync-docker
-                cardano-smash-server-docker
+                cardano-smash-server-docker;
+            } // lib.optionalAttrs (system == "aarch64-darwin") {
+              inherit cardano-db-sync-macos;
+            } // {
+              inherit
+                cardano-smash-server-no-basic-auth
+                profiled
                 project;
 
               default = exes.cardano-db-sync;
-            } // lib.optionalAttrs (system == "x86_64-darwin") {
-              inherit cardano-db-sync-macos;
-            } // {
-              inherit cardano-smash-server-no-basic-auth profiled;
             # Run setGitRev on all packages that have ":exe:" in their key
             } // builtins.mapAttrs exeSetGitRev flake.packages;
           })) // {