release.yml

name: Release

on:
  workflow_dispatch:
    inputs:
      source_ref:
        description: "Branch or tag name that requested the build"
        required: true
        type: string
      source_sha:
        description: "Exact commit SHA to build"
        required: true
        type: string
      tag_name:
        description: "Release/package version name, for example v0.3.0"
        required: true
        type: string
      publish_release:
        description: "Create or update the GitHub release and release Docker tags"
        required: true
        default: false
        type: boolean

concurrency:
  group: ${{ github.workflow }}-${{ inputs.source_ref || github.ref }}
  cancel-in-progress: ${{ inputs.publish_release != true }}

env:
  CMAKE_ARGS: "-DLLAMA_BUILD_EXAMPLES=OFF -DLLAMA_BUILD_TESTS=OFF -DLLAMA_BUILD_TOOLS=ON -DLLAMA_BUILD_SERVER=ON -DGGML_RPC=ON"

jobs:
  release-meta:
    name: Release metadata
    runs-on: ubuntu-24.04
    outputs:
      source_ref: ${{ steps.meta.outputs.source_ref }}
      source_sha: ${{ steps.meta.outputs.source_sha }}
      tag_name: ${{ steps.meta.outputs.tag_name }}
      build_packages: ${{ steps.meta.outputs.build_packages }}
      create_release: ${{ steps.meta.outputs.create_release }}
      publish_docker: ${{ steps.meta.outputs.publish_docker }}
      image_repo: ${{ steps.meta.outputs.image_repo }}
      ref_name_safe: ${{ steps.meta.outputs.ref_name_safe }}
      short_sha: ${{ steps.meta.outputs.short_sha }}
    steps:
      - name: Clone
        uses: actions/checkout@v6
        with:
          fetch-depth: 0

      - name: Resolve release metadata
        id: meta
        shell: bash
        run: |
          set -euo pipefail

          source_ref="${INPUT_SOURCE_REF}"
          source_sha="${INPUT_SOURCE_SHA}"
          tag_name="${INPUT_TAG_NAME}"
          create_release="${INPUT_PUBLISH_RELEASE}"
          build_packages="true"
          publish_docker="true"
          short_sha="${source_sha::12}"
          ref_name_safe="$(printf '%s' "${source_ref}" | tr '[:upper:]' '[:lower:]' | sed -E 's#[^a-z0-9_.-]+#-#g; s#^[.-]+##; s#[.-]+$##')"
          if [[ -z "${ref_name_safe}" ]]; then
            ref_name_safe="${short_sha}"
          fi

          if [[ -z "${source_sha}" || ! "${source_sha}" =~ ^[0-9a-f]{40}$ ]]; then
            echo "source_sha must be a full 40-character commit SHA." >&2
            exit 1
          fi
          if ! git cat-file -e "${source_sha}^{commit}"; then
            git fetch origin +refs/heads/"${source_ref}":refs/remotes/origin/"${source_ref}" --no-tags || true
            git fetch origin +refs/tags/"${source_ref}":refs/tags/"${source_ref}" || true
          fi
          if ! git cat-file -e "${source_sha}^{commit}"; then
            git fetch origin "${source_sha}" --no-tags
          fi
          git cat-file -e "${source_sha}^{commit}"

          git fetch origin +refs/heads/main:refs/remotes/origin/main --no-tags

          if [[ "${create_release}" == "true" ]]; then
            if [[ "${tag_name}" != v* ]]; then
              echo "Release publishing requires a v* tag_name." >&2
              exit 1
            fi
            git fetch origin +refs/tags/"${tag_name}":refs/tags/"${tag_name}"
            tag_commit="$(git rev-list -n 1 "${tag_name}")"
            if [[ "${tag_commit}" != "${source_sha}" ]]; then
              echo "Tag ${tag_name} points to ${tag_commit}, not requested source ${source_sha}." >&2
              exit 1
            fi
            if ! git merge-base --is-ancestor "${source_sha}" refs/remotes/origin/main; then
              echo "Tag ${tag_name} is not reachable from origin/main; refusing to publish." >&2
              exit 1
            fi
          fi

          owner="$(printf '%s' "${GITHUB_REPOSITORY_OWNER}" | tr '[:upper:]' '[:lower:]')"
          repo="$(printf '%s' "${GITHUB_REPOSITORY#*/}" | tr '[:upper:]' '[:lower:]')"
          image_repo="ghcr.io/${owner}/${repo}"

          echo "source_ref=${source_ref}" >> "${GITHUB_OUTPUT}"
          echo "source_sha=${source_sha}" >> "${GITHUB_OUTPUT}"
          echo "tag_name=${tag_name}" >> "${GITHUB_OUTPUT}"
          echo "build_packages=${build_packages}" >> "${GITHUB_OUTPUT}"
          echo "create_release=${create_release}" >> "${GITHUB_OUTPUT}"
          echo "publish_docker=${publish_docker}" >> "${GITHUB_OUTPUT}"
          echo "image_repo=${image_repo}" >> "${GITHUB_OUTPUT}"
          echo "ref_name_safe=${ref_name_safe}" >> "${GITHUB_OUTPUT}"
          echo "short_sha=${short_sha}" >> "${GITHUB_OUTPUT}"
          echo "Using source ref: ${source_ref}"
          echo "Using source SHA: ${source_sha}"
          echo "Using package version: ${tag_name}"
          echo "Build packages: ${build_packages}"
          echo "Publish GitHub release: ${create_release}"
          echo "Publish Docker images: ${publish_docker}"
        env:
          INPUT_SOURCE_REF: ${{ inputs.source_ref }}
          INPUT_SOURCE_SHA: ${{ inputs.source_sha }}
          INPUT_TAG_NAME: ${{ inputs.tag_name }}
          INPUT_PUBLISH_RELEASE: ${{ inputs.publish_release }}

  macos-arm64:
    name: macOS arm64 Metal
    needs: release-meta
    if: ${{ needs.release-meta.outputs.build_packages == 'true' }}
    runs-on: macos-14
    steps:
      - name: Clone
        uses: actions/checkout@v6
        with:
          ref: ${{ needs.release-meta.outputs.source_sha }}
          fetch-depth: 0

      - name: Setup Node.js
        uses: actions/setup-node@v6
        with:
          node-version: "24"
          cache: "npm"
          cache-dependency-path: "tools/ui/package-lock.json"

      - name: ccache
        uses: ggml-org/ccache-action@v1.2.21
        with:
          key: release-macos-arm64
          evict-old-files: 1d

      - name: Build
        run: |
          cmake -B build \
            -DCMAKE_BUILD_TYPE=Release \
            -DGGML_METAL_USE_BF16=ON \
            -DGGML_METAL_EMBED_LIBRARY=ON \
            -DCMAKE_INSTALL_RPATH='@loader_path' \
            -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON \
            -DLLAMA_FATAL_WARNINGS=ON \
            -DLLAMA_BUILD_BORINGSSL=ON \
            ${{ env.CMAKE_ARGS }}
          cmake --build build --config Release -j "$(sysctl -n hw.logicalcpu)"

      - name: ccache stats
        if: always()
        run: ccache --show-stats

      - name: Smoke package binaries
        run: |
          ./build/bin/llama-server --version
          ./build/bin/llama-cli --version

      - name: Pack artifacts
        run: |
          cp LICENSE ./build/bin/
          tar -czvf "beellama-${{ needs.release-meta.outputs.tag_name }}-bin-macos-arm64.tar.gz" \
            -s ",^\.,beellama-${{ needs.release-meta.outputs.tag_name }}," \
            -C ./build/bin .

      - name: Upload artifacts
        uses: actions/upload-artifact@v6
        with:
          path: beellama-${{ needs.release-meta.outputs.tag_name }}-bin-macos-arm64.tar.gz
          name: beellama-bin-macos-arm64.tar.gz

  ubuntu-cpu:
    name: Ubuntu ${{ matrix.build }} CPU
    needs: release-meta
    if: ${{ needs.release-meta.outputs.build_packages == 'true' }}
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix:
        include:
          - build: x64
            os: ubuntu-22.04
          - build: arm64
            os: ubuntu-24.04-arm
    steps:
      - name: Clone
        uses: actions/checkout@v6
        with:
          ref: ${{ needs.release-meta.outputs.source_sha }}
          fetch-depth: 0

      - name: Setup Node.js
        uses: actions/setup-node@v6
        with:
          node-version: "24"
          cache: "npm"
          cache-dependency-path: "tools/ui/package-lock.json"

      - name: ccache
        uses: ggml-org/ccache-action@v1.2.21
        with:
          key: release-${{ matrix.os }}-cpu
          evict-old-files: 1d

      - name: Dependencies
        run: |
          sudo apt-get update
          sudo apt-get install -y build-essential libssl-dev

      - name: Toolchain workaround
        if: ${{ contains(matrix.os, 'ubuntu-24.04') }}
        run: |
          sudo apt-get install -y gcc-14 g++-14
          echo "CC=gcc-14" >> "${GITHUB_ENV}"
          echo "CXX=g++-14" >> "${GITHUB_ENV}"

      - name: Build
        run: |
          cmake -B build \
            -DCMAKE_BUILD_TYPE=Release \
            -DCMAKE_INSTALL_RPATH='$ORIGIN' \
            -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON \
            -DGGML_BACKEND_DL=ON \
            -DGGML_NATIVE=OFF \
            -DGGML_CPU_ALL_VARIANTS=ON \
            -DLLAMA_FATAL_WARNINGS=ON \
            ${{ env.CMAKE_ARGS }}
          cmake --build build --config Release -j "$(nproc)"

      - name: ccache stats
        if: always()
        run: ccache --show-stats

      - name: Smoke package binaries
        run: |
          file ./build/bin/llama-server
          ./build/bin/llama-server --version
          ./build/bin/llama-cli --version

      - name: Pack artifacts
        run: |
          cp LICENSE ./build/bin/
          tar -czvf "beellama-${{ needs.release-meta.outputs.tag_name }}-bin-ubuntu-${{ matrix.build }}.tar.gz" \
            --transform "s,^\.,beellama-${{ needs.release-meta.outputs.tag_name }}," \
            -C ./build/bin .

      - name: Upload artifacts
        uses: actions/upload-artifact@v6
        with:
          path: beellama-${{ needs.release-meta.outputs.tag_name }}-bin-ubuntu-${{ matrix.build }}.tar.gz
          name: beellama-bin-ubuntu-${{ matrix.build }}.tar.gz

  ubuntu-vulkan:
    name: Ubuntu x64 Vulkan
    needs: release-meta
    if: ${{ needs.release-meta.outputs.build_packages == 'true' }}
    runs-on: ubuntu-22.04
    steps:
      - name: Clone
        uses: actions/checkout@v6
        with:
          ref: ${{ needs.release-meta.outputs.source_sha }}
          fetch-depth: 0

      - name: Setup Node.js
        uses: actions/setup-node@v6
        with:
          node-version: "24"
          cache: "npm"
          cache-dependency-path: "tools/ui/package-lock.json"

      - name: ccache
        uses: ggml-org/ccache-action@v1.2.21
        with:
          key: release-ubuntu-vulkan-x64
          evict-old-files: 1d

      - name: Dependencies
        run: |
          wget -qO - https://packages.lunarg.com/lunarg-signing-key-pub.asc | sudo apt-key add -
          sudo wget -qO /etc/apt/sources.list.d/lunarg-vulkan-jammy.list https://packages.lunarg.com/vulkan/lunarg-vulkan-jammy.list
          sudo apt-get update -y
          sudo apt-get install -y build-essential mesa-vulkan-drivers vulkan-sdk libssl-dev

      - name: Build
        run: |
          cmake -B build \
            -DCMAKE_BUILD_TYPE=Release \
            -DCMAKE_INSTALL_RPATH='$ORIGIN' \
            -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON \
            -DGGML_BACKEND_DL=ON \
            -DGGML_NATIVE=OFF \
            -DGGML_CPU_ALL_VARIANTS=ON \
            -DGGML_VULKAN=ON \
            -DLLAMA_FATAL_WARNINGS=ON \
            ${{ env.CMAKE_ARGS }}
          cmake --build build --config Release -j "$(nproc)"

      - name: ccache stats
        if: always()
        run: ccache --show-stats

      - name: Smoke package binaries
        run: |
          ./build/bin/llama-server --version
          ./build/bin/llama-cli --version

      - name: Pack artifacts
        run: |
          cp LICENSE ./build/bin/
          tar -czvf "beellama-${{ needs.release-meta.outputs.tag_name }}-bin-ubuntu-vulkan-x64.tar.gz" \
            --transform "s,^\.,beellama-${{ needs.release-meta.outputs.tag_name }}," \
            -C ./build/bin .

      - name: Upload artifacts
        uses: actions/upload-artifact@v6
        with:
          path: beellama-${{ needs.release-meta.outputs.tag_name }}-bin-ubuntu-vulkan-x64.tar.gz
          name: beellama-bin-ubuntu-vulkan-x64.tar.gz

  ubuntu-rocm:
    name: Ubuntu x64 ROCm
    needs: release-meta
    if: ${{ needs.release-meta.outputs.build_packages == 'true' }}
    runs-on: ubuntu-22.04
    env:
      ROCM_VERSION: "7.2.1"
      GPU_TARGETS: "gfx908;gfx90a;gfx942;gfx1030;gfx1100;gfx1101;gfx1102;gfx1151;gfx1150;gfx1200;gfx1201"
    steps:
      - name: Clone
        uses: actions/checkout@v6
        with:
          ref: ${{ needs.release-meta.outputs.source_sha }}
          fetch-depth: 0

      - name: Setup Node.js
        uses: actions/setup-node@v6
        with:
          node-version: "24"
          cache: "npm"
          cache-dependency-path: "tools/ui/package-lock.json"

      - name: Free up disk space
        uses: ggml-org/free-disk-space@v1.3.1
        with:
          tool-cache: true

      - name: ccache
        uses: ggml-org/ccache-action@v1.2.21
        with:
          key: release-ubuntu-rocm-${{ env.ROCM_VERSION }}-x64
          evict-old-files: 1d

      - name: Dependencies
        run: |
          sudo apt-get update
          sudo apt-get install -y build-essential git cmake wget libssl-dev

      - name: Setup ROCm
        run: |
          sudo mkdir --parents --mode=0755 /etc/apt/keyrings
          wget https://repo.radeon.com/rocm/rocm.gpg.key -O - | \
            gpg --dearmor | sudo tee /etc/apt/keyrings/rocm.gpg > /dev/null
          sudo tee /etc/apt/sources.list.d/rocm.list << EOF
          deb [arch=amd64 signed-by=/etc/apt/keyrings/rocm.gpg] https://repo.radeon.com/rocm/apt/${ROCM_VERSION} jammy main
          EOF
          sudo tee /etc/apt/preferences.d/rocm-pin-600 << EOF
          Package: *
          Pin: release o=repo.radeon.com
          Pin-Priority: 600
          EOF
          sudo apt-get update
          sudo apt-get install -y rocm-hip-sdk

      - name: Build
        run: |
          cmake -B build -S . \
            -DCMAKE_HIP_COMPILER="$(hipconfig -l)/clang" \
            -DCMAKE_BUILD_TYPE=Release \
            -DGGML_BACKEND_DL=ON \
            -DGGML_NATIVE=OFF \
            -DCMAKE_INSTALL_RPATH='$ORIGIN' \
            -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON \
            -DGGML_CPU_ALL_VARIANTS=ON \
            -DGPU_TARGETS="${GPU_TARGETS}" \
            -DGGML_HIP=ON \
            -DGGML_CUDA_FA=ON \
            -DGGML_CUDA_FA_ALL_QUANTS=ON \
            -DHIP_PLATFORM=amd \
            -DGGML_HIP_ROCWMMA_FATTN=ON \
            ${{ env.CMAKE_ARGS }}
          cmake --build build --config Release -j "$(nproc)"

      - name: ccache stats
        if: always()
        run: ccache --show-stats

      - name: Smoke package binaries
        run: |
          ./build/bin/llama-server --version
          ./build/bin/llama-cli --version

      - name: Pack artifacts
        run: |
          rocm_short="$(echo "${ROCM_VERSION}" | cut -d '.' -f 1,2)"
          cp LICENSE ./build/bin/
          tar -czvf "beellama-${{ needs.release-meta.outputs.tag_name }}-bin-ubuntu-rocm-${rocm_short}-x64.tar.gz" \
            --transform "s,^\.,beellama-${{ needs.release-meta.outputs.tag_name }}," \
            -C ./build/bin .

      - name: Upload artifacts
        uses: actions/upload-artifact@v6
        with:
          path: beellama-${{ needs.release-meta.outputs.tag_name }}-bin-ubuntu-rocm-7.2-x64.tar.gz
          name: beellama-bin-ubuntu-rocm-7.2-x64.tar.gz

  ubuntu-sycl:
    name: Ubuntu x64 SYCL
    needs: release-meta
    if: ${{ needs.release-meta.outputs.build_packages == 'true' }}
    runs-on: ubuntu-24.04
    env:
      ONEAPI_ROOT: /opt/intel/oneapi
      ONEAPI_INSTALLER_VERSION: "2025.3.3"
      ONEAPI_INSTALLER_URL: https://registrationcenter-download.intel.com/akdlm/IRC_NAS/56f7923a-adb8-43f3-8b02-2b60fcac8cab/intel-deep-learning-essentials-2025.3.3.16_offline.sh
      LEVEL_ZERO_VERSION: "1.28.2"
      LEVEL_ZERO_UBUNTU_VERSION: "u24.04"
    steps:
      - name: Clone
        uses: actions/checkout@v6
        with:
          ref: ${{ needs.release-meta.outputs.source_sha }}
          fetch-depth: 0

      - name: Setup Node.js
        uses: actions/setup-node@v6
        with:
          node-version: "24"
          cache: "npm"
          cache-dependency-path: "tools/ui/package-lock.json"

      - name: Free up disk space
        uses: ggml-org/free-disk-space@v1.3.1
        with:
          tool-cache: true

      - name: Prepare oneAPI cache path
        run: |
          sudo mkdir -p "${ONEAPI_ROOT}"
          sudo chown -R "${USER}:${USER}" /opt/intel

      - name: Cache oneAPI installation
        id: cache-oneapi
        uses: actions/cache@v5
        with:
          path: ${{ env.ONEAPI_ROOT }}
          key: oneapi-linux-${{ env.ONEAPI_INSTALLER_VERSION }}-${{ runner.os }}

      - name: ccache
        uses: ggml-org/ccache-action@v1.2.21
        with:
          key: release-ubuntu-sycl-${{ env.ONEAPI_INSTALLER_VERSION }}-x64
          evict-old-files: 1d

      - name: Dependencies
        run: |
          sudo apt-get update
          sudo apt-get install -y build-essential git cmake ninja-build wget ca-certificates libssl-dev

      - name: Install oneAPI
        if: steps.cache-oneapi.outputs.cache-hit != 'true'
        run: |
          installer="${RUNNER_TEMP}/intel-deep-learning-essentials-${ONEAPI_INSTALLER_VERSION}.sh"
          wget "${ONEAPI_INSTALLER_URL}" -O "${installer}"
          sudo bash "${installer}" -s -a --silent --eula accept
          sudo chown -R "${USER}:${USER}" "${ONEAPI_ROOT}"

      - name: Install Level Zero SDK
        run: |
          cd /tmp
          wget -q "https://github.com/oneapi-src/level-zero/releases/download/v${LEVEL_ZERO_VERSION}/level-zero_${LEVEL_ZERO_VERSION}%2B${LEVEL_ZERO_UBUNTU_VERSION}_amd64.deb" -O level-zero.deb
          wget -q "https://github.com/oneapi-src/level-zero/releases/download/v${LEVEL_ZERO_VERSION}/level-zero-devel_${LEVEL_ZERO_VERSION}%2B${LEVEL_ZERO_UBUNTU_VERSION}_amd64.deb" -O level-zero-devel.deb
          sudo apt-get -o Dpkg::Options::="--force-overwrite" install -y ./level-zero.deb ./level-zero-devel.deb

      - name: Build
        run: |
          source "${ONEAPI_ROOT}/setvars.sh"
          cmake -B build -G "Ninja" \
            -DCMAKE_BUILD_TYPE=Release \
            -DCMAKE_INSTALL_RPATH='$ORIGIN' \
            -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON \
            -DGGML_SYCL=ON \
            -DGGML_SYCL_F16=OFF \
            -DCMAKE_C_COMPILER=icx \
            -DCMAKE_CXX_COMPILER=icpx \
            -DGGML_BACKEND_DL=ON \
            -DGGML_NATIVE=OFF \
            -DGGML_CPU_ALL_VARIANTS=ON \
            ${{ env.CMAKE_ARGS }}
          cmake --build build --config Release -j "$(nproc)"

      - name: ccache stats
        if: always()
        run: ccache --show-stats

      - name: Smoke package binaries
        run: |
          source "${ONEAPI_ROOT}/setvars.sh"
          test -s ./build/bin/libggml-sycl.so
          ldd ./build/bin/libggml-sycl.so
          mv ./build/bin/libggml-sycl.so ./build/bin/libggml-sycl.so.smoke-disabled
          trap 'mv ./build/bin/libggml-sycl.so.smoke-disabled ./build/bin/libggml-sycl.so' EXIT
          ./build/bin/llama-server --version
          ./build/bin/llama-cli --version

      - name: Pack artifacts
        run: |
          cp LICENSE ./build/bin/
          tar -czvf "beellama-${{ needs.release-meta.outputs.tag_name }}-bin-ubuntu-sycl-x64.tar.gz" \
            --transform "s,^\.,beellama-${{ needs.release-meta.outputs.tag_name }}," \
            -C ./build/bin .

      - name: Upload artifacts
        uses: actions/upload-artifact@v6
        with:
          path: beellama-${{ needs.release-meta.outputs.tag_name }}-bin-ubuntu-sycl-x64.tar.gz
          name: beellama-bin-ubuntu-sycl-x64.tar.gz

  ubuntu-cuda:
    name: Ubuntu x64 CUDA ${{ matrix.cuda }}
    needs: release-meta
    if: ${{ needs.release-meta.outputs.build_packages == 'true' }}
    runs-on: ubuntu-24.04
    container: nvidia/cuda:${{ matrix.cuda_container }}
    env:
      CCACHE_MAXSIZE: 5G
      CCACHE_SLOPPINESS: time_macros
    strategy:
      fail-fast: false
      matrix:
        include:
          - cuda: "12.4"
            cuda_container: "12.4.1-devel-ubuntu22.04"
          - cuda: "13.1"
            cuda_container: "13.1.1-devel-ubuntu24.04"
    steps:
      - name: Dependencies
        run: |
          apt-get update
          apt-get install -y --no-install-recommends build-essential ca-certificates cmake ninja-build ccache git file libssl-dev libgomp1

      - name: Clone
        uses: actions/checkout@v6
        with:
          ref: ${{ needs.release-meta.outputs.source_sha }}
          fetch-depth: 0

      - name: Setup Node.js
        uses: actions/setup-node@v6
        with:
          node-version: "24"
          cache: "npm"
          cache-dependency-path: "tools/ui/package-lock.json"

      - name: ccache
        uses: ggml-org/ccache-action@v1.2.21
        with:
          key: release-ubuntu-cuda-${{ matrix.cuda }}-x64
          evict-old-files: 1d

      - name: Build
        run: |
          cmake -S . -B build -G Ninja \
            -DCMAKE_BUILD_TYPE=Release \
            -DCMAKE_INSTALL_RPATH='$ORIGIN' \
            -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON \
            -DGGML_NATIVE=OFF \
            -DGGML_CUDA=ON \
            -DGGML_CUDA_FA=ON \
            -DGGML_CUDA_FA_ALL_QUANTS=ON \
            -DGGML_BACKEND_DL=ON \
            -DGGML_CPU_ALL_VARIANTS=ON \
            -DCMAKE_C_COMPILER_LAUNCHER=ccache \
            -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
            -DCMAKE_CUDA_COMPILER_LAUNCHER=ccache \
            ${{ env.CMAKE_ARGS }} \
            -DCMAKE_EXE_LINKER_FLAGS=-Wl,--allow-shlib-undefined
          cmake --build build --config Release -j "$(nproc)"

      - name: ccache stats
        if: always()
        run: ccache --show-stats

      - name: Smoke package binaries
        run: |
          file ./build/bin/llama-server
          ./build/bin/llama-server --version
          ./build/bin/llama-cli --version

      - name: Pack artifacts
        run: |
          cp LICENSE ./build/bin/
          tar -czvf "beellama-${{ needs.release-meta.outputs.tag_name }}-bin-ubuntu-cuda-${{ matrix.cuda }}-x64.tar.gz" \
            --transform "s,^\.,beellama-${{ needs.release-meta.outputs.tag_name }}," \
            -C ./build/bin .

      - name: Upload artifacts
        uses: actions/upload-artifact@v6
        with:
          path: beellama-${{ needs.release-meta.outputs.tag_name }}-bin-ubuntu-cuda-${{ matrix.cuda }}-x64.tar.gz
          name: beellama-bin-ubuntu-cuda-${{ matrix.cuda }}-x64.tar.gz

  windows-cpu:
    name: Windows x64 CPU
    needs: release-meta
    if: ${{ needs.release-meta.outputs.build_packages == 'true' }}
    runs-on: windows-2025
    steps:
      - name: Clone
        uses: actions/checkout@v6
        with:
          ref: ${{ needs.release-meta.outputs.source_sha }}
          fetch-depth: 0

      - name: Setup Node.js
        uses: actions/setup-node@v6
        with:
          node-version: "24"
          cache: "npm"
          cache-dependency-path: "tools/ui/package-lock.json"

      - name: ccache
        uses: ggml-org/ccache-action@v1.2.21
        with:
          key: release-windows-cpu-x64
          variant: ccache
          evict-old-files: 1d

      - name: Install Ninja
        run: choco install ninja -y

      - name: Build
        shell: cmd
        run: |
          call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64
          cmake -S . -B build -G "Ninja Multi-Config" ^
            -D CMAKE_TOOLCHAIN_FILE=cmake/x64-windows-llvm.cmake ^
            -DLLAMA_BUILD_BORINGSSL=ON ^
            -DGGML_NATIVE=OFF ^
            -DGGML_BACKEND_DL=ON ^
            -DGGML_CPU_ALL_VARIANTS=ON ^
            -DGGML_OPENMP=ON ^
            %CMAKE_ARGS%
          cmake --build build --config Release

      - name: ccache stats
        if: always()
        run: ccache --show-stats

      - name: Pack artifacts
        shell: pwsh
        run: |
          $ErrorActionPreference = "Stop"
          $bin = ".\build\bin\Release"
          Copy-Item ".\LICENSE" $bin
          $redistRoot = "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Redist\MSVC"
          $omp = Get-ChildItem $redistRoot -Recurse -Filter "libomp140.x86_64.dll" |
            Where-Object { $_.FullName -like "*\debug_nonredist\x64\Microsoft.VC143.OpenMP.LLVM\libomp140.x86_64.dll" } |
            Sort-Object FullName -Descending |
            Select-Object -First 1
          if (-not $omp) {
            throw "VC143 x64 libomp140.x86_64.dll not found under $redistRoot"
          }
          Copy-Item $omp.FullName $bin -Force
          python scripts\verify-windows-package.py $bin
          & "$bin\llama-server.exe" --version
          & "$bin\llama-cli.exe" --version
          7z a -snl "beellama-bin-win-cpu-x64.zip" "$bin\*"

      - name: Upload artifacts
        uses: actions/upload-artifact@v6
        with:
          path: beellama-bin-win-cpu-x64.zip
          name: beellama-bin-win-cpu-x64.zip

  windows-sycl:
    name: Windows x64 SYCL
    needs: release-meta
    if: ${{ needs.release-meta.outputs.build_packages == 'true' }}
    runs-on: windows-2022
    env:
      WINDOWS_BASEKIT_URL: https://registrationcenter-download.intel.com/akdlm/IRC_NAS/b60765d1-2b85-4e85-86b6-cb0e9563a699/intel-deep-learning-essentials-2025.3.3.18_offline.exe
      WINDOWS_DPCPP_MKL: intel.oneapi.win.cpp-dpcpp-common:intel.oneapi.win.mkl.devel:intel.oneapi.win.dnnl:intel.oneapi.win.tbb.devel
      LEVEL_ZERO_SDK_URL: https://github.com/oneapi-src/level-zero/releases/download/v1.28.2/level-zero-win-sdk-1.28.2.zip
      ONEAPI_ROOT: 'C:\Program Files (x86)\Intel\oneAPI'
      ONEAPI_INSTALLER_VERSION: "2025.3.3"
    steps:
      - name: Clone
        uses: actions/checkout@v6
        with:
          ref: ${{ needs.release-meta.outputs.source_sha }}
          fetch-depth: 0

      - name: Cache oneAPI installation
        id: cache-oneapi
        uses: actions/cache@v5
        with:
          path: ${{ env.ONEAPI_ROOT }}
          key: oneapi-windows-${{ env.ONEAPI_INSTALLER_VERSION }}-${{ runner.os }}

      - name: Install oneAPI
        if: steps.cache-oneapi.outputs.cache-hit != 'true'
        shell: cmd
        run: scripts\install-oneapi.bat "%WINDOWS_BASEKIT_URL%" "%WINDOWS_DPCPP_MKL%"

      - name: Install Level Zero SDK
        shell: pwsh
        run: |
          $ErrorActionPreference = "Stop"
          $archive = Join-Path $env:RUNNER_TEMP "level-zero-win-sdk.zip"
          $extractDir = Join-Path $env:RUNNER_TEMP "level-zero-sdk"
          Invoke-WebRequest -Uri "${{ env.LEVEL_ZERO_SDK_URL }}" -OutFile $archive
          Expand-Archive -Path $archive -DestinationPath $extractDir -Force
          $sdkRoot = Get-ChildItem -Path $extractDir -Directory | Select-Object -First 1
          if (-not $sdkRoot) {
            $sdkRoot = Get-Item $extractDir
          }
          "LEVEL_ZERO_V1_SDK_PATH=$($sdkRoot.FullName)" | Out-File -FilePath $env:GITHUB_ENV -Append

      - name: ccache
        uses: ggml-org/ccache-action@v1.2.21
        with:
          key: release-windows-sycl-${{ env.ONEAPI_INSTALLER_VERSION }}-x64
          variant: ccache
          evict-old-files: 1d

      - name: Install Ninja
        run: choco install ninja -y

      - name: Build
        shell: cmd
        run: |
          call "%ONEAPI_ROOT%\setvars.bat" intel64 --force
          icx --version
          cmake -G "Ninja" -B build ^
            -DCMAKE_C_COMPILER=cl ^
            -DCMAKE_CXX_COMPILER=icx ^
            -DCMAKE_BUILD_TYPE=Release ^
            -DGGML_BACKEND_DL=ON ^
            -DBUILD_SHARED_LIBS=ON ^
            -DGGML_CPU=OFF ^
            -DGGML_SYCL=ON ^
            -DGGML_SYCL_F16=OFF ^
            -DLLAMA_BUILD_EXAMPLES=OFF ^
            -DLLAMA_BUILD_TESTS=OFF ^
            -DLLAMA_BUILD_TOOLS=OFF ^
            -DLLAMA_BUILD_SERVER=OFF ^
            -DLLAMA_BUILD_BORINGSSL=ON
          cmake --build build --config Release -j %NUMBER_OF_PROCESSORS% --target ggml-sycl

      - name: ccache stats
        if: always()
        run: ccache --show-stats

      - name: Pack SYCL backend
        shell: pwsh
        run: |
          $ErrorActionPreference = "Stop"
          $bin = ".\build\bin"
          Copy-Item ".\LICENSE" $bin -Force

          $runtimeFiles = @(
            "$env:ONEAPI_ROOT\mkl\latest\bin\mkl_sycl_blas.5.dll",
            "$env:ONEAPI_ROOT\mkl\latest\bin\mkl_core.2.dll",
            "$env:ONEAPI_ROOT\mkl\latest\bin\mkl_tbb_thread.2.dll",
            "$env:ONEAPI_ROOT\compiler\latest\bin\ur_adapter_level_zero.dll",
            "$env:ONEAPI_ROOT\compiler\latest\bin\ur_adapter_level_zero_v2.dll",
            "$env:ONEAPI_ROOT\compiler\latest\bin\ur_adapter_opencl.dll",
            "$env:ONEAPI_ROOT\compiler\latest\bin\ur_loader.dll",
            "$env:ONEAPI_ROOT\compiler\latest\bin\ur_win_proxy_loader.dll",
            "$env:ONEAPI_ROOT\compiler\latest\bin\sycl8.dll",
            "$env:ONEAPI_ROOT\compiler\latest\bin\svml_dispmd.dll",
            "$env:ONEAPI_ROOT\compiler\latest\bin\libmmd.dll",
            "$env:ONEAPI_ROOT\compiler\latest\bin\libiomp5md.dll",
            "$env:ONEAPI_ROOT\compiler\latest\bin\sycl-ls.exe",
            "$env:ONEAPI_ROOT\compiler\latest\bin\libsycl-fallback-bfloat16.spv",
            "$env:ONEAPI_ROOT\compiler\latest\bin\libsycl-native-bfloat16.spv",
            "$env:ONEAPI_ROOT\dnnl\latest\bin\dnnl.dll",
            "$env:ONEAPI_ROOT\tbb\latest\bin\tbb12.dll",
            "$env:ONEAPI_ROOT\tcm\latest\bin\tcm.dll",
            "$env:ONEAPI_ROOT\tcm\latest\bin\libhwloc-15.dll",
            "$env:ONEAPI_ROOT\umf\latest\bin\umf.dll"
          )

          foreach ($file in $runtimeFiles) {
            Copy-Item -LiteralPath $file -Destination $bin -Force
          }

          $zeLoader = Get-ChildItem -Path @($env:ONEAPI_ROOT, $env:LEVEL_ZERO_V1_SDK_PATH) -Recurse -Filter "ze_loader.dll" -ErrorAction SilentlyContinue | Select-Object -First 1
          if ($zeLoader) {
            Copy-Item -LiteralPath $zeLoader.FullName -Destination $bin -Force
          } else {
            Write-Warning "ze_loader.dll was not found in oneAPI or the Level Zero SDK; relying on the system driver runtime"
          }

          python scripts\verify-windows-package.py $bin
          7z a -snl "beellama-bin-win-sycl-x64.zip" "$bin\*"

      - name: Upload SYCL backend
        uses: actions/upload-artifact@v6
        with:
          path: beellama-bin-win-sycl-x64.zip
          name: beellama-bin-win-sycl-x64.zip

  windows-cuda:
    name: Windows x64 CUDA ${{ matrix.cuda }}
    needs: release-meta
    if: ${{ needs.release-meta.outputs.build_packages == 'true' }}
    runs-on: windows-2022
    env:
      CCACHE_MAXSIZE: 5G
      CCACHE_SLOPPINESS: time_macros
    strategy:
      fail-fast: false
      matrix:
        cuda: ["12.4", "13.1"]
    steps:
      - name: Clone
        uses: actions/checkout@v6
        with:
          ref: ${{ needs.release-meta.outputs.source_sha }}
          fetch-depth: 0

      - name: Setup Node.js
        uses: actions/setup-node@v6
        with:
          node-version: "24"
          cache: "npm"
          cache-dependency-path: "tools/ui/package-lock.json"

      - name: Install ccache
        uses: ggml-org/ccache-action@v1.2.21
        with:
          key: release-windows-cuda-${{ matrix.cuda }}
          variant: ccache
          evict-old-files: 1d

      - name: Cache CUDA Toolkit
        id: cache-cuda
        uses: actions/cache@v5
        with:
          path: C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v${{ matrix.cuda }}
          key: cuda-windows-${{ matrix.cuda }}-${{ hashFiles('.github/actions/windows-setup-cuda/action.yml') }}

      - name: Install CUDA Toolkit
        if: steps.cache-cuda.outputs.cache-hit != 'true'
        uses: ./.github/actions/windows-setup-cuda
        with:
          cuda_version: ${{ matrix.cuda }}

      - name: Configure CUDA environment
        shell: pwsh
        run: |
          $ErrorActionPreference = "Stop"
          $cudaPath = "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v${{ matrix.cuda }}"
          $nvcc = Join-Path $cudaPath "bin\nvcc.exe"
          if (-not (Test-Path $nvcc)) {
            throw "CUDA Toolkit not found at $cudaPath"
          }

          Join-Path $cudaPath "bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
          $libNvvp = Join-Path $cudaPath "libnvvp"
          if (Test-Path $libNvvp) {
            $libNvvp | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
          }

          "CUDA_PATH=$cudaPath" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
          $versionEnv = "CUDA_PATH_V${{ matrix.cuda }}".Replace(".", "_")
          "$versionEnv=$cudaPath" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append

      - name: Install Ninja
        run: choco install ninja -y

      - name: Build
        shell: cmd
        run: |
          call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64
          cmake -S . -B build -G "Ninja Multi-Config" ^
            -DGGML_BACKEND_DL=ON ^
            -DGGML_NATIVE=OFF ^
            -DGGML_CPU=OFF ^
            -DGGML_CUDA=ON ^
            -DGGML_CUDA_FA=ON ^
            -DGGML_CUDA_FA_ALL_QUANTS=ON ^
            -DLLAMA_BUILD_BORINGSSL=ON ^
            -DCMAKE_CUDA_COMPILER_LAUNCHER=ccache ^
            -DGGML_CUDA_CUB_3DOT2=ON
          set /A NINJA_JOBS=%NUMBER_OF_PROCESSORS%-1
          cmake --build build --config Release -j %NINJA_JOBS% --target ggml-cuda

      - name: ccache stats
        if: always()
        run: ccache --show-stats

      - name: Pack CUDA backend
        run: 7z a -snl "beellama-bin-win-cuda-${{ matrix.cuda }}-x64.zip" .\build\bin\Release\ggml-cuda.dll

      - name: Upload CUDA backend
        uses: actions/upload-artifact@v6
        with:
          path: beellama-bin-win-cuda-${{ matrix.cuda }}-x64.zip
          name: beellama-bin-win-cuda-${{ matrix.cuda }}-x64.zip

      - name: Pack CUDA runtime
        shell: pwsh
        run: |
          $ErrorActionPreference = "Stop"
          $dst = ".\build\bin\cudart"
          New-Item -ItemType Directory -Force -Path $dst | Out-Null
          $roots = @("${{ env.CUDA_PATH }}\bin", "${{ env.CUDA_PATH }}\lib", "${{ env.CUDA_PATH }}\bin\x64")
          foreach ($root in $roots) {
            if (Test-Path $root) {
              Get-ChildItem -Path (Join-Path $root "*") -File -ErrorAction SilentlyContinue |
                Where-Object { $_.Name -like "cudart64_*.dll" -or $_.Name -like "cublas64_*.dll" -or $_.Name -like "cublasLt64_*.dll" } |
                Copy-Item -Destination $dst -Force
            }
          }
          7z a "cudart-beellama-bin-win-cuda-${{ matrix.cuda }}-x64.zip" "$dst\*"

      - name: Upload CUDA runtime
        uses: actions/upload-artifact@v6
        with:
          path: cudart-beellama-bin-win-cuda-${{ matrix.cuda }}-x64.zip
          name: cudart-beellama-bin-win-cuda-${{ matrix.cuda }}-x64.zip

  windows-hip:
    name: Windows x64 HIP/Radeon
    needs: release-meta
    if: ${{ needs.release-meta.outputs.build_packages == 'true' }}
    runs-on: windows-2022
    env:
      HIPSDK_INSTALLER_VERSION: "26.Q1"
      GPU_TARGETS: "gfx1150;gfx1151;gfx1200;gfx1201;gfx1100;gfx1101;gfx1102;gfx1030;gfx1031;gfx1032"
    steps:
      - name: Clone
        uses: actions/checkout@v6
        with:
          ref: ${{ needs.release-meta.outputs.source_sha }}
          fetch-depth: 0

      - name: Setup Node.js
        uses: actions/setup-node@v6
        with:
          node-version: "24"
          cache: "npm"
          cache-dependency-path: "tools/ui/package-lock.json"

      - name: Grab rocWMMA package
        run: |
          curl -o rocwmma.deb "https://repo.radeon.com/rocm/apt/7.2.1/pool/main/r/rocwmma-dev/rocwmma-dev_2.2.0.70201-81~24.04_amd64.deb"
          7z x rocwmma.deb
          7z x data.tar

      - name: Cache ROCm installation
        id: cache-rocm
        uses: actions/cache@v5
        with:
          path: C:\Program Files\AMD\ROCm
          key: rocm-${{ env.HIPSDK_INSTALLER_VERSION }}-${{ runner.os }}

      - name: ccache
        uses: ggml-org/ccache-action@v1.2.21
        with:
          key: release-windows-hip-${{ env.HIPSDK_INSTALLER_VERSION }}-x64
          evict-old-files: 1d

      - name: Install ROCm
        if: steps.cache-rocm.outputs.cache-hit != 'true'
        run: |
          $ErrorActionPreference = "Stop"
          Invoke-WebRequest -Uri "https://download.amd.com/developer/eula/rocm-hub/AMD-Software-PRO-Edition-${{ env.HIPSDK_INSTALLER_VERSION }}-Win11-For-HIP.exe" -OutFile "${env:RUNNER_TEMP}\rocm-install.exe"
          $installTimeoutSeconds = 1800
          $proc = Start-Process "${env:RUNNER_TEMP}\rocm-install.exe" -ArgumentList '-install' -WindowStyle Hidden -PassThru
          $completed = $proc.WaitForExit($installTimeoutSeconds * 1000)
          if (-not $completed) {
            $proc.Kill()
            Write-Error "ROCm installation timed out after $($installTimeoutSeconds / 60) minutes"
          }
          if ($proc.ExitCode -ne 0) {
            Write-Error "ROCm installation failed with exit code $($proc.ExitCode)"
          }

      - name: Verify ROCm
        run: |
          $clangPath = Get-ChildItem 'C:\Program Files\AMD\ROCm\*\bin\clang.exe' | Select-Object -First 1
          if (-not $clangPath) {
            Write-Error "ROCm installation not found"
          }
          & $clangPath.FullName --version

      - name: Build
        run: |
          $env:HIP_PATH=$(Resolve-Path 'C:\Program Files\AMD\ROCm\*\bin\clang.exe' | Split-Path | Split-Path)
          $env:CMAKE_PREFIX_PATH="${env:HIP_PATH}"
          cmake -G "Unix Makefiles" -B build -S . `
            -DCMAKE_C_COMPILER="${env:HIP_PATH}\bin\clang.exe" `
            -DCMAKE_CXX_COMPILER="${env:HIP_PATH}\bin\clang++.exe" `
            -DCMAKE_CXX_FLAGS="-I$($PWD.Path.Replace('\', '/'))/opt/rocm-7.2.1/include/ -Wno-ignored-attributes -Wno-nested-anon-types" `
            -DCMAKE_BUILD_TYPE=Release `
            -DGGML_BACKEND_DL=ON `
            -DGGML_NATIVE=OFF `
            -DGGML_CPU=OFF `
            -DGPU_TARGETS="${env:GPU_TARGETS}" `
            -DGGML_HIP_ROCWMMA_FATTN=ON `
            -DGGML_HIP=ON `
            -DGGML_CUDA_FA=ON `
            -DGGML_CUDA_FA_ALL_QUANTS=ON `
            -DLLAMA_BUILD_BORINGSSL=ON
          cmake --build build --target ggml-hip -j ${env:NUMBER_OF_PROCESSORS}
          New-Item -ItemType Directory -Force -Path "build\bin\rocblas\library" | Out-Null
          New-Item -ItemType Directory -Force -Path "build\bin\hipblaslt\library" | Out-Null
          Copy-Item "${env:HIP_PATH}\bin\libhipblas.dll" "build\bin\"
          Copy-Item "${env:HIP_PATH}\bin\libhipblaslt.dll" "build\bin\"
          Copy-Item "${env:HIP_PATH}\bin\rocblas.dll" "build\bin\"
          Copy-Item "${env:HIP_PATH}\bin\rocblas\library\*" "build\bin\rocblas\library\"
          Copy-Item "${env:HIP_PATH}\bin\hipblaslt\library\*" "build\bin\hipblaslt\library\"

      - name: ccache stats
        if: always()
        run: ccache --show-stats

      - name: Pack artifacts
        run: |
          python scripts\verify-windows-package.py .\build\bin
          7z a -snl "beellama-bin-win-hip-radeon-x64.zip" .\build\bin\*

      - name: Upload artifacts
        uses: actions/upload-artifact@v6
        with:
          path: beellama-bin-win-hip-radeon-x64.zip
          name: beellama-bin-win-hip-radeon-x64.zip

  package-assets:
    name: Assemble release assets
    needs:
      - release-meta
      - macos-arm64
      - ubuntu-cpu
      - ubuntu-vulkan
      - ubuntu-rocm
      - ubuntu-sycl
      - ubuntu-cuda
      - windows-cpu
      - windows-sycl
      - windows-cuda
      - windows-hip
    runs-on: ubuntu-24.04
    steps:
      - name: Clone
        uses: actions/checkout@v6
        with:
          ref: ${{ needs.release-meta.outputs.source_sha }}
          fetch-depth: 0

      - name: Download package artifacts
        uses: actions/download-artifact@v7
        with:
          pattern: beellama-bin-*
          path: ./artifact
          merge-multiple: true

      - name: Download CUDA runtime artifacts
        uses: actions/download-artifact@v7
        with:
          pattern: cudart-beellama-*
          path: ./artifact
          merge-multiple: true

      - name: Prepare release assets
        shell: bash
        env:
          TAG_NAME: ${{ needs.release-meta.outputs.tag_name }}
        run: |
          set -euo pipefail
          shopt -s nullglob

          mkdir -p release

          cpu_zip="artifact/beellama-bin-win-cpu-x64.zip"
          if [[ -f "${cpu_zip}" ]]; then
            temp_dir="$(mktemp -d)"
            unzip -q "${cpu_zip}" -d "${temp_dir}"
            for target_zip in artifact/beellama-bin-win-*-x64.zip; do
              if [[ "$(basename "${target_zip}")" == "beellama-bin-win-cpu-x64.zip" ]]; then
                continue
              fi
              realpath_target_zip="$(realpath "${target_zip}")"
              (cd "${temp_dir}" && zip -qr "${realpath_target_zip}" .)
            done
            rm -rf "${temp_dir}"
          fi

          for zip_file in artifact/beellama-bin-win-*.zip; do
            base_name="$(basename "${zip_file}" .zip)"
            mv "${zip_file}" "release/beellama-${TAG_NAME}-${base_name#beellama-}.zip"
          done

          for zip_file in artifact/cudart-beellama-bin-win-cuda-*.zip; do
            base_name="$(basename "${zip_file}" .zip)"
            suffix="${base_name#cudart-beellama-bin-win-}"
            mv "${zip_file}" "release/beellama-${TAG_NAME}-cudart-win-${suffix}.zip"
          done

          for tar_file in artifact/*.tar.gz; do
            mv "${tar_file}" release/
          done

          python3 scripts/verify-windows-package.py release/beellama-${TAG_NAME}-bin-win-*.zip
          for cuda_version in 12.4 13.1; do
            package_zip="release/beellama-${TAG_NAME}-bin-win-cuda-${cuda_version}-x64.zip"
            runtime_zip="release/beellama-${TAG_NAME}-cudart-win-cuda-${cuda_version}-x64.zip"
            if [[ -f "${package_zip}" && -f "${runtime_zip}" ]]; then
              temp_dir="$(mktemp -d)"
              unzip -q "${package_zip}" -d "${temp_dir}"
              unzip -q "${runtime_zip}" -d "${temp_dir}"
              python3 scripts/verify-windows-package.py "${temp_dir}"
              rm -rf "${temp_dir}"
            fi
          done
          sha256sum release/* > release/SHA256SUMS.txt
          ls -lh release

      - name: Write release notes
        shell: bash
        env:
          TAG_NAME: ${{ needs.release-meta.outputs.tag_name }}
          IMAGE_REPO: ${{ needs.release-meta.outputs.image_repo }}
        run: |
          set -euo pipefail
          release_url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/releases/download/${TAG_NAME}"
          changelog_url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/blob/main/CHANGELOG.md"
          repo_name="${GITHUB_REPOSITORY#*/}"
          repo_name="${repo_name,,}"
          package_url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/pkgs/container/${repo_name}"
          python3 scripts/extract-changelog.py \
            --tag "${TAG_NAME}" \
            --changelog CHANGELOG.md \
            --fallback-url "${changelog_url}" \
            --output changelog-fragment.md
          python3 scripts/write-release-notes.py \
            --tag "${TAG_NAME}" \
            --release-url "${release_url}" \
            --image-repo "${IMAGE_REPO}" \
            --package-url "${package_url}" \
            --changelog changelog-fragment.md \
            --output release-notes.md

      - name: Upload release assets
        uses: actions/upload-artifact@v6
        with:
          name: beellama-release-assets
          path: |
            release/*
            release-notes.md

  docker-build:
    name: Docker ${{ matrix.config.display }}
    if: ${{ needs.release-meta.outputs.publish_docker == 'true' }}
    needs:
      - release-meta
      - ubuntu-cpu
      - ubuntu-vulkan
      - ubuntu-rocm
      - ubuntu-sycl
      - ubuntu-cuda
    runs-on: ${{ matrix.config.runs_on }}
    strategy:
      fail-fast: false
      matrix:
        config:
          - name: cpu-amd64
            display: CPU amd64
            group: cpu
            arch: amd64
            platform: linux/amd64
            artifact: beellama-bin-ubuntu-x64.tar.gz
            dockerfile: .devops/runtime-server.Dockerfile
            base_image: ubuntu:24.04
            runtime_packages: ca-certificates curl libgomp1
            runs_on: ubuntu-24.04
          - name: cpu-arm64
            display: CPU arm64
            group: cpu
            arch: arm64
            platform: linux/arm64
            artifact: beellama-bin-ubuntu-arm64.tar.gz
            dockerfile: .devops/runtime-server.Dockerfile
            base_image: ubuntu:24.04
            runtime_packages: ca-certificates curl libgomp1
            runs_on: ubuntu-24.04-arm
          - name: cuda12
            display: CUDA 12.4
            group: cuda12
            arch: amd64
            platform: linux/amd64
            artifact: beellama-bin-ubuntu-cuda-12.4-x64.tar.gz
            dockerfile: .devops/runtime-server.Dockerfile
            base_image: nvidia/cuda:12.4.1-runtime-ubuntu22.04
            runtime_packages: ca-certificates curl libgomp1
            runs_on: ubuntu-24.04
          - name: cuda13
            display: CUDA 13.1
            group: cuda13
            arch: amd64
            platform: linux/amd64
            artifact: beellama-bin-ubuntu-cuda-13.1-x64.tar.gz
            dockerfile: .devops/runtime-server.Dockerfile
            base_image: nvidia/cuda:13.1.1-runtime-ubuntu24.04
            runtime_packages: ca-certificates curl libgomp1
            runs_on: ubuntu-24.04
          - name: rocm
            display: ROCm
            group: rocm
            arch: amd64
            platform: linux/amd64
            artifact: beellama-bin-ubuntu-rocm-7.2-x64.tar.gz
            dockerfile: .devops/runtime-server.Dockerfile
            base_image: rocm/dev-ubuntu-24.04:7.2.1-complete
            runtime_packages: ca-certificates curl libgomp1
            runs_on: ubuntu-24.04
          - name: vulkan
            display: Vulkan
            group: vulkan
            arch: amd64
            platform: linux/amd64
            artifact: beellama-bin-ubuntu-vulkan-x64.tar.gz
            dockerfile: .devops/runtime-server.Dockerfile
            base_image: ubuntu:24.04
            runtime_packages: ca-certificates curl libegl1 libgl1 libgles2 libglvnd0 libglx0 libgomp1 libvulkan1 mesa-vulkan-drivers
            runs_on: ubuntu-24.04
          - name: sycl
            display: SYCL
            group: sycl
            arch: amd64
            platform: linux/amd64
            artifact: beellama-bin-ubuntu-sycl-x64.tar.gz
            dockerfile: .devops/runtime-intel-server.Dockerfile
            base_image: unused
            runtime_packages: unused
            runs_on: ubuntu-24.04
    permissions:
      contents: read
      packages: write
    steps:
      - name: Clone
        uses: actions/checkout@v6
        with:
          ref: ${{ needs.release-meta.outputs.source_sha }}
          fetch-depth: 0

      - name: Download package artifact
        uses: actions/download-artifact@v7
        with:
          name: ${{ matrix.config.artifact }}
          path: ./package

      - name: Prepare Docker context
        shell: bash
        run: |
          set -euo pipefail
          mkdir -p docker-context/app
          tar_file="$(find package -type f -name '*.tar.gz' | head -n 1)"
          if [[ -z "${tar_file}" ]]; then
            echo "No tar.gz package found for ${{ matrix.config.artifact }}" >&2
            exit 1
          fi
          tar -xzf "${tar_file}" -C docker-context/app --strip-components=1
          test -x docker-context/app/llama-server
          cp "${{ matrix.config.dockerfile }}" docker-context/Dockerfile
          ls -lh docker-context/app/llama-server

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4

      - name: Log in to GHCR
        uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4
        with:
          registry: ghcr.io
          username: ${{ github.repository_owner }}
          password: ${{ secrets.GITHUB_TOKEN }}

      - name: Get build date
        id: build_date
        run: echo "date=$(date -u +"%Y-%m-%dT%H:%M:%SZ")" >> "${GITHUB_OUTPUT}"

      - name: Build and push image digest
        id: build
        uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v7
        with:
          context: ./docker-context
          file: ./docker-context/Dockerfile
          platforms: ${{ matrix.config.platform }}
          outputs: type=image,name=${{ needs.release-meta.outputs.image_repo }},push-by-digest=true,name-canonical=true,push=true,oci-mediatypes=true
          provenance: false
          build-args: |
            BASE_IMAGE=${{ matrix.config.base_image }}
            RUNTIME_PACKAGES=${{ matrix.config.runtime_packages }}
            BUILD_DATE=${{ steps.build_date.outputs.date }}
            APP_VERSION=${{ needs.release-meta.outputs.tag_name }}
            APP_REVISION=${{ needs.release-meta.outputs.source_sha }}
            IMAGE_URL=${{ github.server_url }}/${{ github.repository }}
            IMAGE_SOURCE=${{ github.server_url }}/${{ github.repository }}
          annotations: |
            manifest:org.opencontainers.image.created=${{ steps.build_date.outputs.date }}
            manifest:org.opencontainers.image.version=${{ needs.release-meta.outputs.tag_name }}
            manifest:org.opencontainers.image.revision=${{ needs.release-meta.outputs.source_sha }}
            manifest:org.opencontainers.image.title=BeeLlama.cpp
            manifest:org.opencontainers.image.description=BeeLlama.cpp GGUF inference with DFlash, TurboQuant, and TCQ cache types
            manifest:org.opencontainers.image.url=${{ github.server_url }}/${{ github.repository }}
            manifest:org.opencontainers.image.source=${{ github.server_url }}/${{ github.repository }}
          cache-from: type=registry,ref=${{ needs.release-meta.outputs.image_repo }}:buildcache-runtime-${{ matrix.config.name }}
          cache-to: type=registry,ref=${{ needs.release-meta.outputs.image_repo }}:buildcache-runtime-${{ matrix.config.name }},mode=max

      - name: Record digest
        shell: bash
        run: |
          set -euo pipefail
          digest="${{ steps.build.outputs.digest }}"
          if [[ -z "${digest}" ]]; then
            echo "Docker build did not return a digest." >&2
            exit 1
          fi
          mkdir -p /tmp/docker-digests
          printf '%s\t%s\t%s\n' "${{ matrix.config.group }}" "${{ matrix.config.arch }}" "${digest}" > "/tmp/docker-digests/${{ matrix.config.name }}.tsv"

      - name: Upload digest metadata
        uses: actions/upload-artifact@v6
        with:
          name: docker-digest-${{ matrix.config.name }}
          path: /tmp/docker-digests/${{ matrix.config.name }}.tsv
          if-no-files-found: error

  docker-merge:
    name: Docker tags ${{ matrix.config.display }}
    if: ${{ needs.release-meta.outputs.publish_docker == 'true' }}
    needs:
      - release-meta
      - docker-build
    runs-on: ubuntu-24.04
    permissions:
      contents: read
      packages: write
    strategy:
      fail-fast: false
      matrix:
        config:
          - group: cpu
            display: CPU
            platforms: linux/amd64 linux/arm64
            tag_suffixes: none -cpu
          - group: cuda12
            display: CUDA 12.4
            platforms: linux/amd64
            tag_suffixes: -cuda -cuda12
          - group: cuda13
            display: CUDA 13.1
            platforms: linux/amd64
            tag_suffixes: -cuda13
          - group: rocm
            display: ROCm
            platforms: linux/amd64
            tag_suffixes: -rocm
          - group: vulkan
            display: Vulkan
            platforms: linux/amd64
            tag_suffixes: -vulkan
          - group: sycl
            display: SYCL
            platforms: linux/amd64
            tag_suffixes: -sycl
    steps:
      - name: Download digest metadata
        uses: actions/download-artifact@v7
        with:
          pattern: docker-digest-*
          path: /tmp/docker-digests
          merge-multiple: true

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4

      - name: Log in to GHCR
        uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4
        with:
          registry: ghcr.io
          username: ${{ github.repository_owner }}
          password: ${{ secrets.GITHUB_TOKEN }}

      - name: Create Docker tags
        shell: bash
        env:
          IMAGE_REPO: ${{ needs.release-meta.outputs.image_repo }}
          TAG_NAME: ${{ needs.release-meta.outputs.tag_name }}
          RELEASE_TAGS: ${{ needs.release-meta.outputs.create_release }}
          REF_NAME_SAFE: ${{ needs.release-meta.outputs.ref_name_safe }}
          SHORT_SHA: ${{ needs.release-meta.outputs.short_sha }}
          GROUP: ${{ matrix.config.group }}
          PLATFORMS: ${{ matrix.config.platforms }}
          TAG_SUFFIXES: ${{ matrix.config.tag_suffixes }}
        run: |
          set -euo pipefail

          digest_glob="/tmp/docker-digests/*.tsv"
          if ! ls ${digest_glob} >/dev/null 2>&1; then
            echo "No Docker digest metadata found." >&2
            exit 1
          fi

          sources=()
          for platform in ${PLATFORMS}; do
            arch="${platform#linux/}"
            digest="$(awk -F '\t' -v g="${GROUP}" -v a="${arch}" '$1 == g && $2 == a { print $3; exit }' ${digest_glob})"
            if [[ -z "${digest}" ]]; then
              echo "Missing digest for group=${GROUP} arch=${arch}" >&2
              exit 1
            fi
            sources+=("${IMAGE_REPO}@${digest}")
          done

          tag_args=()
          for suffix in ${TAG_SUFFIXES}; do
            if [[ "${suffix}" == "none" ]]; then
              suffix=""
            fi
            if [[ "${RELEASE_TAGS}" == "true" ]]; then
              tag_args+=("-t" "${IMAGE_REPO}:server${suffix}")
              tag_args+=("-t" "${IMAGE_REPO}:server${suffix}-${TAG_NAME}")
            else
              tag_args+=("-t" "${IMAGE_REPO}:server${suffix}-${REF_NAME_SAFE}-dev")
              tag_args+=("-t" "${IMAGE_REPO}:server${suffix}-${REF_NAME_SAFE}-${SHORT_SHA}")
            fi
          done

          printf 'Creating Docker tags:\n'
          printf '  %s\n' "${tag_args[@]}"
          docker buildx imagetools create "${tag_args[@]}" "${sources[@]}"

  release:
    name: Publish GitHub release
    if: ${{ needs.release-meta.outputs.create_release == 'true' }}
    needs:
      - release-meta
      - package-assets
      - docker-merge
    runs-on: ubuntu-24.04
    permissions:
      contents: write
    steps:
      - name: Clone
        uses: actions/checkout@v6
        with:
          fetch-depth: 0

      - name: Verify release tag
        env:
          TAG_NAME: ${{ needs.release-meta.outputs.tag_name }}
          SOURCE_SHA: ${{ needs.release-meta.outputs.source_sha }}
        run: |
          set -euo pipefail
          git fetch origin +refs/tags/"${TAG_NAME}":refs/tags/"${TAG_NAME}"
          tag_commit="$(git rev-list -n 1 "${TAG_NAME}")"
          if [[ "${tag_commit}" != "${SOURCE_SHA}" ]]; then
            echo "Tag ${TAG_NAME} moved from ${SOURCE_SHA} to ${tag_commit}; refusing to publish." >&2
            exit 1
          fi

      - name: Download release assets
        uses: actions/download-artifact@v7
        with:
          name: beellama-release-assets
          path: ./release-bundle

      - name: Create or update release
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          TAG_NAME: ${{ needs.release-meta.outputs.tag_name }}
        run: |
          set -euo pipefail
          notes_file="$(find release-bundle -type f -name release-notes.md | head -n 1)"
          if [[ -z "${notes_file}" ]]; then
            echo "release-notes.md not found in release asset bundle" >&2
            exit 1
          fi

          mapfile -t release_files < <(find release-bundle -type f ! -name release-notes.md | sort)
          if [[ "${#release_files[@]}" -eq 0 ]]; then
            echo "No release files found in release asset bundle" >&2
            exit 1
          fi

          if gh release view "${TAG_NAME}" --repo "${GITHUB_REPOSITORY}" >/dev/null 2>&1; then
            gh release upload "${TAG_NAME}" "${release_files[@]}" --clobber --repo "${GITHUB_REPOSITORY}"
            gh release edit "${TAG_NAME}" --repo "${GITHUB_REPOSITORY}" --title "${TAG_NAME}" --notes-file "${notes_file}"
          else
            gh release create "${TAG_NAME}" "${release_files[@]}" --repo "${GITHUB_REPOSITORY}" --title "${TAG_NAME}" --notes-file "${notes_file}" --verify-tag
          fi