release_build.yml

# FIXME: update this workflow
name: release_build

on: # yamllint disable-line rule:truthy
  workflow_dispatch:
    inputs:
      prerelease:
        description: 'pre-release'
        required: false
        type: boolean
        default: false
  # schedule:
  #   # build at 15:30UTC,8:30PST, 23:30CST, on every Tuesday and Friday
  #   - cron:  '30 15 * * 4' #was 30 15 * * 2,5

jobs:
  create_release:
    needs: update_version
    runs-on: ubuntu-latest
    steps:
      - name: Create GitHub release with notes
        uses: softprops/action-gh-release@v2.5.0
        with:
          token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
          tag_name: ${{ needs.update_version.outputs.tag_name }}
          prerelease: ${{ inputs.prerelease }}
          generate_release_notes: true
          body: |
            ## Docker Images Available

            ```bash
            docker pull d.timeplus.com/timeplus-io/proton:latest
            ```

            For detailed usage and more information, check out the Timeplus documentation: https://docs.timeplus.com/

  update_version:
    uses: timeplus-io/proton/.github/workflows/run_command.yml@develop
    with:
      ec2-instance-type: c5.large
      ec2-image-id: ami-042a37e33a285c22b
      submodules: 'false'
      run_mode: 'start' # start ec2 on demand instance
      upjob: update_version
      command: |
        cd $GITHUB_WORKSPACE

        # git config
        git config user.name "proton-robot"
        git config user.email "proton_robot@timeplus.io"

        # update version
        if [ "${{ inputs.prerelease }}" == "true" ]; then
          ./release --rc --version patch
        else
          ./release --version patch
        fi

        # get proton tag first
        PROTON_TAG=`grep "SET(VERSION_DESCRIBE" $GITHUB_WORKSPACE/cmake/autogenerated_versions.txt | sed 's/^.*VERSION_DESCRIBE \(.*\)$/\1/' | sed 's/[) ].*//'`
        echo "Proton tag: $PROTON_TAG"
        echo "tag_name=$PROTON_TAG" >> $GITHUB_OUTPUT
    secrets:
      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
      AWS_REGION: ${{ secrets.AWS_REGION }}
      AWS_S3_BUCKET: ${{ vars.AWS_S3_BUCKET_WEST_2 }}
      GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
      DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
      DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
  Build_Linux_X86_64:
    needs: [update_version, create_release]
    uses: timeplus-io/proton/.github/workflows/run_command.yml@develop
    with:
      ec2-instance-type: c7i.8xlarge
      ec2-image-id: ${{ vars.X64_AMI }}
      submodules: 'true'
      run_mode: 'start' # start ec2 on demand instance
      upload_files: |
        proton-*-Linux-x86_64.tar.gz
        proton-*-Linux-x86_64-debug-symbols.tar.gz
        python-Linux-x86_64.tar.gz
      prerelease: ${{ inputs.prerelease }}
      ref: ${{ needs.update_version.outputs.tag_name }}
      upjob: Build_Linux_X86_64
      generate_release_notes: false
      release_body: ''
      upload_only: true
      command: |
        cd $GITHUB_WORKSPACE

        # git config
        git config user.name "proton-robot"
        git config user.email "proton_robot@timeplus.io"

        # prepare build cache
        aws s3 cp --no-progress s3://tp-internal2/proton-oss/cache.tar.gz .
        mkdir $GITHUB_WORKSPACE/ccache
        tar -zxf ./cache.tar.gz -C $GITHUB_WORKSPACE/ccache
        rm cache.tar.gz

        # compiling
        ./docker/packager/packager --package-type binary --docker-image-version clang-21 --build-type relwithdebinfo --proton-build --enable-proton-local --cache ccache --ccache_dir $GITHUB_WORKSPACE/ccache --output-dir $GITHUB_WORKSPACE/output

        if [ ! -f "$GITHUB_WORKSPACE/output/proton" ]; then
          echo "Compiling proton Failed"
          exit 127
        fi

        # get proton tag first
        PROTON_TAG=`grep "SET(VERSION_DESCRIBE" $GITHUB_WORKSPACE/cmake/autogenerated_versions.txt | sed 's/^.*VERSION_DESCRIBE \(.*\)$/\1/' | sed 's/[) ].*//'`
        echo "Proton tag: $PROTON_TAG"
        echo "tag_name=$PROTON_TAG" >> $GITHUB_OUTPUT

        PROTON_BINARY=proton-$PROTON_TAG-Linux-x86_64
        PROTON_DEBUG=proton-$PROTON_TAG-Linux-x86_64-debug-symbols.tar.gz

        echo "Building: $PROTON_BINARY"
        echo "Debug package: $PROTON_DEBUG"

        # Show original size
        echo "Original binary size:"
        ls -lh $GITHUB_WORKSPACE/output/proton

        # Work directly with the final filename - just rename once
        cd $GITHUB_WORKSPACE
        mv output/proton $PROTON_BINARY

        # Extract debug symbols from the final binary
        echo "Extracting debug symbols..."
        docker run --rm -v $GITHUB_WORKSPACE:/work -w /work timeplus/proton-binary-builder:clang-21 \
          /usr/bin/objcopy --only-keep-debug $PROTON_BINARY ${PROTON_BINARY}.debug

        # Create debug symbols package
        echo "Creating debug symbols package..."
        chmod 0644 ${PROTON_BINARY}.debug 
        tar -czf $PROTON_DEBUG ${PROTON_BINARY}.debug
        rm ${PROTON_BINARY}.debug

        # Strip the binary in-place (conservative approach)
        echo "Stripping binary in conservative approach..."
        docker run --rm -v $GITHUB_WORKSPACE:/work -w /work timeplus/proton-binary-builder:clang-21 \
          /usr/bin/llvm-strip-21 --strip-debug --remove-section=.comment --remove-section=.note $PROTON_BINARY

        # Show size comparison
        echo "Final sizes:"
        ls -lh $PROTON_BINARY $PROTON_DEBUG

        # Package the binary into a tar.gz for faster downloads
        echo "Packaging binary tarball..."
        tar -czf ${PROTON_BINARY}.tar.gz $PROTON_BINARY
        ls -lh ${PROTON_BINARY}.tar.gz

        # Start static server for docker build
        docker run --name static-server -p 8080:80 -v $GITHUB_WORKSPACE:/usr/share/nginx/html:ro -d nginx
        cd $GITHUB_WORKSPACE/docker/server

        # prepare files to be copied to the image
        mkdir -p resources/protos/google/protobuf
        cp -r $GITHUB_WORKSPACE/contrib/google-protobuf/src/google/protobuf/*.proto ./resources/protos/google/protobuf/
        rm -rf resources/protos/google/protobuf/unittest_*

        # build docker image for both Docker Hub and GHCR (using the stripped binary)
        docker build . --network host --build-arg single_binary_location_url=http://localhost:8080/$PROTON_BINARY \
          -t timeplus/proton:${GITHUB_SHA}_amd64 \
          -t ghcr.io/timeplus-io/proton:${GITHUB_SHA}_amd64

        # push docker image to both registries
        docker push timeplus/proton:${GITHUB_SHA}_amd64
        docker push ghcr.io/timeplus-io/proton:${GITHUB_SHA}_amd64

        # upload build cache
        tar -zcf ./cache.tar.gz -C $GITHUB_WORKSPACE/ccache .
        aws s3 cp --no-progress ./cache.tar.gz s3://tp-internal2/proton-oss/

        # prepare python package
        mkdir python-Linux-x86_64
        cd python-Linux-x86_64
        mkdir lib
        mkdir bin
        docker pull python:3.10

        # warning: do not use double quote here. It will be remove when the run_command workflow rending this command script
        docker run --name python_packages python:3.10 sh -c 'pip install --upgrade truststore; pip install --force-reinstall --no-cache-dir -t /usr/local/lib/python3.10/site-packages/ timeplus-neutrino==0.1.12;'
        docker cp python_packages:/usr/local/lib/libpython3.so ./lib/libpython3.so
        docker cp python_packages:/usr/local/lib/libpython3.10.so.1.0 ./lib/libpython3.10.so.1.0
        docker cp python_packages:/usr/local/lib/libpython3.10.so ./lib/libpython3.10.so
        docker cp python_packages:/usr/local/lib/python3.10 ./lib/python3.10
        docker cp python_packages:/usr/local/bin/python3.10 ./bin/python3.10
        tar -zcf $GITHUB_WORKSPACE/python-Linux-x86_64.tar.gz .

    secrets:
      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
      AWS_REGION: ${{ secrets.AWS_REGION }}
      AWS_S3_BUCKET: ${{ vars.AWS_S3_BUCKET_WEST_2 }}
      GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
      DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
      DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
  Build_Linux_Arm64:
    needs: [update_version, create_release]
    uses: timeplus-io/proton/.github/workflows/run_command.yml@develop
    with:
      ec2-instance-type: m7g.8xlarge
      ec2-image-id: ${{ vars.ARM_AMI }}
      submodules: 'true'
      run_mode: 'start' # start ec2 on demand instance
      upload_files: |
        proton-*-Linux-aarch64.tar.gz
      prerelease: ${{ inputs.prerelease }}
      ref: ${{ needs.update_version.outputs.tag_name }}
      upjob: Build_Linux_Arm64
      generate_release_notes: false
      release_body: ''
      upload_only: true
      command: |
        cd $GITHUB_WORKSPACE

        # git config
        git config user.name "proton-robot"
        git config user.email "proton_robot@timeplus.io"

        # prepare build cache
        aws s3 cp --no-progress s3://tp-internal2/proton-oss/cache-arm.tar.gz .
        mkdir $GITHUB_WORKSPACE/ccache
        tar -zxf ./cache-arm.tar.gz -C $GITHUB_WORKSPACE/ccache
        rm cache-arm.tar.gz

        # compiling
        ./docker/packager/packager --package-type binary --docker-image-version clang-21 --build-type release --proton-build --disable-python-udf --cache ccache --ccache_dir $GITHUB_WORKSPACE/ccache --output-dir $GITHUB_WORKSPACE/output

        if [ ! -f "$GITHUB_WORKSPACE/output/proton" ]; then
          echo "Compiling proton Failed"
          exit 127
        fi

        # strip
        ls -lh $GITHUB_WORKSPACE/output/proton
        docker run --rm -v $GITHUB_WORKSPACE/output:/work -w /work timeplus/proton-binary-builder:clang-21 /usr/bin/llvm-strip-21 proton
        ls -lh $GITHUB_WORKSPACE/output/proton

        # get proton tag
        PROTON_TAG=`grep "SET(VERSION_DESCRIBE" $GITHUB_WORKSPACE/cmake/autogenerated_versions.txt | sed 's/^.*VERSION_DESCRIBE \(.*\)$/\1/' | sed 's/[) ].*//'`
        echo "Proton tag: $PROTON_TAG"
        echo "tag_name=$PROTON_TAG" >> $GITHUB_OUTPUT

        PROTON_BINARY=proton-$PROTON_TAG-Linux-aarch64
        echo "Proton Binary Name: $PROTON_BINARY"
        
        docker run --name static-server -p 8080:80 -v $GITHUB_WORKSPACE/output:/usr/share/nginx/html:ro -d nginx
        cd $GITHUB_WORKSPACE/docker/server

        # prepare files to be copied to the image
        mkdir -p resources/protos/google/protobuf
        cp -r $GITHUB_WORKSPACE/contrib/google-protobuf/src/google/protobuf/*.proto ./resources/protos/google/protobuf/
        rm -rf resources/protos/google/protobuf/unittest_*

        # build docker image (skip Python UDF pieces on ARM64)
        docker build . --network host \
          --build-arg ENABLE_PYTHON_UDF=0 \
          --build-arg single_binary_location_url=http://localhost:8080/proton \
          -t timeplus/proton:${GITHUB_SHA}_arm64v8 \
          -t ghcr.io/timeplus-io/proton:${GITHUB_SHA}_arm64v8

        # push docker image to both registries
        docker push timeplus/proton:${GITHUB_SHA}_arm64v8
        docker push ghcr.io/timeplus-io/proton:${GITHUB_SHA}_arm64v8

        # upload build cache
        tar -zcf ./cache-arm.tar.gz -C $GITHUB_WORKSPACE/ccache .
        aws s3 cp --no-progress ./cache-arm.tar.gz s3://tp-internal2/proton-oss/
        
        mv $GITHUB_WORKSPACE/output/proton $GITHUB_WORKSPACE/$PROTON_BINARY

        # Package the binary into a tar.gz for faster downloads
        echo "Packaging binary tarball..."
        cd $GITHUB_WORKSPACE
        tar -czf ${PROTON_BINARY}.tar.gz $PROTON_BINARY
        ls -lh ${PROTON_BINARY}.tar.gz
    secrets:
      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
      AWS_REGION: ${{ secrets.AWS_REGION }}
      AWS_S3_BUCKET: ${{ vars.AWS_S3_BUCKET_WEST_2 }}
      GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
      DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
      DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
  CommitTag:
    needs: [Build_Linux_X86_64, Build_Linux_Arm64]
    uses: timeplus-io/proton/.github/workflows/run_command.yml@develop
    with:
      ec2-instance-type: c5.large
      ec2-image-id: ami-042a37e33a285c22b
      submodules: false
      upjob: CommitTag
      command: |
        cd $GITHUB_WORKSPACE

        # git config
        git config user.name "proton-robot"
        git config user.email "proton_robot@timeplus.io"

        # Pull images from both registries
        docker pull timeplus/proton:${GITHUB_SHA}_amd64
        docker pull timeplus/proton:${GITHUB_SHA}_arm64v8
        docker pull ghcr.io/timeplus-io/proton:${GITHUB_SHA}_amd64
        docker pull ghcr.io/timeplus-io/proton:${GITHUB_SHA}_arm64v8

        # Create and push Docker Hub manifests
        docker manifest create timeplus/proton:develop \
          timeplus/proton:${GITHUB_SHA}_amd64 \
          timeplus/proton:${GITHUB_SHA}_arm64v8

        docker manifest create timeplus/proton:latest \
          timeplus/proton:${GITHUB_SHA}_amd64 \
          timeplus/proton:${GITHUB_SHA}_arm64v8

        docker manifest push timeplus/proton:develop
        docker manifest push timeplus/proton:latest

        # Create and push GHCR manifests
        docker manifest create ghcr.io/timeplus-io/proton:develop \
          ghcr.io/timeplus-io/proton:${GITHUB_SHA}_amd64 \
          ghcr.io/timeplus-io/proton:${GITHUB_SHA}_arm64v8

        docker manifest create ghcr.io/timeplus-io/proton:latest \
          ghcr.io/timeplus-io/proton:${GITHUB_SHA}_amd64 \
          ghcr.io/timeplus-io/proton:${GITHUB_SHA}_arm64v8

        docker manifest push ghcr.io/timeplus-io/proton:develop
        docker manifest push ghcr.io/timeplus-io/proton:latest

        # Get version tag and create version-specific manifests for both registries
        PROTON_TAG="$(docker history --no-trunc timeplus/proton:${GITHUB_SHA}_amd64 | grep -Eo 'ARG version=[0-9]+.[0-9]+.[0-9]+(-[a-zA-Z0-9.]+)?' -o | grep -Eo '[0-9]+.[0-9]+.[0-9]+(-[a-zA-Z0-9.]+)?')"

        # Docker Hub version tag
        docker manifest create timeplus/proton:$PROTON_TAG \
          timeplus/proton:${GITHUB_SHA}_amd64 \
          timeplus/proton:${GITHUB_SHA}_arm64v8

        docker manifest push timeplus/proton:$PROTON_TAG

        # GHCR version tag
        docker manifest create ghcr.io/timeplus-io/proton:$PROTON_TAG \
          ghcr.io/timeplus-io/proton:${GITHUB_SHA}_amd64 \
          ghcr.io/timeplus-io/proton:${GITHUB_SHA}_arm64v8

        docker manifest push ghcr.io/timeplus-io/proton:$PROTON_TAG

        export REF="${GITHUB_REF#refs/heads/}"
        echo "branch_name: $REF"
        curl -L \
          -X POST \
          -H "Accept: application/vnd.github+json" \
          -H "Authorization: token $GH_PERSONAL_ACCESS_TOKEN"\
          -H "X-GitHub-Api-Version: 2022-11-28" \
          https://api.github.com/repos/timeplus-io/proton/actions/workflows/manual_trigger_test.yml/dispatches \
          -d "{\"ref\":\"$REF\",\"inputs\":{\"arch\": \"x64\", \"tag\":\"$PROTON_TAG\", \"test_category\":\"scope.release\", \"nodes\":\"1\"}}\""

        curl -L \
          -X POST \
          -H "Accept: application/vnd.github+json" \
          -H "Authorization: token $GH_PERSONAL_ACCESS_TOKEN"\
          -H "X-GitHub-Api-Version: 2022-11-28" \
          https://api.github.com/repos/timeplus-io/proton/actions/workflows/manual_trigger_test.yml/dispatches \
          -d "{\"ref\":\"$REF\",\"inputs\":{\"arch\": \"arm\", \"tag\":\"$PROTON_TAG\", \"test_category\":\"scope.release\", \"nodes\":\"1\"}}\""

        curl -L \
          -X POST \
          -H "Accept: application/vnd.github+json" \
          -H "Authorization: token $GH_PERSONAL_ACCESS_TOKEN"\
          -H "X-GitHub-Api-Version: 2022-11-28" \
          https://api.github.com/repos/timeplus-io/proton/actions/workflows/manual_trigger_test.yml/dispatches \
          -d "{\"ref\":\"$REF\",\"inputs\":{\"arch\": \"x64\", \"tag\":\"$PROTON_TAG\", \"test_category\":\"smoke\", \"has_feature_enabled\":\"hybrid\", \"nodes\":\"1\"}}\""
    secrets:
      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
      AWS_REGION: ${{ secrets.AWS_REGION }}
      AWS_S3_BUCKET: ${{ vars.AWS_S3_BUCKET_WEST_2 }}
      GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
      DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
      DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
  Build_Darwin_X86_64:
    needs: [update_version, create_release]
    uses: timeplus-io/proton/.github/workflows/run_command.yml@develop
    with:
      ec2-instance-type: c6i.8xlarge
      ec2-image-id: ami-042a37e33a285c22b
      submodules: 'true'
      run_mode: 'start' # start ec2 on demand instance
      upload_files: |
        proton-*-Darwin-x86_64.tar.gz
      prerelease: ${{ inputs.prerelease }}
      ref: ${{ needs.update_version.outputs.tag_name }}
      upjob: Build_Darwin_X86_64
      generate_release_notes: false
      release_body: ''
      upload_only: true
      command: |
        cd $GITHUB_WORKSPACE

        # git config
        git config user.name "proton-robot"
        git config user.email "proton_robot@timeplus.io"

        # download the pre-built binary of v8 (this is only for cross-compile)
        aws s3 cp --no-progress s3://tp-internal2/proton-oss/cross-compile-prebuilt-binary/138/v8-cmake-x64.tar.gz $GITHUB_WORKSPACE/contrib/v8-cmake/
        tar -zxf $GITHUB_WORKSPACE/contrib/v8-cmake/v8-cmake-x64.tar.gz -C $GITHUB_WORKSPACE/contrib/v8-cmake/
        rm $GITHUB_WORKSPACE/contrib/v8-cmake/v8-cmake-x64.tar.gz
        chmod a+x $GITHUB_WORKSPACE/contrib/v8-cmake/bytecode_builtins_list_generator
        chmod a+x $GITHUB_WORKSPACE/contrib/v8-cmake/mksnapshot
        chmod a+x $GITHUB_WORKSPACE/contrib/v8-cmake/torque
        
        # prepare build cache
        mkdir $GITHUB_WORKSPACE/ccache
        echo "max_size = 100.0G" > $GITHUB_WORKSPACE/ccache/ccache.conf

        # compiling
        ./docker/packager/packager --package-type binary --docker-image-version clang-18 --build-type release --proton-build --disable-python-udf --cache ccache --ccache_dir $GITHUB_WORKSPACE/ccache --output-dir $GITHUB_WORKSPACE/output --compiler clang-18-darwin

        if [ ! -f "$GITHUB_WORKSPACE/output/proton" ]; then
          echo "Compiling proton Failed"
          exit 127
        fi

        # strip
        ls -lh $GITHUB_WORKSPACE/output/proton
        docker run --rm -v $GITHUB_WORKSPACE/output:/work -w /work timeplus/proton-binary-builder:clang-18 /usr/bin/llvm-strip-18 proton
        ls -lh $GITHUB_WORKSPACE/output/proton

        # clean ccache and build footprint
        rm -rf $GITHUB_WORKSPACE/ccache
        rm -rf $GITHUB_WORKSPACE/build_docker

        # get proton tag
        PROTON_TAG=`grep "SET(VERSION_DESCRIBE" $GITHUB_WORKSPACE/cmake/autogenerated_versions.txt | sed 's/^.*VERSION_DESCRIBE \(.*\)$/\1/' | sed 's/[) ].*//'`
        echo "Proton tag: $PROTON_TAG"
        echo "tag_name=$PROTON_TAG" >> $GITHUB_OUTPUT

        PROTON_BINARY=proton-$PROTON_TAG-Darwin-x86_64
        echo "Proton Binary Name: $PROTON_BINARY"
        sha256sum $GITHUB_WORKSPACE/output/proton

        mv $GITHUB_WORKSPACE/output/proton $GITHUB_WORKSPACE/$PROTON_BINARY

        # Package the binary into a tar.gz for faster downloads
        echo "Packaging binary tarball..."
        cd $GITHUB_WORKSPACE
        tar -czf ${PROTON_BINARY}.tar.gz $PROTON_BINARY
        ls -lh ${PROTON_BINARY}.tar.gz
    secrets:
      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
      AWS_REGION: ${{ secrets.AWS_REGION }}
      AWS_S3_BUCKET: ${{ vars.AWS_S3_BUCKET_WEST_2 }}
      GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
      DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
      DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
  Build_Darwin_Arm64:
    if: false
    needs: [update_version, create_release]
    uses: timeplus-io/proton/.github/workflows/run_command.yml@develop
    with:
      ec2-instance-type: c6g.4xlarge
      ec2-image-id: ami-0f3dbc4cc9994fdee
      submodules: 'true'
      run_mode: 'start' # start ec2 on demand instance
      upload_files: proton-*-Darwin-arm64
      prerelease: ${{ inputs.prerelease }}
      ref: ${{ needs.update_version.outputs.tag_name }}
      upjob: Build_Darwin_Arm64
      generate_release_notes: false
      release_body: ''
      upload_only: true
      command: |
        cd $GITHUB_WORKSPACE

        # git config
        git config user.name "proton-robot"
        git config user.email "proton_robot@timeplus.io"

        # download the pre-built binary of v8 (this is only for cross-compile)
        aws s3 cp --no-progress s3://tp-internal2/proton-oss/cross-compile-prebuilt-binary/138/v8-cmake-arm64.tar.gz $GITHUB_WORKSPACE/contrib/v8-cmake/
        tar -zxf $GITHUB_WORKSPACE/contrib/v8-cmake/v8-cmake-arm64.tar.gz -C $GITHUB_WORKSPACE/contrib/v8-cmake/
        rm $GITHUB_WORKSPACE/contrib/v8-cmake/v8-cmake-arm64.tar.gz
        chmod a+x $GITHUB_WORKSPACE/contrib/v8-cmake/bytecode_builtins_list_generator
        chmod a+x $GITHUB_WORKSPACE/contrib/v8-cmake/mksnapshot
        chmod a+x $GITHUB_WORKSPACE/contrib/v8-cmake/torque

        # prepare build cache
        mkdir $GITHUB_WORKSPACE/ccache
        echo "max_size = 100.0G" > $GITHUB_WORKSPACE/ccache/ccache.conf

        # compiling
        ./docker/packager/packager --package-type binary --docker-image-version clang-21 --build-type release --proton-build --disable-python-udf --cache ccache --ccache_dir $GITHUB_WORKSPACE/ccache --output-dir $GITHUB_WORKSPACE/output --compiler clang-21-darwin-aarch64
        if [ ! -f "$GITHUB_WORKSPACE/output/proton" ]; then
          echo "Compiling proton Failed"
          exit 127
        fi

        # strip
        ls -lh $GITHUB_WORKSPACE/output/proton
        docker run --rm -v $GITHUB_WORKSPACE/output:/work -w /work timeplus/proton-binary-builder:clang-21 /usr/bin/llvm-strip-21 proton
        ls -lh $GITHUB_WORKSPACE/output/proton

        # clean ccache and build footprint
        rm -rf $GITHUB_WORKSPACE/ccache
        rm -rf $GITHUB_WORKSPACE/build_docker

        # get proton tag
        PROTON_TAG=`grep "SET(VERSION_DESCRIBE" $GITHUB_WORKSPACE/cmake/autogenerated_versions.txt | sed 's/^.*VERSION_DESCRIBE \(.*\)$/\1/' | sed 's/[) ].*//'`
        echo "Proton tag: $PROTON_TAG"
        echo "tag_name=$PROTON_TAG" >> $GITHUB_OUTPUT

        PROTON_BINARY=proton-$PROTON_TAG-Darwin-arm64
        echo "Proton Binary Name: $PROTON_BINARY"
        sha256sum $GITHUB_WORKSPACE/output/proton

        mv $GITHUB_WORKSPACE/output/proton $GITHUB_WORKSPACE/$PROTON_BINARY
    secrets:
      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
      AWS_REGION: ${{ secrets.AWS_REGION }}
      AWS_S3_BUCKET: ${{ vars.AWS_S3_BUCKET_WEST_2 }}
      GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
      DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
      DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}

  Build_Native_Darwin_arm64:
    needs: Build_Linux_X86_64 # Gate the job behind the Linux x86_64 job so the release exists before uploads.
    runs-on: [self-hosted, macOS, ARM64]
    env:
      build_directory: ${{ github.workspace }}/build
      build_type: RelWithDebInfo
    steps:
      - name: Checkout
        uses: actions/checkout@v5.0.1
        with:
          token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
          fetch-depth: 0  # Fetch all history including all tags
          submodules: true

      # Checkout the latest tag
      - name: Checkout latest tag
        run: |
          git fetch --all --tags
          REF="${GITHUB_REF#refs/heads/}"
          LATEST_TAG=$(git describe --tags --abbrev=0 origin/$REF)
          echo "Checking out tag $LATEST_TAG"
          git checkout $LATEST_TAG
          echo "LATEST_TAG=$LATEST_TAG" >> $GITHUB_ENV

      # Setup build environment and configure CMake
      - name: Create and Configure Build
        run: |
          mkdir -p ${{ env.build_directory }}
          export CC=$(brew --prefix llvm@21)/bin/clang
          export CXX=$(brew --prefix llvm@21)/bin/clang++
          export PATH=$(brew --prefix llvm@21)/bin:$PATH
          cmake -B ${{ env.build_directory }} -G "Ninja" -DCMAKE_BUILD_TYPE=${{ env.build_type }} -DENABLE_TESTS=OFF -DENABLE_UTILS=OFF -DENABLE_EXAMPLES=OFF -DENABLE_PROTON_LOCAL=OFF -DENABLE_PULSAR=ON -DENABLE_PYTHON_UDF=ON -DENABLE_PROTON_PYTHON=ON

      # Compile the project using Ninja
      - name: Build with Ninja
        run: cmake --build ${{ env.build_directory }}

      # Strip the binary to reduce its size and rename it
      - name: Strip and Rename binary
        run: |
          ORIGINAL_BINARY=${{ env.build_directory }}/programs/proton
          STRIPPED_BINARY=${{ env.build_directory }}/programs/proton-${{ env.LATEST_TAG }}-Darwin-arm64
          ls -lh $ORIGINAL_BINARY
          /opt/homebrew/opt/llvm@21/bin/llvm-strip $ORIGINAL_BINARY -o $STRIPPED_BINARY
          ls -lh $STRIPPED_BINARY

      # Set up AWS credentials for S3 upload
      - name: Configure AWS credentials
        uses: aws-actions/configure-aws-credentials@v6.0.0
        with:
          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
          aws-region: ${{ secrets.AWS_REGION }}

      # Upload the stripped binary to an AWS S3 bucket with retries
      - name: Upload Artifact To S3
        run: |
          STRIPPED_BINARY=${{ env.build_directory }}/programs/proton-${{ env.LATEST_TAG }}-Darwin-arm64
          S3_PATH="s3://tp-internal2/proton-oss/native_build_macOS/"

          # Function to upload with retry
          upload_with_retry() {
            local max_attempts=5
            local attempt=1
            local wait_time=15

            while [ $attempt -le $max_attempts ]; do
              echo "S3 upload attempt $attempt of $max_attempts for $(basename $1)"

              if aws s3 cp \
                 --no-progress \
                 --cli-connect-timeout 30 \
                 --cli-read-timeout 600 \
                 --only-show-errors \
                 "$1" "$2"; then
                echo "Upload completed successfully"
                return 0
              else
                local status=$?
                echo "Upload failed on attempt $attempt with status $status"

                if [ $attempt -eq $max_attempts ]; then
                  echo "All attempts failed"
                  return 1
                fi

                echo "Waiting $wait_time seconds before retry..."
                sleep $wait_time
                wait_time=$((wait_time * 2))  # Exponential backoff
                attempt=$((attempt + 1))
              fi
            done
          }
          
          # Upload binary with retry
          if ! upload_with_retry "$STRIPPED_BINARY" "${S3_PATH}$(basename $STRIPPED_BINARY)"; then
            echo "Binary upload failed after all attempts"
            exit 1
          fi

          # Prepare python3.10
          aws s3 cp --no-progress ${S3_PATH}python-Darwin-arm64.tar.gz ${{ env.build_directory }}/programs/python-Darwin-arm64.tar.gz

      # Print SHA256 Checksum
      - name: Print SHA256 Checksum
        run: |
          STRIPPED_BINARY=${{ env.build_directory }}/programs/proton-${{ env.LATEST_TAG }}-Darwin-arm64
          shasum -a 256 $STRIPPED_BINARY  

      # Release the binary using GitHub CLI with retry logic
      - name: Release binary using GitHub CLI
        env:
          GITHUB_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
        run: |
          STRIPPED_BINARY=${{ env.build_directory }}/programs/proton-${{ env.LATEST_TAG }}-Darwin-arm64
          PYTHON_PACKAGE=${{ env.build_directory }}/programs/python-Darwin-arm64.tar.gz
          TARBALL=${STRIPPED_BINARY}.tar.gz

          # Create tar.gz package for faster downloads
          echo "Packaging binary tarball..."
          tar -czf "$TARBALL" -C "$(dirname "$STRIPPED_BINARY")" "$(basename "$STRIPPED_BINARY")"
          ls -lh "$TARBALL"
          
          # Function to upload asset with retry
          upload_asset_with_retry() {
            local max_attempts=5
            local attempt=1
            local wait_time=15
            local file_path="$1"
            local tag="$2"
            
            while [ $attempt -le $max_attempts ]; do
              echo "Upload attempt $attempt of $max_attempts for $(basename $file_path)"
              
              if gh release upload "$tag" "$file_path" --clobber --repo ${{ github.repository }}; then
                echo "Upload completed successfully"
                return 0
              else
                local status=$?
                echo "Upload failed on attempt $attempt with status $status"
                
                if [ $attempt -eq $max_attempts ]; then
                  echo "All upload attempts failed"
                  return 1
                fi
                
                echo "Waiting $wait_time seconds before retry..."
                sleep $wait_time
                wait_time=$((wait_time * 2))  # Exponential backoff
                attempt=$((attempt + 1))
              fi
            done
          }
          
          # Upload tarball with retry
          if ! upload_asset_with_retry "$TARBALL" "${{ env.LATEST_TAG }}"; then
            echo "Tarball upload failed after all attempts"
            exit 1
          fi
          
          # Upload Python package with retry
          if ! upload_asset_with_retry "$PYTHON_PACKAGE" "${{ env.LATEST_TAG }}"; then
            echo "Python package upload failed after all attempts"
            exit 1
          fi

  Update_Homebrew_Tap:
    if: ${{ !inputs.prerelease }}
    needs:
      - update_version
      - Build_Darwin_X86_64
      - Build_Native_Darwin_arm64
    uses: ./.github/workflows/update_homebrew_tap.yml
    with:
      tag_name: ${{ needs.update_version.outputs.tag_name }}
    secrets:
      GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}