Build (4.0) #513
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Licensed to the Apache Software Foundation (ASF) under one | |
| # or more contributor license agreements. See the NOTICE file | |
| # distributed with this work for additional information | |
| # regarding copyright ownership. The ASF licenses this file | |
| # to you under the Apache License, Version 2.0 (the | |
| # "License"); you may not use this file except in compliance | |
| # with the License. You may obtain a copy of the License at | |
| # | |
| # http://www.apache.org/licenses/LICENSE-2.0 | |
| # | |
| # Unless required by applicable law or agreed to in writing, | |
| # software distributed under the License is distributed on an | |
| # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | |
| # KIND, either express or implied. See the License for the | |
| # specific language governing permissions and limitations | |
| # under the License. | |
| name: Build (4.0) | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| force_build: | |
| description: "Force run build job when manually triggered" | |
| required: false | |
| default: "true" | |
| schedule: | |
| - cron: '*/30 * * * *' | |
| jobs: | |
| prerelease: | |
| name: Prerelease | |
| runs-on: ubuntu-latest | |
| env: | |
| GH_REPO: ${{ github.repository }} | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| permissions: | |
| contents: write | |
| outputs: | |
| should_release: ${{ steps.check_diff.outputs.should_release }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: 'apache/doris' | |
| ref: 'branch-4.0' | |
| fetch-depth: 0 | |
| - name: Check Diff | |
| id: check_diff | |
| run: | | |
| tag_name='automation-4.0' | |
| title="Apache Doris Third Party Prebuilt (${tag_name/automation-/})" | |
| if [[ -z "$(gh release list)" ]] || | |
| ! gh release list | awk -F "\t" '{ print $3 }' | grep "${tag_name}" >/dev/null; then | |
| gh release create -t "${title}" "${tag_name}" | |
| fi | |
| last_version="$(gh release view "${tag_name}" | sed -n -E 's/Doris Version: \*(.*)\*.*/\1/p')" | |
| current_version="$(git log -1 --format='%H')" | |
| echo "Last Version: ${last_version}" | |
| echo "Current Version: ${current_version}" | |
| should_release=false | |
| if [[ -z "${last_version}" ]]; then | |
| echo "The first release was detected." | |
| should_release=true | |
| elif [[ "${last_version}" != "${current_version}" ]]; then | |
| cmd="git diff --name-only ${last_version} ${current_version} | grep -E '^thirdparty/'" | |
| echo "Execute: ${cmd}" | |
| content="$(eval "${cmd}")" || true | |
| if [[ -n "${content}" ]]; then | |
| echo -e "Detect changes:\n${content}" | |
| should_release=true | |
| fi | |
| fi | |
| if "${should_release}"; then | |
| echo -ne "Update Time: *$(date)*\nDoris Version: *${current_version}*\nStatus: *BUILDING*" >release_note.md | |
| else | |
| gh release view "${tag_name}" | sed -n '/--/,$p' | awk '{ if (NR > 1) print $0 }' | sed "{ | |
| s/Update Time:.*/Update Time: *$(date)*/ | |
| s/Doris Version:.*/Doris Version: *${current_version}*/ | |
| }" >release_note.md | |
| fi | |
| gh release edit -F release_note.md "${tag_name}" | |
| echo "should_release=${should_release}" >> $GITHUB_OUTPUT | |
| - name: Download Source and Upload | |
| if: steps.check_diff.outputs.should_release == 'true' | |
| run: | | |
| tag_name='automation-4.0' | |
| cd thirdparty | |
| # Pre-download packages whose default mirror is blocked or gone on GH Actions runners: | |
| # lzo: fossies.org returns 410 Gone | |
| # libuuid: nchc.dl.sourceforge.net (TW mirror) is blocked from GH Actions | |
| mkdir -p src | |
| curl -fL "https://www.oberhumer.com/opensource/lzo/download/lzo-2.10.tar.gz" \ | |
| -o src/lzo-2.10.tar.gz | |
| curl -fL "https://downloads.sourceforge.net/project/libuuid/libuuid-1.0.3.tar.gz" \ | |
| -o src/libuuid-1.0.3.tar.gz | |
| sed '/# unpacking thirdpart archives/,$d' download-thirdparty.sh | bash - | |
| tar -zcvf doris-thirdparty-source.tgz src | |
| gh release upload --clobber "${tag_name}" doris-thirdparty-source.tgz | |
| build: | |
| name: Build | |
| needs: prerelease | |
| if: needs.prerelease.outputs.should_release == 'true' || (github.event_name == 'workflow_dispatch' && github.event.inputs.force_build == 'true') | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| config: | |
| - name: macOS-arm64 | |
| os: macos-14 | |
| packages: >- | |
| 'm4' | |
| 'automake' | |
| 'autoconf' | |
| 'libtool' | |
| 'pkg-config' | |
| 'texinfo' | |
| 'coreutils' | |
| 'gnu-getopt' | |
| 'python@3' | |
| 'ninja' | |
| 'ccache' | |
| 'bison' | |
| 'byacc' | |
| 'gettext' | |
| 'wget' | |
| 'pcre' | |
| 'openjdk@11' | |
| 'maven' | |
| 'node' | |
| 'llvm@20' | |
| - name: Linux | |
| os: ubuntu-22.04 | |
| packages: >- | |
| 'build-essential' | |
| 'automake' | |
| 'autoconf' | |
| 'libtool-bin' | |
| 'pkg-config' | |
| 'cmake=3.22.1-1ubuntu1.22.04.2' | |
| 'ninja-build' | |
| 'ccache' | |
| 'python-is-python3' | |
| 'bison' | |
| 'byacc' | |
| 'flex' | |
| 'binutils-dev' | |
| 'libiberty-dev' | |
| 'curl' | |
| 'git' | |
| 'zip' | |
| 'unzip' | |
| 'autopoint' | |
| 'openjdk-8-jdk' | |
| 'openjdk-8-jdk-headless' | |
| 'maven' | |
| runs-on: ${{ matrix.config.os }} | |
| env: | |
| GH_REPO: ${{ github.repository }} | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| steps: | |
| - name: Checkout easimon/maximize-build-space | |
| if: ${{ matrix.config.name == 'Linux' }} | |
| run: | | |
| git clone -b v7 https://github.com/easimon/maximize-build-space | |
| #- name: Maximize build space | |
| # if: ${{ matrix.config.name == 'Linux' }} | |
| # uses: ./maximize-build-space | |
| # with: | |
| # root-reserve-mb: 4096 | |
| # swap-size-mb: 8192 | |
| # remove-dotnet: 'true' | |
| # remove-android: 'true' | |
| # remove-haskell: 'true' | |
| # remove-codeql: 'true' | |
| # remove-docker-images: 'true' | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: 'apache/doris' | |
| ref: 'branch-4.0' | |
| - name: Download | |
| run: | | |
| tag_name='automation-4.0' | |
| cd thirdparty | |
| curl -L "https://github.com/${{ github.repository }}/releases/download/${tag_name}/doris-thirdparty-source.tgz" \ | |
| -o doris-thirdparty-source.tgz | |
| tar -zxvf doris-thirdparty-source.tgz | |
| - name: Prepare for ${{ matrix.config.os }} | |
| run: | | |
| if [[ "${{ matrix.config.name }}" =~ macOS-* ]]; then | |
| # Install packages except cmake | |
| brew install ${{ matrix.config.packages }} || true | |
| # Install specific version of cmake | |
| brew unlink cmake || true | |
| wget https://github.com/Kitware/CMake/releases/download/v3.22.1/cmake-3.22.1-macos-universal.tar.gz | |
| tar -xzf cmake-3.22.1-macos-universal.tar.gz | |
| sudo cp -r cmake-3.22.1-macos-universal/CMake.app/Contents/* /usr/local/ | |
| cmake --version | |
| else | |
| export DEFAULT_DIR='/opt/doris' | |
| export PATH="${DEFAULT_DIR}/ldb-toolchain/bin:${PATH}" | |
| sudo apt update | |
| sudo apt-cache policy cmake | |
| sudo DEBIAN_FRONTEND=noninteractive apt install --yes ${{ matrix.config.packages }} | |
| mkdir -p "${DEFAULT_DIR}" | |
| wget https://github.com/amosbird/ldb_toolchain_gen/releases/download/v0.25/ldb_toolchain_gen.sh \ | |
| -q -O /tmp/ldb_toolchain_gen.sh | |
| bash /tmp/ldb_toolchain_gen.sh "${DEFAULT_DIR}/ldb-toolchain" | |
| fi | |
| - name: Build and Upload | |
| run: | | |
| tag_name='automation-4.0' | |
| if [[ "${{ matrix.config.name }}" == 'Linux' ]]; then | |
| export DEFAULT_DIR='/opt/doris' | |
| export PATH="${DEFAULT_DIR}/ldb-toolchain/bin:${PATH}" | |
| export PATH="$(find /usr/lib/jvm/java-8-openjdk* -maxdepth 1 -type d -name 'bin'):${PATH}" | |
| export JAVA_HOME="$(find /usr/lib/jvm/java-8-openjdk* -maxdepth 0)" | |
| export DORIS_TOOLCHAIN=clang | |
| else | |
| export MACOSX_DEPLOYMENT_TARGET=12.0 | |
| fi | |
| export CMAKE_POLICY_VERSION_MINIMUM=3.10 | |
| export CUSTOM_CMAKE="/usr/local/bin/cmake" | |
| cd thirdparty | |
| # Fix packages that may be empty if the source tarball was built before the mirror fixes: | |
| # lzo: fossies.org returns 410 Gone | |
| # libuuid: nchc.dl.sourceforge.net (TW mirror) blocked from GH Actions | |
| if [[ ! -s src/lzo-2.10.tar.gz ]]; then | |
| curl -fL "https://www.oberhumer.com/opensource/lzo/download/lzo-2.10.tar.gz" \ | |
| -o src/lzo-2.10.tar.gz | |
| fi | |
| if [[ ! -s src/libuuid-1.0.3.tar.gz ]]; then | |
| curl -fL "https://downloads.sourceforge.net/project/libuuid/libuuid-1.0.3.tar.gz" \ | |
| -o src/libuuid-1.0.3.tar.gz | |
| fi | |
| ./build-thirdparty.sh -j "$(nproc)" | |
| kernel="$(uname -s | awk '{print tolower($0)}')" | |
| arch="$(uname -m)" | |
| rm -rf "doris-thirdparty-prebuilt-${kernel}-${arch}.tar.xz" | |
| tar -cf - installed | xz -z -T0 - >"doris-thirdparty-prebuilt-${kernel}-${arch}.tar.xz" | |
| # Delete existing asset first to avoid gh CLI --clobber race-condition bug | |
| # (gh release upload --clobber sometimes returns "release not found" exit 1 | |
| # when deleting an existing asset, even though the upload itself succeeds) | |
| gh release delete-asset "${tag_name}" "doris-thirdparty-prebuilt-${kernel}-${arch}.tar.xz" 2>/dev/null || true | |
| gh release upload "${tag_name}" "doris-thirdparty-prebuilt-${kernel}-${arch}.tar.xz" | |
| update-docker: | |
| name: Update Docker Image | |
| needs: [prerelease, build] | |
| if: | | |
| always() && ( | |
| (needs.prerelease.outputs.should_release == 'true' && needs.build.result == 'success') || | |
| (github.event_name == 'workflow_dispatch' && needs.build.result != 'failure') | |
| ) | |
| runs-on: ubuntu-latest | |
| env: | |
| GH_REPO: ${{ github.repository }} | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| steps: | |
| - name: Checkout docker/setup-buildx-action | |
| run: | | |
| rm -rf ./.github/actions/setup-buildx-action | |
| git clone https://github.com/docker/setup-buildx-action .github/actions/setup-buildx-action | |
| pushd .github/actions/setup-buildx-action &>/dev/null | |
| git checkout 8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3 | |
| popd &>/dev/null | |
| - name: Checkout docker/login-action | |
| run: | | |
| rm -rf ./.github/actions/login-action | |
| git clone https://github.com/docker/login-action .github/actions/login-action | |
| pushd .github/actions/login-action &>/dev/null | |
| git checkout c94ce9fb468520275223c153574b00df6fe4bcc9 # v3 | |
| popd &>/dev/null | |
| - name: Checkout docker/build-push-action | |
| run: | | |
| rm -rf ./.github/actions/build-push-action | |
| git clone https://github.com/docker/build-push-action .github/actions/build-push-action | |
| pushd .github/actions/build-push-action &>/dev/null | |
| git checkout ca052bb54ab0790a636c9b5f226502c73d547a25 # v5 | |
| popd &>/dev/null | |
| - name: Set up Docker Buildx | |
| uses: ./.github/actions/setup-buildx-action | |
| - name: Login to Docker Hub | |
| uses: ./.github/actions/login-action | |
| with: | |
| username: ${{ secrets.DOCKERHUB_USERNAME }} | |
| password: ${{ secrets.DOCKERHUB_TOKEN }} | |
| - name: Prepare Dockerfiles | |
| run: | | |
| # Pull the upstream Dockerfile from apache/doris branch-4.0 to stay in sync with | |
| # any toolchain / dependency changes they make. | |
| curl -fsSL https://raw.githubusercontent.com/apache/doris/branch-4.0/docker/compilation/Dockerfile \ | |
| -o Dockerfile.upstream | |
| python3 - << 'PYEOF' | |
| import sys, re | |
| with open('Dockerfile.upstream') as f: | |
| content = f.read() | |
| # Patch 1: fix epel metalink. | |
| # The upstream RUN line installs epel-release then immediately runs yum | |
| # install/clean/makecache, but epel.repo's metalink may be unreliable. | |
| # Insert a sed fix between epel-release install and the next yum install. | |
| old_epel = 'yum install epel-release -y && yum install https://packages.endpointdev.com' | |
| new_epel = ('yum install epel-release -y \\\n' | |
| ' && sed -i \\\n' | |
| ' -e \'s/^metalink=/#metalink=/\' \\\n' | |
| ' -e \'s|^#baseurl=http://download.fedoraproject.org/pub/epel/7|baseurl=https://mirrors.aliyun.com/epel/7|\' \\\n' | |
| ' /etc/yum.repos.d/epel*.repo \\\n' | |
| ' && yum install https://packages.endpointdev.com') | |
| assert old_epel in content, f"Patch 1 failed: target string not found in upstream Dockerfile" | |
| patched = content.replace(old_epel, new_epel, 1) | |
| assert patched != content, "Patch 1 was a no-op: upstream Dockerfile may have changed" | |
| # Patch 2: replace "clone & build thirdparty" block with downloading | |
| # our prebuilt artifact. The block starts with "# clone lastest source | |
| # code" comment and ends with "rm -rf ${DEFAULT_DIR}/doris". | |
| prebuilt_block = ( | |
| '# Download prebuilt thirdparty from GitHub Release (built by doris-thirdparty automation)\n' | |
| 'ARG GITHUB_REPOSITORY\n' | |
| 'RUN mkdir -p /var/local/thirdparty \\\n' | |
| ' && wget -q "https://github.com/${GITHUB_REPOSITORY}/releases/download/automation-4.0/doris-thirdparty-prebuilt-linux-x86_64.tar.xz" \\\n' | |
| ' -O /tmp/prebuilt.tar.xz \\\n' | |
| ' && tar -xf /tmp/prebuilt.tar.xz -C /var/local/thirdparty \\\n' | |
| ' && rm /tmp/prebuilt.tar.xz\n' | |
| ) | |
| patched_2 = re.sub( | |
| r'# clone lastest source code.*?rm -rf \$\{DEFAULT_DIR\}/doris\n', | |
| prebuilt_block, | |
| patched, flags=re.DOTALL | |
| ) | |
| assert patched_2 != patched, "Patch 2 was a no-op: 'clone lastest source code' block not found in upstream Dockerfile" | |
| patched = patched_2 | |
| # Write normal image Dockerfile | |
| with open('Dockerfile.patched', 'w') as f: | |
| f.write(patched) | |
| # Patch 3 (no-avx2): add USE_AVX2=0 to the ENV block in builder stage. | |
| # The ENV block ends with PATH=... just before "# install ccache". | |
| old_avx2 = 'PATH="/var/local/ldb-toolchain/bin/:$PATH"\n # USE_AVX2=0' | |
| new_avx2 = 'PATH="/var/local/ldb-toolchain/bin/:$PATH" \\\n USE_AVX2=0' | |
| assert old_avx2 in patched, "Patch 3 failed: '# USE_AVX2=0' comment not found; upstream Dockerfile may have changed" | |
| noavx2 = patched.replace(old_avx2, new_avx2) | |
| assert noavx2 != patched, "Patch 3 was a no-op: no-avx2 Dockerfile is identical to normal Dockerfile" | |
| with open('Dockerfile.patched-noavx2', 'w') as f: | |
| f.write(noavx2) | |
| print("=== normal: check ===") | |
| for line in open('Dockerfile.patched'): | |
| if any(k in line for k in ['COPY doris', 'build-thirdparty', 'ARG GITHUB', 'prebuilt', 'USE_AVX2']): | |
| print(line, end='') | |
| print("=== noavx2: check ===") | |
| for line in open('Dockerfile.patched-noavx2'): | |
| if any(k in line for k in ['COPY doris', 'build-thirdparty', 'ARG GITHUB', 'prebuilt', 'USE_AVX2']): | |
| print(line, end='') | |
| PYEOF | |
| - name: Build and Push Docker Image (normal) | |
| uses: ./.github/actions/build-push-action | |
| with: | |
| context: . | |
| file: Dockerfile.patched | |
| build-args: | | |
| GITHUB_REPOSITORY=${{ github.repository }} | |
| push: true | |
| tags: ${{ vars.DOCKER_TAGS_4_0 || 'apache/doris:build-env-ldb-toolchain-4.0-latest' }} | |
| platforms: linux/amd64 | |
| - name: Build and Push Docker Image (no-avx2) | |
| uses: ./.github/actions/build-push-action | |
| with: | |
| context: . | |
| file: Dockerfile.patched-noavx2 | |
| build-args: | | |
| GITHUB_REPOSITORY=${{ github.repository }} | |
| push: true | |
| tags: ${{ vars.DOCKER_TAGS_NOAVX2_4_0 || 'apache/doris:build-env-ldb-toolchain-4.0-no-avx2-latest' }} | |
| platforms: linux/amd64 | |
| success: | |
| name: Success | |
| needs: [prerelease, build, update-docker] | |
| if: needs.prerelease.outputs.should_release == 'true' || (github.event_name == 'workflow_dispatch' && github.event.inputs.force_build == 'true') | |
| runs-on: ubuntu-latest | |
| env: | |
| GH_REPO: ${{ github.repository }} | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| permissions: | |
| contents: write | |
| steps: | |
| - name: Update Checksums | |
| run: | | |
| tag_name='automation-4.0' | |
| gh release download "${tag_name}" | |
| content="$(gh release view "${tag_name}" | sed -n '/Update Time:/,/Doris Version:/p')" | |
| echo -ne "${content}\nStatus: *SUCCESS*\n\n## SHA256 Checksums\n\`\`\`\n$(sha256sum *)\n\`\`\`" >release_note.md | |
| gh release edit -F release_note.md "${tag_name}" | |
| failure: | |
| name: Failure | |
| needs: [build, update-docker] | |
| if: always() && failure() | |
| runs-on: ubuntu-latest | |
| env: | |
| GH_REPO: ${{ github.repository }} | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| permissions: | |
| contents: write | |
| steps: | |
| - name: Update Checksums | |
| run: | | |
| tag_name='automation-4.0' | |
| gh release download "${tag_name}" | |
| echo -ne "Status: *FAILURE*\n\n## SHA256 Checksums\n\`\`\`\n$(sha256sum *)\n\`\`\`" >release_note.md | |
| gh release edit -F release_note.md "${tag_name}" |