fix weekly build issue

zhouyuan · philo-he · zhouyuan · commit d454f1402c16 · 2026-07-17T07:58:13.000+01:00
Signed-off-by: Yuan &lt;yuanzhou@apache.org&gt;

Co-authored-by: philo-he &lt;philo@apache.org&gt;
diff --git a/.github/workflows/velox_weekly.yml b/.github/workflows/velox_weekly.yml
@@ -35,87 +35,46 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  build-on-centos-7:
-    strategy:
-      fail-fast: false
-    if: ${{ startsWith(github.repository, 'apache/') }}
-    runs-on: ubuntu-22.04
-    container: centos:7
-    steps:
-      - name: Build Gluten native libraries and package
-        run: |
-          sed -i -e 's|mirrorlist=|#mirrorlist=|g' /etc/yum.repos.d/CentOS-* || true
-          sed -i -e 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-* || true
-          yum update -y
-          yum install -y epel-release sudo dnf
-          yum install -y centos-release-scl
-          rm /etc/yum.repos.d/CentOS-SCLo-scl.repo -f
-          sed -i \
-          -e 's/^mirrorlist/#mirrorlist/' \
-          -e 's/^#baseurl/baseurl/' \
-          -e 's/mirror\.centos\.org/vault.centos.org/' \
-          /etc/yum.repos.d/CentOS-SCLo-scl-rh.repo
-          yum install -y devtoolset-11
-          source /opt/rh/devtoolset-11/enable
-          yum install -y java-1.8.0-openjdk-devel patch wget git perl python3-pip
-          export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk && \
-          export PATH=$JAVA_HOME/bin:$PATH
-          
-          # action/checkout does not work centos7 anymore, so we clone the branch instead.
-          git clone -b main --depth=1 https://github.com/apache/gluten.git && cd gluten/
-          if [ ${{ github.event_name }} = "pull_request" ]; then
-            git fetch origin ${{ github.ref }}:pr_branch && git checkout pr_branch
-          fi
-          ./dev/package.sh --spark_version=3.5
 
-  build-on-centos:
+  weekly-build-on-centos:
     strategy:
       fail-fast: false
       matrix:
         os: [ "centos:8", "quay.io/centos/centos:stream9" ]
-    if: ${{ startsWith(github.repository, 'apache/') }}
+    
     runs-on: ubuntu-22.04
     container: ${{ matrix.os }}
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
-      - name: Update mirror list
-        run: |
-          sed -i -e "s|mirrorlist=|#mirrorlist=|g" /etc/yum.repos.d/CentOS-* || true
-          sed -i -e "s|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g" /etc/yum.repos.d/CentOS-* || true
       - name: build
         run: |
-          yum update -y
-          yum install -y epel-release sudo dnf
-          if [ "${{ matrix.os }}" = "centos:7" ]; then
-            yum install -y centos-release-scl
-            rm /etc/yum.repos.d/CentOS-SCLo-scl.repo -f
-            sed -i \
-            -e 's/^mirrorlist/#mirrorlist/' \
-            -e 's/^#baseurl/baseurl/' \
-            -e 's/mirror\.centos\.org/vault.centos.org/' \
-            /etc/yum.repos.d/CentOS-SCLo-scl-rh.repo
-            yum install -y devtoolset-11
-            source /opt/rh/devtoolset-11/enable
-          elif [ "${{ matrix.os }}" = "quay.io/centos/centos:stream9" ]; then
-            dnf install -y --setopt=install_weak_deps=False gcc-toolset-12
-            source /opt/rh/gcc-toolset-12/enable || exit 1
-          else
-            dnf install -y --setopt=install_weak_deps=False gcc-toolset-11
-            source /opt/rh/gcc-toolset-11/enable || exit 1
+          if [ "${{ matrix.os }}" = "centos:8" ]; then
+            sed -i -e "s|mirrorlist=|#mirrorlist=|g" /etc/yum.repos.d/CentOS-* || true
+            sed -i -e "s|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g" /etc/yum.repos.d/CentOS-* || true
+            dnf update -y
+            dnf install -y centos-release-stream
+            sed -i -e "s|mirrorlist=|#mirrorlist=|g" /etc/yum.repos.d/CentOS-Stream-* || true
+            sed -i -e "s|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g" /etc/yum.repos.d/CentOS-Stream-* || true
+            dnf swap -y --allowerasing centos-{linux,stream}-repos
+            sed -i -e "s|mirrorlist=|#mirrorlist=|g" /etc/yum.repos.d/CentOS-Stream-* || true
+            sed -i -e "s|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g" /etc/yum.repos.d/CentOS-Stream-* || true
           fi
+          dnf install -y epel-release sudo dnf
+          dnf install -y --setopt=install_weak_deps=False gcc-toolset-12
+          source /opt/rh/gcc-toolset-12/enable || exit 1
           yum install -y java-1.8.0-openjdk-devel patch wget git perl
           export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk && \
           export PATH=$JAVA_HOME/bin:$PATH
-
+          export VELOX_BUILD_SHARED=ON
           cd $GITHUB_WORKSPACE/ && ./dev/package.sh --spark_version=3.5
 
-  build-on-ubuntu:
+  weekly-build-on-ubuntu:
     strategy:
       fail-fast: false
       matrix:
-        os: [ "ubuntu:22.04" ] # skip "ubuntu:20.04" due to not enough disk space
-    if: ${{ startsWith(github.repository, 'apache/') }}
+        os: [ "ubuntu:22.04" ]
+    
     runs-on: ubuntu-22.04
     container: ${{ matrix.os }}
     steps:
@@ -125,35 +84,42 @@ jobs:
           # sed -i 's|http://archive|http://us.archive|g' /etc/apt/sources.list
           # To avoid the prompt for region selection during installing tzdata.
           export DEBIAN_FRONTEND=noninteractive
-          apt-get update && apt-get install -y sudo maven wget git curl
-          if [ "${{ matrix.os }}" = "ubuntu:20.04" ]; then
-            sudo apt install -y software-properties-common
-            sudo add-apt-repository ppa:ubuntu-toolchain-r/test
-            sudo apt update && sudo apt install -y gcc-11 g++-11 build-essential cmake
-            export CC=/usr/bin/gcc-11
-            export CXX=/usr/bin/g++-11
-            wget https://github.com/Cyan4973/xxHash/archive/refs/tags/v0.8.1.tar.gz
-            tar xf v0.8.1.tar.gz
-            cd xxHash-0.8.1/
-            make && make install
-            cd /tmp
-            wget https://github.com/google/googletest/archive/refs/tags/v1.13.0.tar.gz
-            tar xf v1.13.0.tar.gz && cd googletest-1.13.0/
-            export CXXFLAGS=-fPIC
-            cmake . && make install -j
-            apt autoremove -y
-            rm -rf /tmp/*
-          fi
-          sudo apt-get install -y openjdk-8-jdk python3-pip cmake
+          export INSTALL_PREFIX=/usr/
+          apt-get update && apt-get install -y sudo maven wget git curl gcc-11 g++-11 build-essential cmake
+          cd /tmp
+          wget https://github.com/gflags/gflags/archive/v2.2.2.tar.gz -O gflags.tar.gz
+          tar -xzf gflags.tar.gz && cd gflags-2.2.2
+          cmake -B build \
+            -DCMAKE_POSITION_INDEPENDENT_CODE=ON \
+            -DBUILD_SHARED_LIBS=ON \
+            -DBUILD_STATIC_LIBS=ON \
+            -DBUILD_gflags_LIB=ON \
+            -DCMAKE_INSTALL_PREFIX=/usr/local \
+            -DCMAKE_BUILD_TYPE=Release
+          cmake --build build -j4 --target install
+          wget https://github.com/Cyan4973/xxHash/archive/refs/tags/v0.8.1.tar.gz
+          tar xf v0.8.1.tar.gz
+          cd xxHash-0.8.1/
+          make && make install
+          cd /tmp
+          wget https://github.com/google/googletest/archive/refs/tags/v1.13.0.tar.gz
+          tar xf v1.13.0.tar.gz && cd googletest-1.13.0/
+          export CXXFLAGS=-fPIC
+          cmake . && make install -j
+          apt autoremove -y
+          rm -rf /tmp/*
+          apt-get install -y openjdk-8-jdk python3-pip
+          pip3 install cmake==3.31.4
           export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64
+          export VELOX_BUILD_SHARED=ON
           cd $GITHUB_WORKSPACE/ && ./dev/package.sh --spark_version=3.5
 
-  build-on-openeuler:
+  weekly-build-on-openeuler:
     strategy:
       fail-fast: false
       matrix:
         os: [ "openeuler/openeuler:24.03" ]
-    if: ${{ startsWith(github.repository, 'apache/') }}
+    
     runs-on: ubuntu-24.04-arm
     container: ${{ matrix.os }}
     steps:
@@ -177,12 +143,12 @@ jobs:
           $TPCH_TEST --extra-conf=spark.gluten.loadLibFromJar=true \
             --extra-conf=spark.gluten.loadLibOS=openEuler --extra-conf=spark.gluten.loadLibOSVersion=24.03
 
-  build-on-openeuler-vcpkg:
+  weekly-build-on-openeuler-vcpkg:
     strategy:
       fail-fast: false
       matrix:
         os: [ "openeuler/openeuler:24.03" ]
-    if: ${{ startsWith(github.repository, 'apache/') }}
+    
     runs-on: ubuntu-24.04-arm
     container: ${{ matrix.os }}
     steps:
@@ -205,10 +171,10 @@ jobs:
           cd $GITHUB_WORKSPACE/tools/gluten-it
           $TPCH_TEST
 
-  build-on-macos:
+  weekly-build-on-macos:
     strategy:
       fail-fast: false
-    if: ${{ startsWith(github.repository, 'apache/') }}
+    
     runs-on: macos-15
     env:
       INSTALL_PREFIX: /tmp/deps-install
diff --git a/dev/build-thirdparty.sh b/dev/build-thirdparty.sh
@@ -38,7 +38,9 @@ function process_setup_ubuntu_2204 {
 function process_setup_centos_9 {
   cp /lib64/{libre2.so.9,libdouble-conversion.so.3,libevent-2.1.so.7,libdwarf.so.0,libicudata.so.67,libicui18n.so.67,libicuuc.so.67,libsodium.so.23} $THIRDPARTY_LIB/
   cp /usr/local/lib/{libboost_context.so.1.84.0,libboost_filesystem.so.1.84.0,libboost_program_options.so.1.84.0,libboost_regex.so.1.84.0,libboost_system.so.1.84.0,libboost_thread.so.1.84.0,libboost_atomic.so.1.84.0} $THIRDPARTY_LIB/
-  cp /usr/local/lib64/{libgflags.so.2.2,libglog.so.1,libgeos.so.3.10.7} $THIRDPARTY_LIB/
+  cp /usr/local/lib/libgflags.so.2.2 $THIRDPARTY_LIB/
+  find /usr/local/lib /usr/local/lib64 -name "libglog.so.1" -exec cp {} $THIRDPARTY_LIB/ \; 2>/dev/null
+  find /usr/local/lib /usr/local/lib64 -name "libgeos.so.3.10.7" -exec cp {} $THIRDPARTY_LIB/ \; 2>/dev/null
 }
 
 function process_setup_centos_8 {
diff --git a/dev/vcpkg/setup-build-depends.sh b/dev/vcpkg/setup-build-depends.sh
@@ -202,7 +202,7 @@ install_openeuler_24.03() {
         gcc g++ cmake ninja-build perl-IPC-Cmd autoconf autoconf-archive automake libtool \
         java-1.8.0-openjdk java-1.8.0-openjdk-devel python3-devel python3-pip libstdc++-static
 
-    pip install cmake==3.28.3
+    pip install cmake==3.31.4
 
     install_maven_from_source
 }
diff --git a/ep/build-velox/src/setup-centos8.sh b/ep/build-velox/src/setup-centos8.sh
@@ -39,8 +39,8 @@ export CPPFLAGS=$CFLAGS  # Used by LZO.
 CMAKE_BUILD_TYPE="${BUILD_TYPE:-Release}"
 BUILD_DUCKDB="${BUILD_DUCKDB:-true}"
 BUILD_GEOS="${BUILD_GEOS:-true}"
-export CC=/opt/rh/gcc-toolset-11/root/bin/gcc
-export CXX=/opt/rh/gcc-toolset-11/root/bin/g++
+export CC=/opt/rh/gcc-toolset-12/root/bin/gcc
+export CXX=/opt/rh/gcc-toolset-12/root/bin/g++
 DEPENDENCY_DIR=${DEPENDENCY_DIR:-$(pwd)/deps-download}
 
 FB_OS_VERSION="v2026.01.05.00"
@@ -57,9 +57,10 @@ function dnf_install {
 function install_build_prerequisites {
   dnf update -y
   dnf_install epel-release dnf-plugins-core # For ccache, ninja
-  dnf config-manager --set-enabled powertools
+  dnf config-manager --set-enabled powertools || true # For centos8, powertools is needed for some dependencies.
+  dnf config-manager --set-enabled Stream-PowerTools || true
   dnf update -y
-  dnf_install ninja-build curl ccache gcc-toolset-11 git wget which expat-devel gettext-devel
+  dnf_install ninja-build curl ccache gcc-toolset-12 git wget which expat-devel gettext-devel
   dnf_install yasm
   dnf_install autoconf automake python39 python39-devel python39-pip libtool
   pip3.9 install cmake==3.28.3
@@ -172,7 +173,10 @@ function install_wangle {
 
 function install_fbthrift {
   wget_and_untar https://github.com/facebook/fbthrift/archive/refs/tags/${FB_OS_VERSION}.tar.gz fbthrift
-  cmake_install_dir fbthrift -Denable_tests=OFF -DBUILD_SHARED_LIBS=OFF -DBUILD_TESTS=OFF -DBUILD_SHARED_LIBS=OFF
+  cd ${DEPENDENCY_DIR}/fbthrift
+  git apply ${VELOX_HOME:-${DEPENDENCY_DIR}/../}/CMake/resolve_dependency_modules/fbthrift/compactv1-protocol-refiller.patch 2>/dev/null || true
+  cd ${DEPENDENCY_DIR}
+  cmake_install_dir fbthrift -Denable_tests=OFF -DBUILD_TESTS=OFF -DBUILD_SHARED_LIBS=OFF
 }
 
 function install_mvfst {
@@ -228,7 +232,7 @@ function install_velox_deps {
 (
   if [[ $# -ne 0 ]]; then
     # Activate gcc11; enable errors on unset variables afterwards.
-    source /opt/rh/gcc-toolset-11/enable || exit 1
+    source /opt/rh/gcc-toolset-12/enable || exit 1
     set -u
     for cmd in "$@"; do
       run_and_time "${cmd}"
@@ -242,7 +246,7 @@ function install_velox_deps {
       echo "Skipping installation of build dependencies since INSTALL_PREREQUISITES is not set"
     fi
     # Activate gcc11; enable errors on unset variables afterwards.
-    source /opt/rh/gcc-toolset-11/enable || exit 1
+    source /opt/rh/gcc-toolset-12/enable || exit 1
     set -u
     install_velox_deps
     echo "All dependencies for Velox installed!"
diff --git a/ep/build-velox/src/setup-openeuler24.sh b/ep/build-velox/src/setup-openeuler24.sh
@@ -36,6 +36,7 @@ NPROC=$(getconf _NPROCESSORS_ONLN)
 export CFLAGS=$(get_cxx_flags $CPU_TARGET)  # Used by LZO.
 export CXXFLAGS=$CFLAGS  # Used by boost.
 export CPPFLAGS=$CFLAGS  # Used by LZO.
+EXTRA_PKG_CXXFLAGS=" -isystem ${INSTALL_PREFIX}/include"
 CMAKE_BUILD_TYPE="${BUILD_TYPE:-Release}"
 VELOX_BUILD_SHARED=${VELOX_BUILD_SHARED:-"OFF"} #Build folly and gflags shared for use in libvelox.so.
 BUILD_DUCKDB="${BUILD_DUCKDB:-true}"
@@ -44,8 +45,8 @@ VERSION=$(cat /etc/os-release | grep VERSION_ID)
 export INSTALL_PREFIX=${INSTALL_PREFIX:-"/usr/local"}
 DEPENDENCY_DIR=${DEPENDENCY_DIR:-$(pwd)/deps-download}
 
-FB_OS_VERSION="v2024.07.01.00"
-FMT_VERSION="10.1.1"
+FB_OS_VERSION="v2026.01.05.00"
+FMT_VERSION="11.2.0"
 BOOST_VERSION="boost-1.84.0"
 DUCKDB_VERSION="v0.8.1"
 GEOS_VERSION="3.10.7"
@@ -65,7 +66,7 @@ function install_build_prerequisites {
   dnf_install autoconf automake python3-devel python3-pip libtool
   dnf_install libxml2-devel libgsasl-devel libuuid-devel
 
-  pip install cmake==3.28.3
+  pip install cmake==3.31.4
 }
 
 # Install dependencies from the package managers.
@@ -110,11 +111,23 @@ function install_boost {
   )
 }
 
+function install_xxhash {
+  wget_and_untar https://github.com/Cyan4973/xxHash/archive/refs/tags/v0.8.2.tar.gz xxHash
+  cd ${DEPENDENCY_DIR}/xxHash
+  make "-j${NPROC}"
+  make install PREFIX=${INSTALL_PREFIX}
+}
+
 function install_snappy {
   wget_and_untar https://github.com/google/snappy/archive/1.1.8.tar.gz snappy
   cmake_install_dir snappy -DSNAPPY_BUILD_TESTS=OFF
 }
 
+function install_fast_float {
+  wget_and_untar https://github.com/fastfloat/fast_float/archive/refs/tags/v8.0.2.tar.gz fast_float
+  cmake_install_dir fast_float -DBUILD_TESTS=OFF
+}
+
 function install_fmt {
   wget_and_untar https://github.com/fmtlib/fmt/archive/${FMT_VERSION}.tar.gz fmt
   cmake_install_dir fmt -DFMT_TEST=OFF
@@ -133,7 +146,7 @@ function install_protobuf {
 
 function install_fizz {
   wget_and_untar https://github.com/facebookincubator/fizz/archive/refs/tags/${FB_OS_VERSION}.tar.gz fizz
-  cmake_install_dir fizz/fizz -DBUILD_TESTS=OFF
+  cmake_install_dir fizz/fizz -DBUILD_TESTS=OFF -DBUILD_EXAMPLES=OFF
 }
 
 function install_folly {
@@ -143,11 +156,14 @@ function install_folly {
 
 function install_wangle {
   wget_and_untar https://github.com/facebook/wangle/archive/refs/tags/${FB_OS_VERSION}.tar.gz wangle
-  cmake_install_dir wangle/wangle -DBUILD_TESTS=OFF
+  cmake_install_dir wangle/wangle -DBUILD_TESTS=OFF -DBUILD_EXAMPLES=OFF
 }
 
 function install_fbthrift {
   wget_and_untar https://github.com/facebook/fbthrift/archive/refs/tags/${FB_OS_VERSION}.tar.gz fbthrift
+  cd ${DEPENDENCY_DIR}/fbthrift
+  git apply ${VELOX_HOME:-${DEPENDENCY_DIR}/../}/CMake/resolve_dependency_modules/fbthrift/compactv1-protocol-refiller.patch 2>/dev/null || true
+  cd ${DEPENDENCY_DIR}
   cmake_install_dir fbthrift -Denable_tests=OFF -DBUILD_TESTS=OFF -DBUILD_SHARED_LIBS=OFF
 }
 
@@ -198,8 +214,10 @@ function install_velox_deps {
   run_and_time install_gflags
   run_and_time install_glog
   run_and_time install_lzo
+  run_and_time install_xxhash
   run_and_time install_snappy
   run_and_time install_boost
+  run_and_time install_fast_float
   run_and_time install_protobuf
   run_and_time install_fmt
   run_and_time install_folly
@@ -233,4 +251,4 @@ function install_velox_deps {
     echo "All dependencies for Velox installed!"
     dnf clean all
   fi
-)
+)

Original file line number	Diff line number	Diff line change
`@@ -202,7 +202,7 @@ install_openeuler_24.03() {`
`202`	`202`	`gcc g++ cmake ninja-build perl-IPC-Cmd autoconf autoconf-archive automake libtool \`
`203`	`203`	`java-1.8.0-openjdk java-1.8.0-openjdk-devel python3-devel python3-pip libstdc++-static`
`204`	`204`
`205`		`- pip install cmake==3.28.3`
	`205`	`+ pip install cmake==3.31.4`
`206`	`206`
`207`	`207`	`install_maven_from_source`
`208`	`208`	`}`