Fix build issues reported by weekly jobs

.github/workflows/velox_weekly.yml

@@ -35,87 +35,46 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  build-on-centos-7:
-    strategy:
-      fail-fast: false
-    if: ${{ startsWith(github.repository, 'apache/') }}
-    runs-on: ubuntu-22.04
-    container: centos:7
-    steps:
-      - name: Build Gluten native libraries and package
-        run: |
-          sed -i -e 's|mirrorlist=|#mirrorlist=|g' /etc/yum.repos.d/CentOS-* || true
-          sed -i -e 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-* || true
-          yum update -y
-          yum install -y epel-release sudo dnf
-          yum install -y centos-release-scl
-          rm /etc/yum.repos.d/CentOS-SCLo-scl.repo -f
-          sed -i \
-          -e 's/^mirrorlist/#mirrorlist/' \
-          -e 's/^#baseurl/baseurl/' \
-          -e 's/mirror\.centos\.org/vault.centos.org/' \
-          /etc/yum.repos.d/CentOS-SCLo-scl-rh.repo
-          yum install -y devtoolset-11
-          source /opt/rh/devtoolset-11/enable
-          yum install -y java-1.8.0-openjdk-devel patch wget git perl python3-pip
-          export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk && \
-          export PATH=$JAVA_HOME/bin:$PATH
-
-          # action/checkout does not work centos7 anymore, so we clone the branch instead.
-          git clone -b main --depth=1 https://github.com/apache/gluten.git && cd gluten/
-          if [ ${{ github.event_name }} = "pull_request" ]; then
-            git fetch origin ${{ github.ref }}:pr_branch && git checkout pr_branch
-          fi
-          ./dev/package.sh --spark_version=3.5
 
-  build-on-centos:
+  weekly-build-on-centos:
     strategy:
       fail-fast: false
       matrix:
         os: [ "centos:8", "quay.io/centos/centos:stream9" ]
-    if: ${{ startsWith(github.repository, 'apache/') }}
+
     runs-on: ubuntu-22.04
     container: ${{ matrix.os }}
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
-      - name: Update mirror list
-        run: |
-          sed -i -e "s|mirrorlist=|#mirrorlist=|g" /etc/yum.repos.d/CentOS-* || true
-          sed -i -e "s|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g" /etc/yum.repos.d/CentOS-* || true
       - name: build
         run: |
-          yum update -y
-          yum install -y epel-release sudo dnf
-          if [ "${{ matrix.os }}" = "centos:7" ]; then
-            yum install -y centos-release-scl
-            rm /etc/yum.repos.d/CentOS-SCLo-scl.repo -f
-            sed -i \
-            -e 's/^mirrorlist/#mirrorlist/' \
-            -e 's/^#baseurl/baseurl/' \
-            -e 's/mirror\.centos\.org/vault.centos.org/' \
-            /etc/yum.repos.d/CentOS-SCLo-scl-rh.repo
-            yum install -y devtoolset-11
-            source /opt/rh/devtoolset-11/enable
-          elif [ "${{ matrix.os }}" = "quay.io/centos/centos:stream9" ]; then
-            dnf install -y --setopt=install_weak_deps=False gcc-toolset-12
-            source /opt/rh/gcc-toolset-12/enable || exit 1
-          else
-            dnf install -y --setopt=install_weak_deps=False gcc-toolset-11
-            source /opt/rh/gcc-toolset-11/enable || exit 1
+          if [ "${{ matrix.os }}" = "centos:8" ]; then
+            sed -i -e "s|mirrorlist=|#mirrorlist=|g" /etc/yum.repos.d/CentOS-* || true
+            sed -i -e "s|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g" /etc/yum.repos.d/CentOS-* || true
+            dnf update -y
+            dnf install -y centos-release-stream
+            sed -i -e "s|mirrorlist=|#mirrorlist=|g" /etc/yum.repos.d/CentOS-Stream-* || true
+            sed -i -e "s|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g" /etc/yum.repos.d/CentOS-Stream-* || true
+            dnf swap -y --allowerasing centos-{linux,stream}-repos
+            sed -i -e "s|mirrorlist=|#mirrorlist=|g" /etc/yum.repos.d/CentOS-Stream-* || true
+            sed -i -e "s|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g" /etc/yum.repos.d/CentOS-Stream-* || true
           fi
+          dnf install -y epel-release sudo dnf
+          dnf install -y --setopt=install_weak_deps=False gcc-toolset-12
+          source /opt/rh/gcc-toolset-12/enable || exit 1
           yum install -y java-1.8.0-openjdk-devel patch wget git perl
           export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk && \
           export PATH=$JAVA_HOME/bin:$PATH
-
+          export VELOX_BUILD_SHARED=ON
           cd $GITHUB_WORKSPACE/ && ./dev/package.sh --spark_version=3.5
 
-  build-on-ubuntu:
+  weekly-build-on-ubuntu:
     strategy:
       fail-fast: false
       matrix:
-        os: [ "ubuntu:22.04" ] # skip "ubuntu:20.04" due to not enough disk space
-    if: ${{ startsWith(github.repository, 'apache/') }}
+        os: [ "ubuntu:22.04" ]
+
     runs-on: ubuntu-22.04
     container: ${{ matrix.os }}
     steps:
@@ -125,35 +84,40 @@ jobs:
           # sed -i 's|http://archive|http://us.archive|g' /etc/apt/sources.list
           # To avoid the prompt for region selection during installing tzdata.
           export DEBIAN_FRONTEND=noninteractive
-          apt-get update && apt-get install -y sudo maven wget git curl
-          if [ "${{ matrix.os }}" = "ubuntu:20.04" ]; then
-            sudo apt install -y software-properties-common
-            sudo add-apt-repository ppa:ubuntu-toolchain-r/test
-            sudo apt update && sudo apt install -y gcc-11 g++-11 build-essential cmake
-            export CC=/usr/bin/gcc-11
-            export CXX=/usr/bin/g++-11
-            wget https://github.com/Cyan4973/xxHash/archive/refs/tags/v0.8.1.tar.gz
-            tar xf v0.8.1.tar.gz
-            cd xxHash-0.8.1/
-            make && make install
-            cd /tmp
-            wget https://github.com/google/googletest/archive/refs/tags/v1.13.0.tar.gz
-            tar xf v1.13.0.tar.gz && cd googletest-1.13.0/
-            export CXXFLAGS=-fPIC
-            cmake . && make install -j
-            apt autoremove -y
-            rm -rf /tmp/*
-          fi
-          sudo apt-get install -y openjdk-8-jdk python3-pip cmake
+          export INSTALL_PREFIX=/usr/
+          apt-get update && apt-get install -y sudo maven wget git curl gcc-11 g++-11 build-essential cmake openjdk-8-jdk python3-pip
+          pip3 install cmake==3.31.4
+          cd /tmp
+          wget https://github.com/gflags/gflags/archive/v2.2.2.tar.gz -O gflags.tar.gz
+          tar -xzf gflags.tar.gz && cd gflags-2.2.2
+          cmake -B build \
+            -DCMAKE_POSITION_INDEPENDENT_CODE=ON \
+            -DBUILD_SHARED_LIBS=ON \
+            -DBUILD_STATIC_LIBS=ON \
+            -DBUILD_gflags_LIB=ON \
+            -DCMAKE_INSTALL_PREFIX=/usr/local \
+            -DCMAKE_BUILD_TYPE=Release
+          cmake --build build -j4 --target install
+          wget https://github.com/Cyan4973/xxHash/archive/refs/tags/v0.8.1.tar.gz
+          tar xf v0.8.1.tar.gz
+          cd xxHash-0.8.1/
+          make && make install
+          cd /tmp
+          wget https://github.com/google/googletest/archive/refs/tags/v1.13.0.tar.gz
+          tar xf v1.13.0.tar.gz && cd googletest-1.13.0/
+          export CXXFLAGS=-fPIC
+          cmake . && make install -j
+          apt autoremove -y
           export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64
+          export VELOX_BUILD_SHARED=ON
           cd $GITHUB_WORKSPACE/ && ./dev/package.sh --spark_version=3.5
 
-  build-on-openeuler:
+  weekly-build-on-openeuler:
     strategy:
       fail-fast: false
       matrix:
         os: [ "openeuler/openeuler:24.03" ]
-    if: ${{ startsWith(github.repository, 'apache/') }}
+
     runs-on: ubuntu-24.04-arm
     container: ${{ matrix.os }}
     steps:
@@ -177,12 +141,12 @@ jobs:
           $TPCH_TEST --extra-conf=spark.gluten.loadLibFromJar=true \
             --extra-conf=spark.gluten.loadLibOS=openEuler --extra-conf=spark.gluten.loadLibOSVersion=24.03
 
-  build-on-openeuler-vcpkg:
+  weekly-build-on-openeuler-vcpkg:
     strategy:
       fail-fast: false
       matrix:
         os: [ "openeuler/openeuler:24.03" ]
-    if: ${{ startsWith(github.repository, 'apache/') }}
+
     runs-on: ubuntu-24.04-arm
     container: ${{ matrix.os }}
     steps:
@@ -205,10 +169,10 @@ jobs:
           cd $GITHUB_WORKSPACE/tools/gluten-it
           $TPCH_TEST
 
-  build-on-macos:
+  weekly-build-on-macos:
     strategy:
       fail-fast: false
-    if: ${{ startsWith(github.repository, 'apache/') }}
+
     runs-on: macos-15
     env:
       INSTALL_PREFIX: /tmp/deps-install

cpp/CMake/ConfigArrow.cmake

@@ -44,20 +44,32 @@ function(FIND_ARROW_LIB LIB_NAME)
     endif()
     message(STATUS "Found Arrow library: ${ARROW_LIB_${LIB_NAME}}")
 
-    # Get the parent-parent directory of the lib file. For example:
-    #
-    # * ${ARROW_LIB_${LIB_NAME}}: /usr/local/lib/libarrow.a
-    # * ${ARROW_LIB_INCLUDE_DIR}: /usr/local
-    #
-    # Then we can get our include directory: /usr/local/include
-    get_filename_component(ARROW_LIB_INCLUDE_DIR "${ARROW_LIB_${LIB_NAME}}"
-                           PATH)
-    get_filename_component(ARROW_LIB_INCLUDE_DIR "${ARROW_LIB_INCLUDE_DIR}"
-                           PATH)
+    # Walk up from the library directory until we find a parent that contains an
+    # "include/arrow/api.h" anchor file.  This handles both conventional layouts
+    # (/usr/local/lib/libarrow.a → /usr/local/include) and multi-arch ones
+    # (/usr/lib/x86_64-linux-gnu/libarrow.a → /usr/include) without hard-coding
+    # the number of directory levels to strip.
+    get_filename_component(_arrow_search_dir "${ARROW_LIB_${LIB_NAME}}" PATH)
+    set(ARROW_LIB_INCLUDE_DIR "")
+    foreach(_level RANGE 5)
+      if(EXISTS "${_arrow_search_dir}/include/arrow/api.h")
+        set(ARROW_LIB_INCLUDE_DIR "${_arrow_search_dir}/include")
+        break()
+      endif()
+      get_filename_component(_arrow_search_dir "${_arrow_search_dir}" PATH)
+    endforeach()
+    if(NOT ARROW_LIB_INCLUDE_DIR)
+      message(
+        FATAL_ERROR
+          "Could not locate Arrow headers near ${ARROW_LIB_${LIB_NAME}}. "
+          "Set ARROW_HOME to the Arrow installation prefix (the directory "
+          "that contains lib/ and include/).")
+    endif()
+    message(STATUS "Found Arrow include: ${ARROW_LIB_INCLUDE_DIR}")
 
     set_target_properties(
       Arrow::${LIB_NAME}
       PROPERTIES IMPORTED_LOCATION ${ARROW_LIB_${LIB_NAME}}
-                 INTERFACE_INCLUDE_DIRECTORIES ${ARROW_LIB_INCLUDE_DIR}/include)
+                 INTERFACE_INCLUDE_DIRECTORIES ${ARROW_LIB_INCLUDE_DIR})
   endif()
 endfunction()

dev/build-thirdparty.sh

@@ -32,13 +32,16 @@ function process_setup_ubuntu_2004 {
 
 function process_setup_ubuntu_2204 {
   cp /usr/lib/${ARCH}-linux-gnu/{libre2.so.9,libdouble-conversion.so.3,libglog.so.0,libgflags.so.2.2,libevent-2.1.so.7,libsnappy.so.1,libunwind.so.8,libcurl.so.4,libxml2.so.2,libicui18n.so.70,libicuuc.so.70,libnghttp2.so.14,libldap-2.5.so.0,liblber-2.5.so.0,librtmp.so.1,libsasl2.so.2,libssh.so.4,libicudata.so.70} $THIRDPARTY_LIB/
-  cp /usr/local/lib/{libboost_context.so.1.84.0,libboost_regex.so.1.84.0} $THIRDPARTY_LIB/
+  find /usr/local/lib /usr/local/lib64 -name "libboost_context.so.1.84.0" -exec cp {} $THIRDPARTY_LIB/ \; 2>/dev/null
+  find /usr/local/lib /usr/local/lib64 -name "libboost_regex.so.1.84.0" -exec cp {} $THIRDPARTY_LIB/ \; 2>/dev/null
 }
 
 function process_setup_centos_9 {
   cp /lib64/{libre2.so.9,libdouble-conversion.so.3,libevent-2.1.so.7,libdwarf.so.0,libicudata.so.67,libicui18n.so.67,libicuuc.so.67,libsodium.so.23} $THIRDPARTY_LIB/
   cp /usr/local/lib/{libboost_context.so.1.84.0,libboost_filesystem.so.1.84.0,libboost_program_options.so.1.84.0,libboost_regex.so.1.84.0,libboost_system.so.1.84.0,libboost_thread.so.1.84.0,libboost_atomic.so.1.84.0} $THIRDPARTY_LIB/
-  cp /usr/local/lib64/{libgflags.so.2.2,libglog.so.1,libgeos.so.3.10.7} $THIRDPARTY_LIB/
+  cp /usr/local/lib/libgflags.so.2.2 $THIRDPARTY_LIB/
+  find /usr/local/lib /usr/local/lib64 -name "libglog.so.1" -exec cp {} $THIRDPARTY_LIB/ \; 2>/dev/null
+  find /usr/local/lib /usr/local/lib64 -name "libgeos.so.3.10.7" -exec cp {} $THIRDPARTY_LIB/ \; 2>/dev/null
 }
 
 function process_setup_centos_8 {

dev/vcpkg/setup-build-depends.sh

@@ -202,7 +202,7 @@ install_openeuler_24.03() {
         gcc g++ cmake ninja-build perl-IPC-Cmd autoconf autoconf-archive automake libtool \
         java-1.8.0-openjdk java-1.8.0-openjdk-devel python3-devel python3-pip libstdc++-static
 
-    pip install cmake==3.28.3
+    pip install cmake==3.31.4
 
     install_maven_from_source
 }

ep/build-velox/src/get-velox.sh

@@ -65,6 +65,7 @@ if [ "$VELOX_HOME" == "" ]; then
 fi
 
 function process_setup_ubuntu {
+  sed -i "s|run_and_time install_arrow||g" scripts/setup-centos9.sh
   echo "Using setup script from Velox"
 }
 

ep/build-velox/src/setup-centos8.sh

@@ -39,8 +39,8 @@ export CPPFLAGS=$CFLAGS  # Used by LZO.
 CMAKE_BUILD_TYPE="${BUILD_TYPE:-Release}"
 BUILD_DUCKDB="${BUILD_DUCKDB:-true}"
 BUILD_GEOS="${BUILD_GEOS:-true}"
-export CC=/opt/rh/gcc-toolset-11/root/bin/gcc
-export CXX=/opt/rh/gcc-toolset-11/root/bin/g++
+export CC=/opt/rh/gcc-toolset-12/root/bin/gcc
+export CXX=/opt/rh/gcc-toolset-12/root/bin/g++
 DEPENDENCY_DIR=${DEPENDENCY_DIR:-$(pwd)/deps-download}
 
 FB_OS_VERSION="v2026.01.05.00"
@@ -57,9 +57,10 @@ function dnf_install {
 function install_build_prerequisites {
   dnf update -y
   dnf_install epel-release dnf-plugins-core # For ccache, ninja
-  dnf config-manager --set-enabled powertools
+  dnf config-manager --set-enabled powertools || true # For centos8, powertools is needed for some dependencies.
+  dnf config-manager --set-enabled Stream-PowerTools || true
   dnf update -y
-  dnf_install ninja-build curl ccache gcc-toolset-11 git wget which expat-devel gettext-devel
+  dnf_install ninja-build curl ccache gcc-toolset-12 git wget which expat-devel gettext-devel
   dnf_install yasm
   dnf_install autoconf automake python39 python39-devel python39-pip libtool
   pip3.9 install cmake==3.28.3
@@ -172,7 +173,10 @@ function install_wangle {
 
 function install_fbthrift {
   wget_and_untar https://github.com/facebook/fbthrift/archive/refs/tags/${FB_OS_VERSION}.tar.gz fbthrift
-  cmake_install_dir fbthrift -Denable_tests=OFF -DBUILD_SHARED_LIBS=OFF -DBUILD_TESTS=OFF -DBUILD_SHARED_LIBS=OFF
+  cd ${DEPENDENCY_DIR}/fbthrift
+  git apply ${VELOX_HOME:-${DEPENDENCY_DIR}/../}/CMake/resolve_dependency_modules/fbthrift/compactv1-protocol-refiller.patch 2>/dev/null || true
+  cd ${DEPENDENCY_DIR}
+  cmake_install_dir fbthrift -Denable_tests=OFF -DBUILD_TESTS=OFF -DBUILD_SHARED_LIBS=OFF
 }
 
 function install_mvfst {
@@ -228,7 +232,7 @@ function install_velox_deps {
 (
   if [[ $# -ne 0 ]]; then
     # Activate gcc11; enable errors on unset variables afterwards.
-    source /opt/rh/gcc-toolset-11/enable || exit 1
+    source /opt/rh/gcc-toolset-12/enable || exit 1
     set -u
     for cmd in "$@"; do
       run_and_time "${cmd}"
@@ -242,7 +246,7 @@ function install_velox_deps {
       echo "Skipping installation of build dependencies since INSTALL_PREREQUISITES is not set"
     fi
     # Activate gcc11; enable errors on unset variables afterwards.
-    source /opt/rh/gcc-toolset-11/enable || exit 1
+    source /opt/rh/gcc-toolset-12/enable || exit 1
     set -u
     install_velox_deps
     echo "All dependencies for Velox installed!"