Skip to content

Commit f87bbfb

Browse files
zhouyuanphilo-he
zhouyuan
and
philo-he
committed
fix weekly build issue
Signed-off-by: Yuan <yuanzhou@apache.org> Co-authored-by: philo-he <philo@apache.org>
1 parent 8a60e59 commit f87bbfb

6 files changed

Lines changed: 67 additions & 78 deletions

File tree

‎.github/workflows/velox_weekly.yml‎

Lines changed: 34 additions & 68 deletions
Original file line numberDiff line numberDiff line change
@@ -35,45 +35,13 @@ concurrency:
3535
cancel-in-progress: true
3636

3737
jobs:
38-
build-on-centos-7:
39-
strategy:
40-
fail-fast: false
41-
if: ${{ startsWith(github.repository, 'apache/') }}
42-
runs-on: ubuntu-22.04
43-
container: centos:7
44-
steps:
45-
- name: Build Gluten native libraries and package
46-
run: |
47-
sed -i -e 's|mirrorlist=|#mirrorlist=|g' /etc/yum.repos.d/CentOS-* || true
48-
sed -i -e 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-* || true
49-
yum update -y
50-
yum install -y epel-release sudo dnf
51-
yum install -y centos-release-scl
52-
rm /etc/yum.repos.d/CentOS-SCLo-scl.repo -f
53-
sed -i \
54-
-e 's/^mirrorlist/#mirrorlist/' \
55-
-e 's/^#baseurl/baseurl/' \
56-
-e 's/mirror\.centos\.org/vault.centos.org/' \
57-
/etc/yum.repos.d/CentOS-SCLo-scl-rh.repo
58-
yum install -y devtoolset-11
59-
source /opt/rh/devtoolset-11/enable
60-
yum install -y java-1.8.0-openjdk-devel patch wget git perl python3-pip
61-
export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk && \
62-
export PATH=$JAVA_HOME/bin:$PATH
63-
64-
# action/checkout does not work centos7 anymore, so we clone the branch instead.
65-
git clone -b main --depth=1 https://github.com/apache/gluten.git && cd gluten/
66-
if [ ${{ github.event_name }} = "pull_request" ]; then
67-
git fetch origin ${{ github.ref }}:pr_branch && git checkout pr_branch
68-
fi
69-
./dev/package.sh --spark_version=3.5
7038

7139
build-on-centos:
7240
strategy:
7341
fail-fast: false
7442
matrix:
7543
os: [ "centos:8", "quay.io/centos/centos:stream9" ]
76-
if: ${{ startsWith(github.repository, 'apache/') }}
44+
7745
runs-on: ubuntu-22.04
7846
container: ${{ matrix.os }}
7947
steps:
@@ -87,17 +55,7 @@ jobs:
8755
run: |
8856
yum update -y
8957
yum install -y epel-release sudo dnf
90-
if [ "${{ matrix.os }}" = "centos:7" ]; then
91-
yum install -y centos-release-scl
92-
rm /etc/yum.repos.d/CentOS-SCLo-scl.repo -f
93-
sed -i \
94-
-e 's/^mirrorlist/#mirrorlist/' \
95-
-e 's/^#baseurl/baseurl/' \
96-
-e 's/mirror\.centos\.org/vault.centos.org/' \
97-
/etc/yum.repos.d/CentOS-SCLo-scl-rh.repo
98-
yum install -y devtoolset-11
99-
source /opt/rh/devtoolset-11/enable
100-
elif [ "${{ matrix.os }}" = "quay.io/centos/centos:stream9" ]; then
58+
if [ "${{ matrix.os }}" = "quay.io/centos/centos:stream9" ]; then
10159
dnf install -y --setopt=install_weak_deps=False gcc-toolset-12
10260
source /opt/rh/gcc-toolset-12/enable || exit 1
10361
else
@@ -107,15 +65,15 @@ jobs:
10765
yum install -y java-1.8.0-openjdk-devel patch wget git perl
10866
export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk && \
10967
export PATH=$JAVA_HOME/bin:$PATH
110-
68+
export VELOX_BUILD_SHARED=ON
11169
cd $GITHUB_WORKSPACE/ && ./dev/package.sh --spark_version=3.5
11270
11371
build-on-ubuntu:
11472
strategy:
11573
fail-fast: false
11674
matrix:
117-
os: [ "ubuntu:22.04" ] # skip "ubuntu:20.04" due to not enough disk space
118-
if: ${{ startsWith(github.repository, 'apache/') }}
75+
os: [ "ubuntu:22.04" ]
76+
11977
runs-on: ubuntu-22.04
12078
container: ${{ matrix.os }}
12179
steps:
@@ -126,34 +84,42 @@ jobs:
12684
# To avoid the prompt for region selection during installing tzdata.
12785
export DEBIAN_FRONTEND=noninteractive
12886
apt-get update && apt-get install -y sudo maven wget git curl
129-
if [ "${{ matrix.os }}" = "ubuntu:20.04" ]; then
130-
sudo apt install -y software-properties-common
131-
sudo add-apt-repository ppa:ubuntu-toolchain-r/test
132-
sudo apt update && sudo apt install -y gcc-11 g++-11 build-essential cmake
133-
export CC=/usr/bin/gcc-11
134-
export CXX=/usr/bin/g++-11
135-
wget https://github.com/Cyan4973/xxHash/archive/refs/tags/v0.8.1.tar.gz
136-
tar xf v0.8.1.tar.gz
137-
cd xxHash-0.8.1/
138-
make && make install
139-
cd /tmp
140-
wget https://github.com/google/googletest/archive/refs/tags/v1.13.0.tar.gz
141-
tar xf v1.13.0.tar.gz && cd googletest-1.13.0/
142-
export CXXFLAGS=-fPIC
143-
cmake . && make install -j
144-
apt autoremove -y
145-
rm -rf /tmp/*
146-
fi
87+
sudo apt update && sudo apt install -y gcc-12 g++-12 build-essential cmake
88+
export CC=/usr/bin/gcc-12
89+
export CXX=/usr/bin/g++-12
90+
cd /tmp
91+
wget https://github.com/gflags/gflags/archive/v2.2.2.tar.gz -O gflags.tar.gz
92+
tar -xzf gflags.tar.gz && cd gflags-2.2.2
93+
cmake -B build \
94+
-DCMAKE_POSITION_INDEPENDENT_CODE=ON \
95+
-DBUILD_SHARED_LIBS=ON \
96+
-DBUILD_STATIC_LIBS=ON \
97+
-DBUILD_gflags_LIB=ON \
98+
-DCMAKE_INSTALL_PREFIX=/usr/local \
99+
-DCMAKE_BUILD_TYPE=Release
100+
sudo cmake --build build -j4 --target install
101+
wget https://github.com/Cyan4973/xxHash/archive/refs/tags/v0.8.1.tar.gz
102+
tar xf v0.8.1.tar.gz
103+
cd xxHash-0.8.1/
104+
make && make install
105+
cd /tmp
106+
wget https://github.com/google/googletest/archive/refs/tags/v1.13.0.tar.gz
107+
tar xf v1.13.0.tar.gz && cd googletest-1.13.0/
108+
export CXXFLAGS=-fPIC
109+
cmake . && make install -j
110+
apt autoremove -y
111+
rm -rf /tmp/*
147112
sudo apt-get install -y openjdk-8-jdk python3-pip cmake
148113
export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64
114+
export VELOX_BUILD_SHARED=ON
149115
cd $GITHUB_WORKSPACE/ && ./dev/package.sh --spark_version=3.5
150116
151117
build-on-openeuler:
152118
strategy:
153119
fail-fast: false
154120
matrix:
155121
os: [ "openeuler/openeuler:24.03" ]
156-
if: ${{ startsWith(github.repository, 'apache/') }}
122+
157123
runs-on: ubuntu-24.04-arm
158124
container: ${{ matrix.os }}
159125
steps:
@@ -182,7 +148,7 @@ jobs:
182148
fail-fast: false
183149
matrix:
184150
os: [ "openeuler/openeuler:24.03" ]
185-
if: ${{ startsWith(github.repository, 'apache/') }}
151+
186152
runs-on: ubuntu-24.04-arm
187153
container: ${{ matrix.os }}
188154
steps:
@@ -208,7 +174,7 @@ jobs:
208174
build-on-macos:
209175
strategy:
210176
fail-fast: false
211-
if: ${{ startsWith(github.repository, 'apache/') }}
177+
212178
runs-on: macos-15
213179
env:
214180
INSTALL_PREFIX: /tmp/deps-install

‎dev/build-thirdparty.sh‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,9 @@ function process_setup_ubuntu_2204 {
3838
function process_setup_centos_9 {
3939
cp /lib64/{libre2.so.9,libdouble-conversion.so.3,libevent-2.1.so.7,libdwarf.so.0,libicudata.so.67,libicui18n.so.67,libicuuc.so.67,libsodium.so.23} $THIRDPARTY_LIB/
4040
cp /usr/local/lib/{libboost_context.so.1.84.0,libboost_filesystem.so.1.84.0,libboost_program_options.so.1.84.0,libboost_regex.so.1.84.0,libboost_system.so.1.84.0,libboost_thread.so.1.84.0,libboost_atomic.so.1.84.0} $THIRDPARTY_LIB/
41-
cp /usr/local/lib64/{libgflags.so.2.2,libglog.so.1,libgeos.so.3.10.7} $THIRDPARTY_LIB/
41+
cp /usr/local/lib/libgflags.so.2.2 $THIRDPARTY_LIB/
42+
find /usr/local/lib /usr/local/lib64 -name "libglog.so.1" -exec cp {} $THIRDPARTY_LIB/ \; 2>/dev/null
43+
find /usr/local/lib /usr/local/lib64 -name "libgeos.so.3.10.7" -exec cp {} $THIRDPARTY_LIB/ \; 2>/dev/null
4244
}
4345

4446
function process_setup_centos_8 {

‎dev/vcpkg/setup-build-depends.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -202,7 +202,7 @@ install_openeuler_24.03() {
202202
gcc g++ cmake ninja-build perl-IPC-Cmd autoconf autoconf-archive automake libtool \
203203
java-1.8.0-openjdk java-1.8.0-openjdk-devel python3-devel python3-pip libstdc++-static
204204

205-
pip install cmake==3.28.3
205+
pip install cmake==3.31.4
206206

207207
install_maven_from_source
208208
}

‎ep/build-velox/src/build-velox.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -110,7 +110,7 @@ function compile {
110110
# -Wno-unknown-warning-option is a Clang-originated flag. GCC ignores unrecognized -Wno- flags to
111111
# maintain compatibility, but it prints a diagnostic note about the unknown flag if a true warning
112112
# or error occurs.
113-
CXX_FLAGS='-Wno-error=stringop-overflow -Wno-error=cpp -Wno-missing-field-initializers \
113+
CXX_FLAGS='-Wno-error=stringop-overflow -Wno-error=stringop-overlap -Wno-error=cpp -Wno-missing-field-initializers \
114114
-Wno-error=uninitialized -Wno-unknown-warning-option -Wno-deprecated-declarations'
115115
if [[ "$(uname)" == "Darwin" ]]; then
116116
CXX_FLAGS="$CXX_FLAGS -Wno-inconsistent-missing-override -Wno-macro-redefined"

‎ep/build-velox/src/setup-centos8.sh‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -172,7 +172,10 @@ function install_wangle {
172172

173173
function install_fbthrift {
174174
wget_and_untar https://github.com/facebook/fbthrift/archive/refs/tags/${FB_OS_VERSION}.tar.gz fbthrift
175-
cmake_install_dir fbthrift -Denable_tests=OFF -DBUILD_SHARED_LIBS=OFF -DBUILD_TESTS=OFF -DBUILD_SHARED_LIBS=OFF
175+
cd ${DEPENDENCY_DIR}/fbthrift
176+
git apply ${VELOX_HOME:-${DEPENDENCY_DIR}/../}/CMake/resolve_dependency_modules/fbthrift/compactv1-protocol-refiller.patch 2>/dev/null || true
177+
cd ${DEPENDENCY_DIR}
178+
cmake_install_dir fbthrift -Denable_tests=OFF -DBUILD_TESTS=OFF -DBUILD_SHARED_LIBS=OFF
176179
}
177180

178181
function install_mvfst {

‎ep/build-velox/src/setup-openeuler24.sh‎

Lines changed: 24 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ NPROC=$(getconf _NPROCESSORS_ONLN)
3636
export CFLAGS=$(get_cxx_flags $CPU_TARGET) # Used by LZO.
3737
export CXXFLAGS=$CFLAGS # Used by boost.
3838
export CPPFLAGS=$CFLAGS # Used by LZO.
39+
EXTRA_PKG_CXXFLAGS=" -isystem ${INSTALL_PREFIX}/include"
3940
CMAKE_BUILD_TYPE="${BUILD_TYPE:-Release}"
4041
VELOX_BUILD_SHARED=${VELOX_BUILD_SHARED:-"OFF"} #Build folly and gflags shared for use in libvelox.so.
4142
BUILD_DUCKDB="${BUILD_DUCKDB:-true}"
@@ -44,8 +45,8 @@ VERSION=$(cat /etc/os-release | grep VERSION_ID)
4445
export INSTALL_PREFIX=${INSTALL_PREFIX:-"/usr/local"}
4546
DEPENDENCY_DIR=${DEPENDENCY_DIR:-$(pwd)/deps-download}
4647

47-
FB_OS_VERSION="v2024.07.01.00"
48-
FMT_VERSION="10.1.1"
48+
FB_OS_VERSION="v2026.01.05.00"
49+
FMT_VERSION="11.2.0"
4950
BOOST_VERSION="boost-1.84.0"
5051
DUCKDB_VERSION="v0.8.1"
5152
GEOS_VERSION="3.10.7"
@@ -65,7 +66,7 @@ function install_build_prerequisites {
6566
dnf_install autoconf automake python3-devel python3-pip libtool
6667
dnf_install libxml2-devel libgsasl-devel libuuid-devel
6768

68-
pip install cmake==3.28.3
69+
pip install cmake==3.31.4
6970
}
7071

7172
# Install dependencies from the package managers.
@@ -110,11 +111,23 @@ function install_boost {
110111
)
111112
}
112113

114+
function install_xxhash {
115+
wget_and_untar https://github.com/Cyan4973/xxHash/archive/refs/tags/v0.8.2.tar.gz xxHash
116+
cd ${DEPENDENCY_DIR}/xxHash
117+
make "-j${NPROC}"
118+
make install PREFIX=${INSTALL_PREFIX}
119+
}
120+
113121
function install_snappy {
114122
wget_and_untar https://github.com/google/snappy/archive/1.1.8.tar.gz snappy
115123
cmake_install_dir snappy -DSNAPPY_BUILD_TESTS=OFF
116124
}
117125

126+
function install_fast_float {
127+
wget_and_untar https://github.com/fastfloat/fast_float/archive/refs/tags/v8.0.2.tar.gz fast_float
128+
cmake_install_dir fast_float -DBUILD_TESTS=OFF
129+
}
130+
118131
function install_fmt {
119132
wget_and_untar https://github.com/fmtlib/fmt/archive/${FMT_VERSION}.tar.gz fmt
120133
cmake_install_dir fmt -DFMT_TEST=OFF
@@ -133,7 +146,7 @@ function install_protobuf {
133146

134147
function install_fizz {
135148
wget_and_untar https://github.com/facebookincubator/fizz/archive/refs/tags/${FB_OS_VERSION}.tar.gz fizz
136-
cmake_install_dir fizz/fizz -DBUILD_TESTS=OFF
149+
cmake_install_dir fizz/fizz -DBUILD_TESTS=OFF -DBUILD_EXAMPLES=OFF
137150
}
138151

139152
function install_folly {
@@ -143,11 +156,14 @@ function install_folly {
143156

144157
function install_wangle {
145158
wget_and_untar https://github.com/facebook/wangle/archive/refs/tags/${FB_OS_VERSION}.tar.gz wangle
146-
cmake_install_dir wangle/wangle -DBUILD_TESTS=OFF
159+
cmake_install_dir wangle/wangle -DBUILD_TESTS=OFF -DBUILD_EXAMPLES=OFF
147160
}
148161

149162
function install_fbthrift {
150163
wget_and_untar https://github.com/facebook/fbthrift/archive/refs/tags/${FB_OS_VERSION}.tar.gz fbthrift
164+
cd ${DEPENDENCY_DIR}/fbthrift
165+
git apply ${VELOX_HOME:-${DEPENDENCY_DIR}/../}/CMake/resolve_dependency_modules/fbthrift/compactv1-protocol-refiller.patch 2>/dev/null || true
166+
cd ${DEPENDENCY_DIR}
151167
cmake_install_dir fbthrift -Denable_tests=OFF -DBUILD_TESTS=OFF -DBUILD_SHARED_LIBS=OFF
152168
}
153169

@@ -198,8 +214,10 @@ function install_velox_deps {
198214
run_and_time install_gflags
199215
run_and_time install_glog
200216
run_and_time install_lzo
217+
run_and_time install_xxhash
201218
run_and_time install_snappy
202219
run_and_time install_boost
220+
run_and_time install_fast_float
203221
run_and_time install_protobuf
204222
run_and_time install_fmt
205223
run_and_time install_folly
@@ -233,4 +251,4 @@ function install_velox_deps {
233251
echo "All dependencies for Velox installed!"
234252
dnf clean all
235253
fi
236-
)
254+
)

0 commit comments

Comments
 (0)