Skip to content

Commit e04cc1c

Browse files
taiyang-liliyang.127
taiyang-li
authored and
liyang.127
committed
add bolt backend in gluten
1 parent 5a5516b commit e04cc1c

911 files changed

Lines changed: 241192 additions & 64 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎Makefile‎

Lines changed: 226 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,226 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one or more
2+
# contributor license agreements. See the NOTICE file distributed with
3+
# this work for additional information regarding copyright ownership.
4+
# The ASF licenses this file to You under the Apache License, Version 2.0
5+
# (the "License"); you may not use this file except in compliance with
6+
# the License. You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
16+
ROOT_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST))))
17+
BUILD_DIR := ${ROOT_DIR}/cpp/build
18+
CONAN_FILE_DIR := ${ROOT_DIR}/cpp/
19+
BUILD_TYPE=Debug
20+
ENABLE_ASAN ?= False
21+
LDB_BUILD ?= False
22+
BUILD_BENCHMARKS ?= False
23+
BUILD_TESTS ?= False
24+
BUILD_EXAMPLES ?= False
25+
BUILD_ORC ?= False
26+
ENABLE_PROTON ?= False
27+
28+
# conan package info
29+
GLUTEN_BUILD_VERSION ?= main
30+
BOLT_BUILD_VERSION ?= main
31+
BUILD_USER ?=
32+
BUILD_CHANNEL ?=
33+
34+
ENABLE_HDFS ?= True
35+
ENABLE_S3 ?= False
36+
RSS_PROFILE ?= ''
37+
38+
ifeq ($(BUILD_BENCHMARKS),True)
39+
BUILD_ORC = True
40+
endif
41+
42+
ARCH := $(shell arch)
43+
ifeq ($(ARCH), x86_64)
44+
ARCH := amd64
45+
endif
46+
47+
SHARED_LIBRARY ?= True
48+
49+
# Gluten package version, read from package/pom.xml so jar paths stay in sync
50+
# when the project version bumps.
51+
GLUTEN_PACKAGE_VERSION := $(shell sed -n '0,/<version>\(.*\)<\/version>/s//\1/p' \
52+
${ROOT_DIR}/package/pom.xml | head -1 | xargs)
53+
54+
# Manually specify the number of bolt compilation threads by setting the BOLT_NUM_THREADS environment variable.
55+
# e.g. export BOLT_NUM_THREADS=50
56+
ifndef CI_NUM_THREADS
57+
ifdef BOLT_NUM_THREADS
58+
NUM_THREADS ?= $(BOLT_NUM_THREADS)
59+
else
60+
NUM_THREADS ?= $$(( $(shell grep -c ^processor /proc/cpuinfo) / 2 ))
61+
endif
62+
else
63+
NUM_THREADS ?= $(CI_NUM_THREADS)
64+
endif
65+
66+
ALLOWED_VERSIONS := 11 17
67+
ifeq ($(JAVA_HOME),)
68+
$(error ERROR: JAVA_HOME is not set)
69+
endif
70+
ifneq ($(wildcard $(JAVA_HOME)/bin/java),)
71+
ifneq ($(wildcard $(JAVA_HOME)/bin/javac),)
72+
JDK_VERSION := $(shell $(JAVA_HOME)/bin/java -version 2>&1 | sed -n 's/.*version "\(1\.\)\{0,1\}\([0-9]\+\).*/\2/p')
73+
ifneq ($(filter $(JDK_VERSION),$(ALLOWED_VERSIONS)),$(JDK_VERSION))
74+
$(error ERROR: JDK version $(JDK_VERSION) is not supported, only 11 and 17 are allowed now)
75+
endif
76+
endif
77+
endif
78+
79+
.PHONY: build clean clean_cpp debug release RelWithDebInfo \
80+
install_debug install_release release-with-tests debug-with-tests \
81+
release-with-benchmarks debug-with-benchmarks \
82+
release-with-tests-and-benchmarks debug-with-tests-and-benchmarks \
83+
java jar jar-skip-check jar_spark33 jar_spark34 jar_spark35 \
84+
fast-jar zip fast-zip test test_spark35 cpp-test-release cpp-test-debug
85+
86+
bolt-recipe:
87+
@echo "Install Bolt recipe into local cache"
88+
rm -rf ep/bolt
89+
git clone --depth=1 --branch ${BOLT_BUILD_VERSION} https://github.com/bytedance/bolt.git ep/bolt &&\
90+
bash ep/bolt/scripts/install-bolt-deps.sh && \
91+
conan export ep/bolt/conanfile.py --name=bolt --version=${BOLT_BUILD_VERSION} --user=${BUILD_USER} --channel=${BUILD_CHANNEL}
92+
@echo "Bolt recipe has been installed"
93+
94+
build:
95+
mkdir -p ${BUILD_DIR} && mkdir -p ${BUILD_DIR}/releases &&\
96+
cd ${CONAN_FILE_DIR} && export BOLT_BUILD_VERSION=${BOLT_BUILD_VERSION} &&\
97+
ALL_CONAN_OPTIONS=" -o gluten/*:shared=${SHARED_LIBRARY} \
98+
-o gluten/*:enable_hdfs=${ENABLE_HDFS} \
99+
-o gluten/*:enable_s3=${ENABLE_S3} \
100+
-o gluten/*:enable_asan=${ENABLE_ASAN} \
101+
-o gluten/*:build_benchmarks=${BUILD_BENCHMARKS} \
102+
-o gluten/*:build_tests=${BUILD_TESTS} \
103+
-o gluten/*:build_examples=${BUILD_EXAMPLES} " && \
104+
conan graph info . --name=gluten --version=${GLUTEN_BUILD_VERSION} --user=${BUILD_USER} --channel=${BUILD_CHANNEL} -c "arrow/*:tools.build:download_source=True" $${ALL_CONAN_OPTIONS} --format=html > gluten.conan.graph.html && \
105+
NUM_THREADS=$(NUM_THREADS) conan install . --name=gluten --version=${GLUTEN_BUILD_VERSION} --user=${BUILD_USER} --channel=${BUILD_CHANNEL} \
106+
-s llvm-core/*:build_type=Release -s build_type=${BUILD_TYPE} --build=missing $${ALL_CONAN_OPTIONS} && \
107+
cmake --preset `echo conan-${BUILD_TYPE} | tr A-Z a-z` && \
108+
cmake --build build/${BUILD_TYPE} -j $(NUM_THREADS) && \
109+
if [ "${SHARED_LIBRARY}" = "True" ]; then cmake --build ${BUILD_DIR}/${BUILD_TYPE} --target install ; fi && \
110+
if [ "${SHARED_LIBRARY}" = "False" ]; then \
111+
conan export-pkg . --name=gluten --version=${GLUTEN_BUILD_VERSION} --user=${BUILD_USER} --channel=${BUILD_CHANNEL} -s build_type=${BUILD_TYPE} \
112+
$${ALL_CONAN_OPTIONS} ; \
113+
fi && cd -
114+
115+
release :
116+
$(MAKE) build BUILD_TYPE=Release GLUTEN_BUILD_VERSION=${GLUTEN_BUILD_VERSION} BOLT_BUILD_VERSION=${BOLT_BUILD_VERSION} BUILD_USER=${BUILD_USER} BUILD_CHANNEL=${BUILD_CHANNEL}
117+
118+
debug:
119+
$(MAKE) build BUILD_TYPE=Debug GLUTEN_BUILD_VERSION=${GLUTEN_BUILD_VERSION} BOLT_BUILD_VERSION=${BOLT_BUILD_VERSION} BUILD_USER=${BUILD_USER} BUILD_CHANNEL=${BUILD_CHANNEL}
120+
121+
RelWithDebInfo:
122+
$(MAKE) build BUILD_TYPE=RelWithDebInfo GLUTEN_BUILD_VERSION=${GLUTEN_BUILD_VERSION} BUILD_USER=${BUILD_USER} BUILD_CHANNEL=${BUILD_CHANNEL}
123+
124+
clean_cpp:
125+
rm -rf ${ROOT_DIR}/cpp/build &&\
126+
rm -f cpp/conan.lock cpp/conaninfo.txt cpp/graph_info.json CMakeCache.txt
127+
128+
install_debug:
129+
$(MAKE) clean_cpp
130+
$(MAKE) debug SHARED_LIBRARY=False
131+
132+
install_release:
133+
$(MAKE) clean_cpp
134+
$(MAKE) release SHARED_LIBRARY=False
135+
136+
release-with-tests :
137+
$(MAKE) build BUILD_TYPE=Release GLUTEN_BUILD_VERSION=${GLUTEN_BUILD_VERSION} BOLT_BUILD_VERSION=${BOLT_BUILD_VERSION} BUILD_USER=${BUILD_USER} BUILD_CHANNEL=${BUILD_CHANNEL} BUILD_TESTS=True
138+
139+
debug-with-tests :
140+
$(MAKE) build BUILD_TYPE=Debug GLUTEN_BUILD_VERSION=${GLUTEN_BUILD_VERSION} BOLT_BUILD_VERSION=${BOLT_BUILD_VERSION} BUILD_USER=${BUILD_USER} BUILD_CHANNEL=${BUILD_CHANNEL} BUILD_TESTS=True
141+
142+
release-with-benchmarks :
143+
$(MAKE) build BUILD_TYPE=Release GLUTEN_BUILD_VERSION=${GLUTEN_BUILD_VERSION} BOLT_BUILD_VERSION=${BOLT_BUILD_VERSION} B UILD_USER=${BUILD_USER} BUILD_CHANNEL=${BUILD_CHANNEL} BUILD_BENCHMARKS=True
144+
145+
debug-with-benchmarks :
146+
$(MAKE) build BUILD_TYPE=Debug GLUTEN_BUILD_VERSION=${GLUTEN_BUILD_VERSION} BOLT_BUILD_VERSION=${BOLT_BUILD_VERSION} BUILD_USER=${BUILD_USER} BUILD_CHANNEL=${BUILD_CHANNEL} BUILD_BENCHMARKS=True
147+
148+
release-with-tests-and-benchmarks :
149+
$(MAKE) build BUILD_TYPE=Release GLUTEN_BUILD_VERSION=${GLUTEN_BUILD_VERSION} BOLT_BUILD_VERSION=${BOLT_BUILD_VERSION} BUILD_USER=${BUILD_USER} BUILD_CHANNEL=${BUILD_CHANNEL} BUILD_BENCHMARKS=True BUILD_TESTS=True
150+
151+
debug-with-tests-and-benchmarks :
152+
$(MAKE) build BUILD_TYPE=Debug GLUTEN_BUILD_VERSION=${GLUTEN_BUILD_VERSION} BOLT_BUILD_VERSION=${BOLT_BUILD_VERSION} BUILD_USER=${BUILD_USER} BUILD_CHANNEL=${BUILD_CHANNEL} BUILD_BENCHMARKS=True BUILD_TESTS=True
153+
154+
arrow:
155+
bash dev/build_bolt_arrow.sh
156+
157+
# build gluten jar
158+
jar:
159+
java -version && mvn package -Pbackends-bolt -Pspark-3.3 -Pceleborn -DskipTests -Denforcer.skip=true -Pjava-8 -Ppaimon &&\
160+
mkdir -p output && \
161+
rm -rf output/gluten-spark*.jar
162+
mv package/target/gluten-package-$(GLUTEN_PACKAGE_VERSION).jar output/gluten-spark3.2_2.12-1.0.0-SNAPSHOT-jar-with-dependencies.jar
163+
164+
jar-skip-check:
165+
java -version && mvn package -Pbackends-bolt -Pspark-3.2 -Pceleborn -DskipTests -Denforcer.skip=true -Pjava-8 -Ppaimon -Dcheckstyle.skip=true -Dspotless.check.skip=true &&\
166+
mkdir -p output && \
167+
rm -rf output/gluten-spark*.jar
168+
mv package/target/gluten-package-$(GLUTEN_PACKAGE_VERSION).jar output/gluten-spark3.2_2.12-1.0.0-SNAPSHOT-jar-with-dependencies.jar
169+
170+
spark32-las:
171+
java -version && mvn package -Pbackends-bolt -Pspark-3.2-las -Pceleborn -DskipTests -Denforcer.skip=true -Pjava-8 -Ppaimon &&\
172+
mkdir -p output && \
173+
rm -rf output/gluten-spark*.jar
174+
mv package/target/gluten-package-$(GLUTEN_PACKAGE_VERSION).jar output/gluten-spark3.2_2.12-1.0.0-SNAPSHOT-jar-with-dependencies.jar
175+
176+
fast-jar:
177+
if [ ! -f "output/gluten-spark3.2_2.12-1.0.0-SNAPSHOT-jar-with-dependencies.jar" ] ; then \
178+
$(MAKE) jar; \
179+
else \
180+
jar uf output/gluten-spark3.2_2.12-1.0.0-SNAPSHOT-jar-with-dependencies.jar -C cpp/build/releases/ libbolt_backend.so; \
181+
fi
182+
183+
zip:
184+
$(MAKE) jar
185+
rm -rf output/gluten-spark*.zip
186+
zip -j output/gluten-spark3.2_2.12-1.0.0-SNAPSHOT-jar-with-dependencies.zip output/gluten-spark3.2_2.12-1.0.0-SNAPSHOT-jar-with-dependencies.jar
187+
188+
fast-zip:
189+
$(MAKE) fast-jar
190+
rm -rf output/gluten-spark*.zip
191+
zip -j output/gluten-spark3.2_2.12-1.0.0-SNAPSHOT-jar-with-dependencies.zip output/gluten-spark3.2_2.12-1.0.0-SNAPSHOT-jar-with-dependencies.jar
192+
193+
jar_spark33:
194+
java -version && mvn -T32 clean package -Pbackends-bolt -Pspark-3.3 -Pceleborn -Piceberg -DskipTests -Denforcer.skip=true -Ppaimon && \
195+
mkdir -p output && \
196+
rm -rf output/gluten-spark*.jar
197+
mv package/target/gluten-package-$(GLUTEN_PACKAGE_VERSION).jar output/gluten-spark3.3_2.12-1.0.0-SNAPSHOT-jar-with-dependencies.jar
198+
199+
jar_spark34:
200+
java -version && mvn clean package -Pbackends-bolt -Pspark-3.4 -Pceleborn -Piceberg -DskipTests -Denforcer.skip=true -Ppaimon && \
201+
mkdir -p output && \
202+
rm -rf output/gluten-spark*.jar
203+
mv package/target/gluten-package-$(GLUTEN_PACKAGE_VERSION).jar output/gluten-spark3.4_2.12-1.0.0-SNAPSHOT-jar-with-dependencies.jar
204+
205+
jar_spark35:
206+
java -version && mvn -T32 package -Pbackends-bolt -Pspark-3.5 -Phadoop-3.2 -Pceleborn -Piceberg -DskipTests -Denforcer.skip=true && \
207+
mkdir -p output && \
208+
rm -rf output/gluten-spark*.jar
209+
mv package/target/gluten-package-$(GLUTEN_PACKAGE_VERSION).jar output/gluten-spark3.5_2.12-1.0.0-SNAPSHOT-jar-with-dependencies.jar
210+
211+
test:
212+
mvn -Pbackends-bolt -Pspark-3.2 -Pceleborn -Ppaimon package -Denforcer.skip=true
213+
214+
test_spark35:
215+
mvn -Pbackends-bolt -Pspark-3.5 -Ppaimon -Phadoop-3.2 -Pceleborn -Piceberg package -Denforcer.skip=true -DfailIfNoTests=false -Dexec.skip -Dmaven.test.failure.ignore=true
216+
217+
cpp-test-release: release-with-tests
218+
cd $(BUILD_DIR)/Release && ctest --timeout 7200 -j $(NUM_THREADS) --output-on-failure -V
219+
220+
cpp-test-debug: debug-with-tests
221+
cd $(BUILD_DIR)/Debug && ctest --timeout 7200 -j $(NUM_THREADS) --output-on-failure -V
222+
223+
clean :
224+
$(MAKE) clean_cpp
225+
mvn clean -Pbackends-bolt -Pspark-3.2 -Pceleborn -Ppaimon -DskipTests -Denforcer.skip=true && \
226+
rm -rf ${ROOT_DIR}/output/gluten-*.jar

‎README.md‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -154,6 +154,60 @@ ClickHouse backend demonstrated an average speedup of 2.12x, with up to 3.48x sp
154154

155155
<sub>Test environment: a 8-nodes AWS cluster with 1TB data, using Spark 3.1.1 as the baseline and with Gluten integrated into the same Spark version.</sub>
156156

157+
### Bolt Backend
158+
#### Prerequisites
159+
* Linux operating system
160+
* GCC 10/11/12 or Clang 16
161+
* python 3 (virtualenv or Conda) for conan
162+
163+
Linux with kernel version(>5.4) is preferred, since Bolt will enable io-uring when the kernel supports.
164+
165+
if the system gcc version is too older, it is recommended to install GCC from source code:
166+
```shell
167+
# run with root privilege
168+
bash ./dev/install-gcc.sh 12.5.0
169+
```
170+
171+
Bolt adopts Conan as its package manager. Conan is an open-source, cross-platform package management tool.
172+
We provide dedicated scripts to assist developers in setting up and installing Bolt's dependencies.
173+
```shell
174+
bash ./dev/install-conan.sh
175+
```
176+
177+
We also provide a Dockerfile to build a Docker image for the **Bolt** backend, it includes all the prerequisites required to build Gluten with Bolt backend.
178+
```shell
179+
docker buildx build -t bolt -f dev/docker/Dockerfile.centos8-bolt .
180+
```
181+
182+
#### Build Bolt Backend
183+
To install bolt recipe for Gluten:
184+
```shell
185+
# Install the recipes of Bolt and its third-party dependencies
186+
make bolt-recipe
187+
188+
# specific a version of Bolt (release or branch)
189+
# `main` branch is the default
190+
make bolt-recipe BOLT_BUILD_VERSION=main
191+
```
192+
193+
To build bolt backend:
194+
```shell
195+
make release
196+
197+
# or specific the version for Bolt, and the version for Gluten
198+
make release BOLT_BUILD_VERSION=main GLUTEN_BUILD_VERSION=main
199+
```
200+
Note that, the missing third-parties binaries will be built from source for the first time.
201+
202+
To build gluten:
203+
204+
```shell
205+
# install arrow dependency for gluten
206+
make arrow
207+
208+
make jar_spark35
209+
```
210+
157211
## 8. Qualification Tool
158212

159213
The [Qualification Tool](./tools/qualification-tool/README.md) is a utility to analyze Spark event log files and assess the compatibility and performance of SQL workloads with Gluten. This tool helps users understand how their workloads can benefit from Gluten.

‎backends-bolt/benchmark/ColumnarTableCacheBenchmark-results.txt‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
OpenJDK 64-Bit Server VM 1.8.0_322-b06 on Mac OS X 13.5
2+
Apple M1 Pro
3+
table cache count: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
4+
------------------------------------------------------------------------------------------------------------------------
5+
disable columnar table cache 16773 17024 401 1.2 838.7 1.0X
6+
enable columnar table cache 9985 10051 65 2.0 499.3 1.0X
7+
8+
9+
OpenJDK 64-Bit Server VM 1.8.0_322-b06 on Mac OS X 13.5
10+
Apple M1 Pro
11+
table cache column pruning: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
12+
------------------------------------------------------------------------------------------------------------------------
13+
disable columnar table cache 16429 16873 688 1.2 821.5 1.0X
14+
enable columnar table cache 15118 15495 456 1.3 755.9 1.0X
15+
16+
17+
OpenJDK 64-Bit Server VM 1.8.0_322-b06 on Mac OS X 13.5
18+
Apple M1 Pro
19+
table cache filter: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
20+
------------------------------------------------------------------------------------------------------------------------
21+
disable columnar table cache 22895 23527 722 0.9 1144.7 1.0X
22+
enable columnar table cache 16673 17462 765 1.2 833.7 1.0X
23+

0 commit comments

Comments
 (0)