Skip to content

Commit 3f3e83b

Browse files
feat(fs): add object store and S3 file systems (alibaba#433)
Add a reusable read-only object store layer with a curl-based HTTP transport. Use the AWS C authentication components for credential resolution and request signing while keeping S3 data access independent of the full AWS SDK.
1 parent e5ba17e commit 3f3e83b

20 files changed

Lines changed: 3131 additions & 1 deletion

.github/workflows/build_and_test.yaml

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,10 @@ jobs:
7171
if: ${{ !matrix.skip_rust }}
7272
shell: bash
7373
run: ci/scripts/setup_rust.sh
74+
- name: Install HTTP and TLS development dependencies
75+
run: |
76+
sudo apt-get update
77+
sudo apt-get install -y libcurl4-openssl-dev libssl-dev
7478
- name: Build Paimon
7579
shell: bash
7680
env:

.github/workflows/gcc8_test.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@ jobs:
4242
- name: Install dependencies
4343
run: |
4444
apt-get update
45-
DEBIAN_FRONTEND=noninteractive apt-get install -y gcc-8 g++-8 ninja-build git git-lfs tar curl tzdata zip unzip pkg-config build-essential python3-dev gdb sudo
45+
DEBIAN_FRONTEND=noninteractive apt-get install -y gcc-8 g++-8 ninja-build git git-lfs tar curl libcurl4-openssl-dev libssl-dev tzdata zip unzip pkg-config build-essential python3-dev gdb sudo
4646
curl -L -O https://github.com/Kitware/CMake/releases/download/v3.28.3/cmake-3.28.3-linux-x86_64.tar.gz
4747
tar -zxvf cmake-3.28.3-linux-x86_64.tar.gz -C /usr/local --strip-components=1
4848
rm cmake-3.28.3-linux-x86_64.tar.gz

CMakeLists.txt

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,7 @@ option(PAIMON_USE_CXX11_ABI "Use C++11 ABI" ON)
5959
option(PAIMON_ENABLE_AVRO "Whether to enable avro file format" ON)
6060
option(PAIMON_ENABLE_ORC "Whether to enable orc file format" ON)
6161
option(PAIMON_ENABLE_JINDO "Whether to enable jindo file system" OFF)
62+
option(PAIMON_ENABLE_S3 "Whether to enable S3 file system" OFF)
6263
option(PAIMON_ENABLE_NETWORK_TESTS
6364
"Whether to enable tests that access real remote services over the network" OFF)
6465
option(PAIMON_ENABLE_LUMINA "Whether to enable lumina vector index" OFF)
@@ -74,6 +75,9 @@ endif()
7475
if(PAIMON_ENABLE_JINDO)
7576
add_definitions(-DPAIMON_ENABLE_JINDO)
7677
endif()
78+
if(PAIMON_ENABLE_S3)
79+
add_definitions(-DPAIMON_ENABLE_S3)
80+
endif()
7781
if(PAIMON_ENABLE_NETWORK_TESTS)
7882
add_definitions(-DPAIMON_ENABLE_NETWORK_TESTS)
7983
endif()
@@ -429,6 +433,13 @@ if(PAIMON_BUILD_TESTS)
429433
paimon_jindo_file_system_shared)
430434
list(APPEND TEST_STATIC_LINK_LIBS ${TEST_PLUGIN_LINK_LIBS})
431435
endif()
436+
if(PAIMON_ENABLE_S3)
437+
paimon_link_libraries_whole_archive(PAIMON_S3_FILE_SYSTEM_STATIC_LINK_LIBS
438+
paimon_s3_file_system_static)
439+
paimon_link_libraries_no_as_needed(TEST_PLUGIN_LINK_LIBS
440+
paimon_s3_file_system_shared)
441+
list(APPEND TEST_STATIC_LINK_LIBS ${TEST_PLUGIN_LINK_LIBS})
442+
endif()
432443
if(PAIMON_ENABLE_LUMINA)
433444
paimon_link_libraries_whole_archive(PAIMON_LUMINA_INDEX_STATIC_LINK_LIBS
434445
paimon_lumina_index_static)
@@ -476,9 +487,13 @@ install(FILES "${CMAKE_CURRENT_BINARY_DIR}/PaimonConfig.cmake"
476487
477488
config_summary_message()
478489
490+
if(PAIMON_ENABLE_S3)
491+
find_package(CURL REQUIRED)
492+
endif()
479493
add_subdirectory(src/paimon)
480494
add_subdirectory(src/paimon/fs/local)
481495
add_subdirectory(src/paimon/fs/jindo)
496+
add_subdirectory(src/paimon/fs/s3)
482497
add_subdirectory(src/paimon/format/blob)
483498
add_subdirectory(src/paimon/format/orc)
484499
add_subdirectory(src/paimon/format/parquet)

ci/scripts/build_paimon.sh

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -126,6 +126,7 @@ CMAKE_ARGS=(
126126
"-DCMAKE_BUILD_TYPE=${build_type}"
127127
"-DPAIMON_BUILD_TESTS=ON"
128128
"-DPAIMON_ENABLE_JINDO=ON"
129+
"-DPAIMON_ENABLE_S3=ON"
129130
"-DPAIMON_ENABLE_LUMINA=${ENABLE_LUMINA}"
130131
"-DPAIMON_ENABLE_LUCENE=ON"
131132
"-DPAIMON_ENABLE_TANTIVY=${ENABLE_TANTIVY}"

cmake_modules/BuildAwsAuth.cmake

Lines changed: 187 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,187 @@
1+
# Copyright 2026-present Alibaba Inc.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
include(ExternalProject)
16+
include(GNUInstallDirs)
17+
18+
function(paimon_add_s2n_project)
19+
if(DEFINED ENV{PAIMON_AWS_S2N_URL})
20+
set(S2N_URL "$ENV{PAIMON_AWS_S2N_URL}")
21+
elseif(EXISTS "${THIRDPARTY_DIR}/${PAIMON_AWS_S2N_PKG_NAME}")
22+
set(S2N_URL "${THIRDPARTY_DIR}/${PAIMON_AWS_S2N_PKG_NAME}")
23+
else()
24+
set(S2N_URL
25+
"${THIRDPARTY_MIRROR_URL}https://github.com/aws/s2n-tls/archive/refs/tags/${PAIMON_AWS_S2N_BUILD_VERSION}.zip"
26+
)
27+
endif()
28+
set(S2N_C_FLAGS "${EP_C_FLAGS}")
29+
set(S2N_CXX_FLAGS "${EP_CXX_FLAGS}")
30+
string(REPLACE "-Wdocumentation" "" S2N_C_FLAGS "${S2N_C_FLAGS}")
31+
string(REPLACE "-Wdocumentation" "" S2N_CXX_FLAGS "${S2N_CXX_FLAGS}")
32+
externalproject_add(s2n_ep
33+
URL ${S2N_URL}
34+
URL_HASH SHA256=${PAIMON_AWS_S2N_BUILD_SHA256_CHECKSUM}
35+
DOWNLOAD_EXTRACT_TIMESTAMP TRUE
36+
CMAKE_ARGS ${EP_COMMON_CMAKE_ARGS}
37+
-DCMAKE_C_FLAGS=${S2N_C_FLAGS}
38+
-DCMAKE_CXX_FLAGS=${S2N_CXX_FLAGS}
39+
-DCMAKE_INSTALL_PREFIX=${AWS_AUTH_PREFIX}
40+
-DCMAKE_INSTALL_LIBDIR=${AWS_AUTH_INSTALL_LIBDIR}
41+
-DCMAKE_PREFIX_PATH=${AWS_AUTH_PREFIX}
42+
-Dcrypto_INCLUDE_DIR=${OPENSSL_INCLUDE_DIR}
43+
-Dcrypto_LIBRARY=${OPENSSL_CRYPTO_LIBRARY}
44+
-DCMAKE_POSITION_INDEPENDENT_CODE=ON
45+
-DS2N_INTERN_LIBCRYPTO=OFF
46+
BUILD_BYPRODUCTS ${AWS_AUTH_LIB_DIR}/libs2n.a
47+
${THIRDPARTY_LOG_OPTIONS})
48+
endfunction()
49+
50+
function(paimon_add_aws_c_project
51+
NAME
52+
VERSION
53+
CHECKSUM
54+
DEPENDS)
55+
string(REPLACE "aws-c-" "AWS_C_" URL_NAME "${NAME}")
56+
string(TOUPPER "${URL_NAME}" URL_NAME)
57+
set(URL_VAR "PAIMON_${URL_NAME}_URL")
58+
set(PKG_NAME_VAR "PAIMON_${URL_NAME}_PKG_NAME")
59+
if(DEFINED ENV{${URL_VAR}})
60+
set(URL "$ENV{${URL_VAR}}")
61+
elseif(EXISTS "${THIRDPARTY_DIR}/${${PKG_NAME_VAR}}")
62+
set(URL "${THIRDPARTY_DIR}/${${PKG_NAME_VAR}}")
63+
else()
64+
set(URL
65+
"${THIRDPARTY_MIRROR_URL}https://github.com/awslabs/${NAME}/archive/refs/tags/${VERSION}.tar.gz"
66+
)
67+
endif()
68+
set(AWS_PLATFORM_CMAKE_ARGS)
69+
list(APPEND
70+
AWS_PLATFORM_CMAKE_ARGS
71+
-DOPENSSL_INCLUDE_DIR=${OPENSSL_INCLUDE_DIR}
72+
-DOPENSSL_SSL_LIBRARY=${OPENSSL_SSL_LIBRARY}
73+
-DOPENSSL_CRYPTO_LIBRARY=${OPENSSL_CRYPTO_LIBRARY}
74+
-Dcrypto_INCLUDE_DIR=${OPENSSL_INCLUDE_DIR}
75+
-Dcrypto_LIBRARY=${OPENSSL_CRYPTO_LIBRARY})
76+
if(NAME STREQUAL "aws-c-cal")
77+
list(APPEND AWS_PLATFORM_CMAKE_ARGS -DUSE_OPENSSL=ON)
78+
endif()
79+
if(APPLE)
80+
list(APPEND AWS_PLATFORM_CMAKE_ARGS -DAWS_USE_SECITEM=ON)
81+
endif()
82+
# aws-c-io's installed package unconditionally looks for s2n on Unix even
83+
# when it was built with Apple's native TLS backend.
84+
if(APPLE AND (NAME STREQUAL "aws-c-http" OR NAME STREQUAL "aws-c-auth"))
85+
list(APPEND AWS_PLATFORM_CMAKE_ARGS -DBYO_CRYPTO=ON)
86+
endif()
87+
externalproject_add(${NAME}_ep
88+
URL ${URL}
89+
URL_HASH SHA256=${CHECKSUM}
90+
DOWNLOAD_EXTRACT_TIMESTAMP TRUE
91+
CMAKE_ARGS ${EP_COMMON_CMAKE_ARGS}
92+
-DCMAKE_INSTALL_PREFIX=${AWS_AUTH_PREFIX}
93+
-DCMAKE_INSTALL_LIBDIR=${AWS_AUTH_INSTALL_LIBDIR}
94+
-DCMAKE_PREFIX_PATH=${AWS_AUTH_PREFIX}
95+
-DCMAKE_POSITION_INDEPENDENT_CODE=ON
96+
-DENABLE_TESTING=OFF
97+
${AWS_PLATFORM_CMAKE_ARGS}
98+
DEPENDS ${DEPENDS}
99+
BUILD_BYPRODUCTS ${AWS_AUTH_LIB_DIR}/lib${NAME}.a
100+
${THIRDPARTY_LOG_OPTIONS})
101+
endfunction()
102+
103+
function(build_aws_auth)
104+
set(AWS_AUTH_PREFIX
105+
"${CMAKE_CURRENT_BINARY_DIR}/aws-auth_ep-install"
106+
PARENT_SCOPE)
107+
set(AWS_AUTH_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/aws-auth_ep-install")
108+
set(AWS_AUTH_INSTALL_LIBDIR "${CMAKE_INSTALL_LIBDIR}")
109+
set(AWS_AUTH_LIB_DIR "${AWS_AUTH_PREFIX}/${AWS_AUTH_INSTALL_LIBDIR}")
110+
file(MAKE_DIRECTORY "${AWS_AUTH_PREFIX}/include")
111+
file(MAKE_DIRECTORY "${AWS_AUTH_LIB_DIR}")
112+
113+
find_package(OpenSSL REQUIRED)
114+
115+
set(AWS_IO_DEPENDS "aws-c-common_ep;aws-c-cal_ep")
116+
if(CMAKE_SYSTEM_NAME STREQUAL "Linux")
117+
paimon_add_s2n_project()
118+
list(APPEND AWS_IO_DEPENDS s2n_ep)
119+
endif()
120+
121+
paimon_add_aws_c_project(aws-c-common ${PAIMON_AWS_C_COMMON_BUILD_VERSION}
122+
${PAIMON_AWS_C_COMMON_BUILD_SHA256_CHECKSUM} "")
123+
paimon_add_aws_c_project(aws-c-sdkutils ${PAIMON_AWS_C_SDKUTILS_BUILD_VERSION}
124+
${PAIMON_AWS_C_SDKUTILS_BUILD_SHA256_CHECKSUM}
125+
aws-c-common_ep)
126+
paimon_add_aws_c_project(aws-c-cal ${PAIMON_AWS_C_CAL_BUILD_VERSION}
127+
${PAIMON_AWS_C_CAL_BUILD_SHA256_CHECKSUM} aws-c-common_ep)
128+
paimon_add_aws_c_project(aws-c-compression ${PAIMON_AWS_C_COMPRESSION_BUILD_VERSION}
129+
${PAIMON_AWS_C_COMPRESSION_BUILD_SHA256_CHECKSUM}
130+
aws-c-common_ep)
131+
paimon_add_aws_c_project(aws-c-io ${PAIMON_AWS_C_IO_BUILD_VERSION}
132+
${PAIMON_AWS_C_IO_BUILD_SHA256_CHECKSUM} "${AWS_IO_DEPENDS}")
133+
paimon_add_aws_c_project(aws-c-http ${PAIMON_AWS_C_HTTP_BUILD_VERSION}
134+
${PAIMON_AWS_C_HTTP_BUILD_SHA256_CHECKSUM}
135+
"aws-c-common_ep;aws-c-io_ep;aws-c-compression_ep")
136+
paimon_add_aws_c_project(aws-c-auth
137+
${PAIMON_AWS_C_AUTH_BUILD_VERSION}
138+
${PAIMON_AWS_C_AUTH_BUILD_SHA256_CHECKSUM}
139+
"aws-c-common_ep;aws-c-sdkutils_ep;aws-c-cal_ep;aws-c-io_ep;aws-c-http_ep"
140+
)
141+
142+
find_package(Threads REQUIRED)
143+
set(AWS_AUTH_PLATFORM_LIBS Threads::Threads ${CMAKE_DL_LIBS})
144+
if(APPLE)
145+
list(APPEND
146+
AWS_AUTH_PLATFORM_LIBS
147+
"-framework Security"
148+
"-framework CoreFoundation"
149+
"-framework Network")
150+
endif()
151+
set(AWS_AUTH_TLS_LIBS)
152+
if(CMAKE_SYSTEM_NAME STREQUAL "Linux")
153+
list(APPEND AWS_AUTH_TLS_LIBS "${AWS_AUTH_LIB_DIR}/libs2n.a")
154+
endif()
155+
set(AWS_AUTH_RUNTIME_LIBS
156+
"${AWS_AUTH_LIB_DIR}/libaws-c-http.a"
157+
"${AWS_AUTH_LIB_DIR}/libaws-c-io.a"
158+
"${AWS_AUTH_LIB_DIR}/libaws-c-compression.a"
159+
"${AWS_AUTH_LIB_DIR}/libaws-c-cal.a"
160+
"${AWS_AUTH_LIB_DIR}/libaws-c-sdkutils.a"
161+
"${AWS_AUTH_LIB_DIR}/libaws-c-common.a"
162+
${AWS_AUTH_TLS_LIBS})
163+
if(NOT APPLE)
164+
list(PREPEND AWS_AUTH_RUNTIME_LIBS "-Wl,--start-group")
165+
list(APPEND AWS_AUTH_RUNTIME_LIBS "-Wl,--end-group")
166+
endif()
167+
# Keep OpenSSL after the static AWS archives. In embedded builds CMake can
168+
# resolve these targets to static archives, where link order matters.
169+
list(APPEND AWS_AUTH_RUNTIME_LIBS "${OPENSSL_SSL_LIBRARY}"
170+
"${OPENSSL_CRYPTO_LIBRARY}")
171+
add_library(aws_auth_minimal STATIC IMPORTED GLOBAL)
172+
set_target_properties(aws_auth_minimal
173+
PROPERTIES IMPORTED_LOCATION
174+
"${AWS_AUTH_LIB_DIR}/libaws-c-auth.a"
175+
INTERFACE_INCLUDE_DIRECTORIES
176+
"${AWS_AUTH_PREFIX}/include"
177+
INTERFACE_LINK_LIBRARIES
178+
"${AWS_AUTH_RUNTIME_LIBS};${AWS_AUTH_PLATFORM_LIBS}")
179+
add_dependencies(aws_auth_minimal aws-c-auth_ep)
180+
181+
set(AWS_AUTH_INCLUDE_DIR
182+
"${AWS_AUTH_PREFIX}/include"
183+
PARENT_SCOPE)
184+
set(AWS_AUTH_LIB_DIR
185+
"${AWS_AUTH_LIB_DIR}"
186+
PARENT_SCOPE)
187+
endfunction()

cmake_modules/ThirdpartyToolchain.cmake

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1887,6 +1887,10 @@ if(PAIMON_ENABLE_JINDO)
18871887
build_jindosdk_c()
18881888
build_jindosdk_nextarch()
18891889
endif()
1890+
if(PAIMON_ENABLE_S3)
1891+
include(BuildAwsAuth)
1892+
build_aws_auth()
1893+
endif()
18901894
if(PAIMON_ENABLE_LUMINA)
18911895
build_lumina()
18921896
endif()

cmake_modules/arrow.diff

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -415,6 +415,14 @@ diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/Thi
415415
# and crosscompiling emulator (for try_run() )
416416
if(CMAKE_CROSSCOMPILING_EMULATOR)
417417
string(REPLACE ";" ${EP_LIST_SEPARATOR} EP_CMAKE_CROSSCOMPILING_EMULATOR
418+
@@ -1720,6 +1725,7 @@ macro(build_thrift)
419+
-DWITH_JAVASCRIPT=OFF
420+
-DWITH_LIBEVENT=OFF
421+
-DWITH_NODEJS=OFF
422+
+ -DWITH_OPENSSL=OFF
423+
-DWITH_PYTHON=OFF
424+
-DWITH_QT5=OFF
425+
-DWITH_ZLIB=OFF)
418426
diff --git a/cpp/cmake_modules/BuildUtils.cmake b/cpp/cmake_modules/BuildUtils.cmake
419427
--- a/cpp/cmake_modules/BuildUtils.cmake
420428
+++ b/cpp/cmake_modules/BuildUtils.cmake

src/paimon/CMakeLists.txt

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -178,6 +178,12 @@ set(PAIMON_COMMON_SRCS
178178
common/utils/status.cpp
179179
common/utils/string_utils.cpp)
180180

181+
if(PAIMON_ENABLE_S3)
182+
list(APPEND PAIMON_COMMON_SRCS common/fs/http_client.cpp
183+
common/fs/object_store_file_system.cpp)
184+
set(PAIMON_OBJECT_STORE_LINK_LIBS CURL::libcurl)
185+
endif()
186+
181187
set(PAIMON_CORE_SRCS
182188
core/disk/file_io_channel.cpp
183189
core/mergetree/spill_reader.cpp
@@ -410,6 +416,7 @@ add_paimon_lib(paimon
410416
xxhash
411417
Threads::Threads
412418
RapidJSON
419+
${PAIMON_OBJECT_STORE_LINK_LIBS}
413420
STATIC_LINK_LIBS
414421
arrow
415422
tbb
@@ -419,6 +426,7 @@ add_paimon_lib(paimon
419426
xxhash
420427
Threads::Threads
421428
RapidJSON
429+
${PAIMON_OBJECT_STORE_LINK_LIBS}
422430
SHARED_LINK_FLAGS
423431
${PAIMON_VERSION_SCRIPT_FLAGS})
424432

@@ -838,10 +846,17 @@ if(PAIMON_BUILD_TESTS)
838846
${PAIMON_JINDO_FILE_SYSTEM_STATIC_LINK_LIBS})
839847
endif()
840848

849+
set(PAIMON_OBJECT_STORE_FS_TEST_SOURCES)
850+
if(PAIMON_ENABLE_S3)
851+
list(APPEND PAIMON_OBJECT_STORE_FS_TEST_SOURCES
852+
common/fs/object_store_file_system_test.cpp)
853+
endif()
854+
841855
add_paimon_test(fs_test
842856
SOURCES
843857
common/fs/file_system_test.cpp
844858
common/fs/resolving_file_system_test.cpp
859+
${PAIMON_OBJECT_STORE_FS_TEST_SOURCES}
845860
fs/local/local_file_test.cpp
846861
${PAIMON_JINDO_FS_TEST_SOURCES}
847862
STATIC_LINK_LIBS

0 commit comments

Comments
 (0)