Skip to content

Commit 1d2f7d1

Browse files
mrdrivingduckcodex
andcommitted
feat(fs): add object store and S3 file systems
Add a reusable read-only object store layer with a curl-based HTTP transport. Use the AWS C authentication components for credential resolution and request signing while keeping S3 data access independent of the full AWS SDK. Co-authored-by: GPT-5.6 Terra <codex@users.noreply.github.com>
1 parent debf91d commit 1d2f7d1

20 files changed

Lines changed: 2733 additions & 1 deletion

.github/workflows/build_and_test.yaml

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,10 @@ jobs:
7171
if: ${{ !matrix.skip_rust }}
7272
shell: bash
7373
run: ci/scripts/setup_rust.sh
74+
- name: Install HTTP and TLS development dependencies
75+
run: |
76+
sudo apt-get update
77+
sudo apt-get install -y libcurl4-openssl-dev libssl-dev
7478
- name: Build Paimon
7579
shell: bash
7680
env:

.github/workflows/gcc8_test.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@ jobs:
4242
- name: Install dependencies
4343
run: |
4444
apt-get update
45-
DEBIAN_FRONTEND=noninteractive apt-get install -y gcc-8 g++-8 ninja-build git git-lfs tar curl tzdata zip unzip pkg-config build-essential python3-dev gdb sudo
45+
DEBIAN_FRONTEND=noninteractive apt-get install -y gcc-8 g++-8 ninja-build git git-lfs tar curl libcurl4-openssl-dev libssl-dev tzdata zip unzip pkg-config build-essential python3-dev gdb sudo
4646
curl -L -O https://github.com/Kitware/CMake/releases/download/v3.28.3/cmake-3.28.3-linux-x86_64.tar.gz
4747
tar -zxvf cmake-3.28.3-linux-x86_64.tar.gz -C /usr/local --strip-components=1
4848
rm cmake-3.28.3-linux-x86_64.tar.gz

CMakeLists.txt

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,7 @@ option(PAIMON_ENABLE_AVRO "Whether to enable avro file format" ON)
5757
option(PAIMON_ENABLE_ORC "Whether to enable orc file format" ON)
5858
option(PAIMON_ENABLE_LANCE "Whether to enable lance file format" OFF)
5959
option(PAIMON_ENABLE_JINDO "Whether to enable jindo file system" OFF)
60+
option(PAIMON_ENABLE_S3 "Whether to enable S3 file system" OFF)
6061
option(PAIMON_ENABLE_NETWORK_TESTS
6162
"Whether to enable tests that access real remote services over the network" OFF)
6263
option(PAIMON_ENABLE_LUMINA "Whether to enable lumina vector index" OFF)
@@ -72,6 +73,9 @@ endif()
7273
if(PAIMON_ENABLE_JINDO)
7374
add_definitions(-DPAIMON_ENABLE_JINDO)
7475
endif()
76+
if(PAIMON_ENABLE_S3)
77+
add_definitions(-DPAIMON_ENABLE_S3)
78+
endif()
7579
if(PAIMON_ENABLE_NETWORK_TESTS)
7680
if(NOT PAIMON_BUILD_TESTS)
7781
message(FATAL_ERROR "PAIMON_ENABLE_NETWORK_TESTS requires PAIMON_BUILD_TESTS=ON")
@@ -455,6 +459,13 @@ if(PAIMON_BUILD_TESTS)
455459
paimon_jindo_file_system_shared)
456460
list(APPEND TEST_STATIC_LINK_LIBS ${TEST_PLUGIN_LINK_LIBS})
457461
endif()
462+
if(PAIMON_ENABLE_S3)
463+
paimon_link_libraries_whole_archive(PAIMON_S3_FILE_SYSTEM_STATIC_LINK_LIBS
464+
paimon_s3_file_system_static)
465+
paimon_link_libraries_no_as_needed(TEST_PLUGIN_LINK_LIBS
466+
paimon_s3_file_system_shared)
467+
list(APPEND TEST_STATIC_LINK_LIBS ${TEST_PLUGIN_LINK_LIBS})
468+
endif()
458469
if(PAIMON_ENABLE_LUMINA)
459470
paimon_link_libraries_whole_archive(PAIMON_LUMINA_INDEX_STATIC_LINK_LIBS
460471
paimon_lumina_index_static)
@@ -505,6 +516,11 @@ config_summary_message()
505516
add_subdirectory(src/paimon)
506517
add_subdirectory(src/paimon/fs/local)
507518
add_subdirectory(src/paimon/fs/jindo)
519+
if(PAIMON_ENABLE_S3)
520+
find_package(CURL REQUIRED)
521+
endif()
522+
add_subdirectory(src/paimon/fs/object_store)
523+
add_subdirectory(src/paimon/fs/s3)
508524
add_subdirectory(src/paimon/format/blob)
509525
add_subdirectory(src/paimon/format/orc)
510526
add_subdirectory(src/paimon/format/parquet)

ci/scripts/build_paimon.sh

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -133,6 +133,7 @@ CMAKE_ARGS=(
133133
"-DPAIMON_BUILD_TESTS=ON"
134134
"-DPAIMON_ENABLE_LANCE=${ENABLE_LANCE}"
135135
"-DPAIMON_ENABLE_JINDO=ON"
136+
"-DPAIMON_ENABLE_S3=ON"
136137
"-DPAIMON_ENABLE_LUMINA=${ENABLE_LUMINA}"
137138
"-DPAIMON_ENABLE_LUCENE=ON"
138139
"-DPAIMON_ENABLE_TANTIVY=${ENABLE_TANTIVY}"

cmake_modules/BuildAwsAuth.cmake

Lines changed: 187 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,187 @@
1+
# Copyright 2026-present Alibaba Inc.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
include(ExternalProject)
16+
include(GNUInstallDirs)
17+
18+
function(paimon_add_s2n_project)
19+
if(DEFINED ENV{PAIMON_AWS_S2N_URL})
20+
set(S2N_URL "$ENV{PAIMON_AWS_S2N_URL}")
21+
elseif(EXISTS "${THIRDPARTY_DIR}/${PAIMON_AWS_S2N_PKG_NAME}")
22+
set(S2N_URL "${THIRDPARTY_DIR}/${PAIMON_AWS_S2N_PKG_NAME}")
23+
else()
24+
set(S2N_URL
25+
"${THIRDPARTY_MIRROR_URL}https://codeload.github.com/awslabs/s2n/zip/${PAIMON_AWS_S2N_BUILD_REVISION}"
26+
)
27+
endif()
28+
externalproject_add(s2n_ep
29+
URL ${S2N_URL}
30+
URL_HASH SHA256=${PAIMON_AWS_S2N_BUILD_SHA256_CHECKSUM}
31+
DOWNLOAD_EXTRACT_TIMESTAMP TRUE
32+
CMAKE_ARGS ${EP_COMMON_TOOLCHAIN}
33+
-DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
34+
-DCMAKE_INSTALL_PREFIX=${AWS_AUTH_PREFIX}
35+
-DCMAKE_INSTALL_LIBDIR=${AWS_AUTH_INSTALL_LIBDIR}
36+
-DCMAKE_PREFIX_PATH=${AWS_AUTH_PREFIX}
37+
-Dcrypto_INCLUDE_DIR=${OPENSSL_INCLUDE_DIR}
38+
-Dcrypto_LIBRARY=${OPENSSL_CRYPTO_LIBRARY}
39+
-DBUILD_SHARED_LIBS=OFF
40+
-DCMAKE_POSITION_INDEPENDENT_CODE=ON
41+
-DBUILD_TESTING=OFF
42+
-DS2N_INTERN_LIBCRYPTO=OFF
43+
BUILD_BYPRODUCTS ${AWS_AUTH_LIB_DIR}/libs2n.a
44+
${THIRDPARTY_LOG_OPTIONS})
45+
endfunction()
46+
47+
function(paimon_add_aws_c_project
48+
NAME
49+
VERSION
50+
CHECKSUM
51+
DEPENDS)
52+
string(REPLACE "aws-c-" "AWS_C_" URL_NAME "${NAME}")
53+
string(TOUPPER "${URL_NAME}" URL_NAME)
54+
set(URL_VAR "PAIMON_${URL_NAME}_URL")
55+
set(PKG_NAME_VAR "PAIMON_${URL_NAME}_PKG_NAME")
56+
if(DEFINED ENV{${URL_VAR}})
57+
set(URL "$ENV{${URL_VAR}}")
58+
elseif(EXISTS "${THIRDPARTY_DIR}/${${PKG_NAME_VAR}}")
59+
set(URL "${THIRDPARTY_DIR}/${${PKG_NAME_VAR}}")
60+
else()
61+
set(URL
62+
"${THIRDPARTY_MIRROR_URL}https://github.com/awslabs/${NAME}/archive/refs/tags/${VERSION}.tar.gz"
63+
)
64+
endif()
65+
set(AWS_PLATFORM_CMAKE_ARGS)
66+
list(APPEND
67+
AWS_PLATFORM_CMAKE_ARGS
68+
-DOPENSSL_INCLUDE_DIR=${OPENSSL_INCLUDE_DIR}
69+
-DOPENSSL_SSL_LIBRARY=${OPENSSL_SSL_LIBRARY}
70+
-DOPENSSL_CRYPTO_LIBRARY=${OPENSSL_CRYPTO_LIBRARY}
71+
-Dcrypto_INCLUDE_DIR=${OPENSSL_INCLUDE_DIR}
72+
-Dcrypto_LIBRARY=${OPENSSL_CRYPTO_LIBRARY})
73+
if(NAME STREQUAL "aws-c-cal")
74+
list(APPEND AWS_PLATFORM_CMAKE_ARGS -DUSE_OPENSSL=ON)
75+
endif()
76+
if(APPLE)
77+
list(APPEND AWS_PLATFORM_CMAKE_ARGS -DAWS_USE_SECITEM=ON)
78+
endif()
79+
# aws-c-io's installed package unconditionally looks for s2n on Unix even
80+
# when it was built with Apple's native TLS backend.
81+
if(APPLE AND (NAME STREQUAL "aws-c-http" OR NAME STREQUAL "aws-c-auth"))
82+
list(APPEND AWS_PLATFORM_CMAKE_ARGS -DBYO_CRYPTO=ON)
83+
endif()
84+
externalproject_add(${NAME}_ep
85+
URL ${URL}
86+
URL_HASH SHA256=${CHECKSUM}
87+
DOWNLOAD_EXTRACT_TIMESTAMP TRUE
88+
CMAKE_ARGS ${EP_COMMON_TOOLCHAIN}
89+
-DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
90+
-DCMAKE_INSTALL_PREFIX=${AWS_AUTH_PREFIX}
91+
-DCMAKE_INSTALL_LIBDIR=${AWS_AUTH_INSTALL_LIBDIR}
92+
-DCMAKE_PREFIX_PATH=${AWS_AUTH_PREFIX}
93+
-DBUILD_SHARED_LIBS=OFF
94+
-DCMAKE_POSITION_INDEPENDENT_CODE=ON
95+
-DBUILD_TESTING=OFF
96+
-DENABLE_TESTING=OFF
97+
${AWS_PLATFORM_CMAKE_ARGS}
98+
DEPENDS ${DEPENDS}
99+
BUILD_BYPRODUCTS ${AWS_AUTH_LIB_DIR}/lib${NAME}.a
100+
${THIRDPARTY_LOG_OPTIONS})
101+
endfunction()
102+
103+
function(build_aws_auth)
104+
set(AWS_AUTH_PREFIX
105+
"${CMAKE_CURRENT_BINARY_DIR}/aws-auth_ep-install"
106+
PARENT_SCOPE)
107+
set(AWS_AUTH_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/aws-auth_ep-install")
108+
set(AWS_AUTH_INSTALL_LIBDIR "${CMAKE_INSTALL_LIBDIR}")
109+
set(AWS_AUTH_LIB_DIR "${AWS_AUTH_PREFIX}/${AWS_AUTH_INSTALL_LIBDIR}")
110+
file(MAKE_DIRECTORY "${AWS_AUTH_PREFIX}/include")
111+
file(MAKE_DIRECTORY "${AWS_AUTH_LIB_DIR}")
112+
113+
find_package(OpenSSL REQUIRED)
114+
115+
set(AWS_IO_DEPENDS "aws-c-common_ep;aws-c-cal_ep")
116+
if(CMAKE_SYSTEM_NAME STREQUAL "Linux")
117+
paimon_add_s2n_project()
118+
list(APPEND AWS_IO_DEPENDS s2n_ep)
119+
endif()
120+
121+
paimon_add_aws_c_project(aws-c-common ${PAIMON_AWS_C_COMMON_BUILD_VERSION}
122+
${PAIMON_AWS_C_COMMON_BUILD_SHA256_CHECKSUM} "")
123+
paimon_add_aws_c_project(aws-c-sdkutils ${PAIMON_AWS_C_SDKUTILS_BUILD_VERSION}
124+
${PAIMON_AWS_C_SDKUTILS_BUILD_SHA256_CHECKSUM}
125+
aws-c-common_ep)
126+
paimon_add_aws_c_project(aws-c-cal ${PAIMON_AWS_C_CAL_BUILD_VERSION}
127+
${PAIMON_AWS_C_CAL_BUILD_SHA256_CHECKSUM} aws-c-common_ep)
128+
paimon_add_aws_c_project(aws-c-compression ${PAIMON_AWS_C_COMPRESSION_BUILD_VERSION}
129+
${PAIMON_AWS_C_COMPRESSION_BUILD_SHA256_CHECKSUM}
130+
aws-c-common_ep)
131+
paimon_add_aws_c_project(aws-c-io ${PAIMON_AWS_C_IO_BUILD_VERSION}
132+
${PAIMON_AWS_C_IO_BUILD_SHA256_CHECKSUM} "${AWS_IO_DEPENDS}")
133+
paimon_add_aws_c_project(aws-c-http ${PAIMON_AWS_C_HTTP_BUILD_VERSION}
134+
${PAIMON_AWS_C_HTTP_BUILD_SHA256_CHECKSUM}
135+
"aws-c-common_ep;aws-c-io_ep;aws-c-compression_ep")
136+
paimon_add_aws_c_project(aws-c-auth
137+
${PAIMON_AWS_C_AUTH_BUILD_VERSION}
138+
${PAIMON_AWS_C_AUTH_BUILD_SHA256_CHECKSUM}
139+
"aws-c-common_ep;aws-c-sdkutils_ep;aws-c-cal_ep;aws-c-io_ep;aws-c-http_ep"
140+
)
141+
142+
find_package(Threads REQUIRED)
143+
set(AWS_AUTH_PLATFORM_LIBS Threads::Threads ${CMAKE_DL_LIBS})
144+
if(APPLE)
145+
list(APPEND
146+
AWS_AUTH_PLATFORM_LIBS
147+
"-framework Security"
148+
"-framework CoreFoundation"
149+
"-framework Network")
150+
endif()
151+
set(AWS_AUTH_TLS_LIBS)
152+
if(CMAKE_SYSTEM_NAME STREQUAL "Linux")
153+
list(APPEND AWS_AUTH_TLS_LIBS "${AWS_AUTH_LIB_DIR}/libs2n.a")
154+
endif()
155+
set(AWS_AUTH_RUNTIME_LIBS
156+
"${AWS_AUTH_LIB_DIR}/libaws-c-http.a"
157+
"${AWS_AUTH_LIB_DIR}/libaws-c-io.a"
158+
"${AWS_AUTH_LIB_DIR}/libaws-c-compression.a"
159+
"${AWS_AUTH_LIB_DIR}/libaws-c-cal.a"
160+
"${AWS_AUTH_LIB_DIR}/libaws-c-sdkutils.a"
161+
"${AWS_AUTH_LIB_DIR}/libaws-c-common.a"
162+
${AWS_AUTH_TLS_LIBS})
163+
if(NOT APPLE)
164+
list(PREPEND AWS_AUTH_RUNTIME_LIBS "-Wl,--start-group")
165+
list(APPEND AWS_AUTH_RUNTIME_LIBS "-Wl,--end-group")
166+
endif()
167+
# Keep OpenSSL after the static AWS archives. In embedded builds CMake can
168+
# resolve these targets to static archives, where link order matters.
169+
list(APPEND AWS_AUTH_RUNTIME_LIBS "${OPENSSL_SSL_LIBRARY}"
170+
"${OPENSSL_CRYPTO_LIBRARY}")
171+
add_library(aws_auth_minimal STATIC IMPORTED GLOBAL)
172+
set_target_properties(aws_auth_minimal
173+
PROPERTIES IMPORTED_LOCATION
174+
"${AWS_AUTH_LIB_DIR}/libaws-c-auth.a"
175+
INTERFACE_INCLUDE_DIRECTORIES
176+
"${AWS_AUTH_PREFIX}/include"
177+
INTERFACE_LINK_LIBRARIES
178+
"${AWS_AUTH_RUNTIME_LIBS};${AWS_AUTH_PLATFORM_LIBS}")
179+
add_dependencies(aws_auth_minimal aws-c-auth_ep)
180+
181+
set(AWS_AUTH_INCLUDE_DIR
182+
"${AWS_AUTH_PREFIX}/include"
183+
PARENT_SCOPE)
184+
set(AWS_AUTH_LIB_DIR
185+
"${AWS_AUTH_LIB_DIR}"
186+
PARENT_SCOPE)
187+
endfunction()

cmake_modules/ThirdpartyToolchain.cmake

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@ foreach(_VERSION_ENTRY ${TOOLCHAIN_VERSIONS_TXT})
4949
if(NOT
5050
((_VERSION_ENTRY MATCHES "^[^#][A-Za-z0-9-_]+_VERSION=")
5151
OR (_VERSION_ENTRY MATCHES "^[^#][A-Za-z0-9-_]+_CHECKSUM=")
52+
OR (_VERSION_ENTRY MATCHES "^[^#][A-Za-z0-9-_]+_REVISION=")
5253
OR (_VERSION_ENTRY MATCHES "^[^#][A-Za-z0-9-_]+_PKG_NAME=")))
5354
continue()
5455
endif()
@@ -1874,6 +1875,10 @@ if(PAIMON_ENABLE_JINDO)
18741875
build_jindosdk_c()
18751876
build_jindosdk_nextarch()
18761877
endif()
1878+
if(PAIMON_ENABLE_S3)
1879+
include(BuildAwsAuth)
1880+
build_aws_auth()
1881+
endif()
18771882
if(PAIMON_ENABLE_LUMINA)
18781883
build_lumina()
18791884
endif()

cmake_modules/arrow.diff

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -415,6 +415,14 @@ diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/Thi
415415
# and crosscompiling emulator (for try_run() )
416416
if(CMAKE_CROSSCOMPILING_EMULATOR)
417417
string(REPLACE ";" ${EP_LIST_SEPARATOR} EP_CMAKE_CROSSCOMPILING_EMULATOR
418+
@@ -1720,6 +1725,7 @@ macro(build_thrift)
419+
-DWITH_JAVASCRIPT=OFF
420+
-DWITH_LIBEVENT=OFF
421+
-DWITH_NODEJS=OFF
422+
+ -DWITH_OPENSSL=OFF
423+
-DWITH_PYTHON=OFF
424+
-DWITH_QT5=OFF
425+
-DWITH_ZLIB=OFF)
418426
diff --git a/cpp/cmake_modules/BuildUtils.cmake b/cpp/cmake_modules/BuildUtils.cmake
419427
--- a/cpp/cmake_modules/BuildUtils.cmake
420428
+++ b/cpp/cmake_modules/BuildUtils.cmake
Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
# Copyright 2026-present Alibaba Inc.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
if(PAIMON_ENABLE_S3)
16+
add_paimon_lib(paimon_object_store
17+
SOURCES
18+
http_client.cpp
19+
object_store_file_system.cpp
20+
DEPENDENCIES
21+
paimon_shared
22+
CURL::libcurl
23+
STATIC_LINK_LIBS
24+
CURL::libcurl
25+
fmt
26+
SHARED_LINK_LIBS
27+
paimon_shared
28+
SHARED_LINK_FLAGS
29+
${PAIMON_VERSION_SCRIPT_FLAGS})
30+
31+
if(PAIMON_BUILD_TESTS)
32+
add_paimon_test(object_store_file_system_test
33+
SOURCES
34+
object_store_file_system_test.cpp
35+
STATIC_LINK_LIBS
36+
paimon_shared
37+
test_utils_static
38+
paimon_object_store_static
39+
${GTEST_LINK_TOOLCHAIN})
40+
endif()
41+
endif()

0 commit comments

Comments
 (0)