feat(fs): add lightweight object-store and S3 filesystem support

.github/workflows/build_and_test.yaml

@@ -71,6 +71,10 @@ jobs:
         if: ${{ !matrix.skip_rust }}
         shell: bash
         run: ci/scripts/setup_rust.sh
+      - name: Install HTTP and TLS development dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y libcurl4-openssl-dev libssl-dev
       - name: Build Paimon
         shell: bash
         env:

.github/workflows/gcc8_test.yaml

@@ -42,7 +42,7 @@ jobs:
       - name: Install dependencies
         run: |
           apt-get update
-          DEBIAN_FRONTEND=noninteractive apt-get install -y gcc-8 g++-8 ninja-build git git-lfs tar curl tzdata zip unzip pkg-config build-essential python3-dev gdb sudo
+          DEBIAN_FRONTEND=noninteractive apt-get install -y gcc-8 g++-8 ninja-build git git-lfs tar curl libcurl4-openssl-dev libssl-dev tzdata zip unzip pkg-config build-essential python3-dev gdb sudo
           curl -L -O https://github.com/Kitware/CMake/releases/download/v3.28.3/cmake-3.28.3-linux-x86_64.tar.gz
           tar -zxvf cmake-3.28.3-linux-x86_64.tar.gz -C /usr/local --strip-components=1
           rm cmake-3.28.3-linux-x86_64.tar.gz

CMakeLists.txt

@@ -57,6 +57,7 @@ option(PAIMON_ENABLE_AVRO "Whether to enable avro file format" ON)
 option(PAIMON_ENABLE_ORC "Whether to enable orc file format" ON)
 option(PAIMON_ENABLE_LANCE "Whether to enable lance file format" OFF)
 option(PAIMON_ENABLE_JINDO "Whether to enable jindo file system" OFF)
+option(PAIMON_ENABLE_S3 "Whether to enable S3 file system" OFF)
 option(PAIMON_ENABLE_NETWORK_TESTS
        "Whether to enable tests that access real remote services over the network" OFF)
 option(PAIMON_ENABLE_LUMINA "Whether to enable lumina vector index" OFF)
@@ -72,6 +73,9 @@ endif()
 if(PAIMON_ENABLE_JINDO)
     add_definitions(-DPAIMON_ENABLE_JINDO)
 endif()
+if(PAIMON_ENABLE_S3)
+    add_definitions(-DPAIMON_ENABLE_S3)
+endif()
 if(PAIMON_ENABLE_NETWORK_TESTS)
     if(NOT PAIMON_BUILD_TESTS)
         message(FATAL_ERROR "PAIMON_ENABLE_NETWORK_TESTS requires PAIMON_BUILD_TESTS=ON")
@@ -455,6 +459,13 @@ if(PAIMON_BUILD_TESTS)
                                            paimon_jindo_file_system_shared)
         list(APPEND TEST_STATIC_LINK_LIBS ${TEST_PLUGIN_LINK_LIBS})
     endif()
+    if(PAIMON_ENABLE_S3)
+        paimon_link_libraries_whole_archive(PAIMON_S3_FILE_SYSTEM_STATIC_LINK_LIBS
+                                            paimon_s3_file_system_static)
+        paimon_link_libraries_no_as_needed(TEST_PLUGIN_LINK_LIBS
+                                           paimon_s3_file_system_shared)
+        list(APPEND TEST_STATIC_LINK_LIBS ${TEST_PLUGIN_LINK_LIBS})
+    endif()
     if(PAIMON_ENABLE_LUMINA)
         paimon_link_libraries_whole_archive(PAIMON_LUMINA_INDEX_STATIC_LINK_LIBS
                                             paimon_lumina_index_static)
@@ -505,6 +516,11 @@ config_summary_message()
 add_subdirectory(src/paimon)
 add_subdirectory(src/paimon/fs/local)
 add_subdirectory(src/paimon/fs/jindo)
+if(PAIMON_ENABLE_S3)
+    find_package(CURL REQUIRED)
+endif()
+add_subdirectory(src/paimon/fs/object_store)
+add_subdirectory(src/paimon/fs/s3)
 add_subdirectory(src/paimon/format/blob)
 add_subdirectory(src/paimon/format/orc)
 add_subdirectory(src/paimon/format/parquet)

ci/scripts/build_paimon.sh

@@ -133,6 +133,7 @@ CMAKE_ARGS=(
     "-DPAIMON_BUILD_TESTS=ON"
     "-DPAIMON_ENABLE_LANCE=${ENABLE_LANCE}"
     "-DPAIMON_ENABLE_JINDO=ON"
+    "-DPAIMON_ENABLE_S3=ON"
     "-DPAIMON_ENABLE_LUMINA=${ENABLE_LUMINA}"
     "-DPAIMON_ENABLE_LUCENE=ON"
     "-DPAIMON_ENABLE_TANTIVY=${ENABLE_TANTIVY}"

cmake_modules/BuildAwsAuth.cmake

@@ -0,0 +1,187 @@
+# Copyright 2026-present Alibaba Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+include(ExternalProject)
+include(GNUInstallDirs)
+
+function(paimon_add_s2n_project)
+    if(DEFINED ENV{PAIMON_AWS_S2N_URL})
+        set(S2N_URL "$ENV{PAIMON_AWS_S2N_URL}")
+    elseif(EXISTS "${THIRDPARTY_DIR}/${PAIMON_AWS_S2N_PKG_NAME}")
+        set(S2N_URL "${THIRDPARTY_DIR}/${PAIMON_AWS_S2N_PKG_NAME}")
+    else()
+        set(S2N_URL
+            "${THIRDPARTY_MIRROR_URL}https://codeload.github.com/awslabs/s2n/zip/${PAIMON_AWS_S2N_BUILD_REVISION}"
+        )
+    endif()
+    externalproject_add(s2n_ep
+                        URL ${S2N_URL}
+                        URL_HASH SHA256=${PAIMON_AWS_S2N_BUILD_SHA256_CHECKSUM}
+                        DOWNLOAD_EXTRACT_TIMESTAMP TRUE
+                        CMAKE_ARGS ${EP_COMMON_TOOLCHAIN}
+                                   -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
+                                   -DCMAKE_INSTALL_PREFIX=${AWS_AUTH_PREFIX}
+                                   -DCMAKE_INSTALL_LIBDIR=${AWS_AUTH_INSTALL_LIBDIR}
+                                   -DCMAKE_PREFIX_PATH=${AWS_AUTH_PREFIX}
+                                   -Dcrypto_INCLUDE_DIR=${OPENSSL_INCLUDE_DIR}
+                                   -Dcrypto_LIBRARY=${OPENSSL_CRYPTO_LIBRARY}
+                                   -DBUILD_SHARED_LIBS=OFF
+                                   -DCMAKE_POSITION_INDEPENDENT_CODE=ON
+                                   -DBUILD_TESTING=OFF
+                                   -DS2N_INTERN_LIBCRYPTO=OFF
+                        BUILD_BYPRODUCTS ${AWS_AUTH_LIB_DIR}/libs2n.a
+                                         ${THIRDPARTY_LOG_OPTIONS})
+endfunction()
+
+function(paimon_add_aws_c_project
+         NAME
+         VERSION
+         CHECKSUM
+         DEPENDS)
+    string(REPLACE "aws-c-" "AWS_C_" URL_NAME "${NAME}")
+    string(TOUPPER "${URL_NAME}" URL_NAME)
+    set(URL_VAR "PAIMON_${URL_NAME}_URL")
+    set(PKG_NAME_VAR "PAIMON_${URL_NAME}_PKG_NAME")
+    if(DEFINED ENV{${URL_VAR}})
+        set(URL "$ENV{${URL_VAR}}")
+    elseif(EXISTS "${THIRDPARTY_DIR}/${${PKG_NAME_VAR}}")
+        set(URL "${THIRDPARTY_DIR}/${${PKG_NAME_VAR}}")
+    else()
+        set(URL
+            "${THIRDPARTY_MIRROR_URL}https://github.com/awslabs/${NAME}/archive/refs/tags/${VERSION}.tar.gz"
+        )
+    endif()
+    set(AWS_PLATFORM_CMAKE_ARGS)
+    list(APPEND
+         AWS_PLATFORM_CMAKE_ARGS
+         -DOPENSSL_INCLUDE_DIR=${OPENSSL_INCLUDE_DIR}
+         -DOPENSSL_SSL_LIBRARY=${OPENSSL_SSL_LIBRARY}
+         -DOPENSSL_CRYPTO_LIBRARY=${OPENSSL_CRYPTO_LIBRARY}
+         -Dcrypto_INCLUDE_DIR=${OPENSSL_INCLUDE_DIR}
+         -Dcrypto_LIBRARY=${OPENSSL_CRYPTO_LIBRARY})
+    if(NAME STREQUAL "aws-c-cal")
+        list(APPEND AWS_PLATFORM_CMAKE_ARGS -DUSE_OPENSSL=ON)
+    endif()
+    if(APPLE)
+        list(APPEND AWS_PLATFORM_CMAKE_ARGS -DAWS_USE_SECITEM=ON)
+    endif()
+    # aws-c-io's installed package unconditionally looks for s2n on Unix even
+    # when it was built with Apple's native TLS backend.
+    if(APPLE AND (NAME STREQUAL "aws-c-http" OR NAME STREQUAL "aws-c-auth"))
+        list(APPEND AWS_PLATFORM_CMAKE_ARGS -DBYO_CRYPTO=ON)
+    endif()
+    externalproject_add(${NAME}_ep
+                        URL ${URL}
+                        URL_HASH SHA256=${CHECKSUM}
+                        DOWNLOAD_EXTRACT_TIMESTAMP TRUE
+                        CMAKE_ARGS ${EP_COMMON_TOOLCHAIN}
+                                   -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
+                                   -DCMAKE_INSTALL_PREFIX=${AWS_AUTH_PREFIX}
+                                   -DCMAKE_INSTALL_LIBDIR=${AWS_AUTH_INSTALL_LIBDIR}
+                                   -DCMAKE_PREFIX_PATH=${AWS_AUTH_PREFIX}
+                                   -DBUILD_SHARED_LIBS=OFF
+                                   -DCMAKE_POSITION_INDEPENDENT_CODE=ON
+                                   -DBUILD_TESTING=OFF
+                                   -DENABLE_TESTING=OFF
+                                   ${AWS_PLATFORM_CMAKE_ARGS}
+                        DEPENDS ${DEPENDS}
+                        BUILD_BYPRODUCTS ${AWS_AUTH_LIB_DIR}/lib${NAME}.a
+                                         ${THIRDPARTY_LOG_OPTIONS})
+endfunction()
+
+function(build_aws_auth)
+    set(AWS_AUTH_PREFIX
+        "${CMAKE_CURRENT_BINARY_DIR}/aws-auth_ep-install"
+        PARENT_SCOPE)
+    set(AWS_AUTH_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/aws-auth_ep-install")
+    set(AWS_AUTH_INSTALL_LIBDIR "${CMAKE_INSTALL_LIBDIR}")
+    set(AWS_AUTH_LIB_DIR "${AWS_AUTH_PREFIX}/${AWS_AUTH_INSTALL_LIBDIR}")
+    file(MAKE_DIRECTORY "${AWS_AUTH_PREFIX}/include")
+    file(MAKE_DIRECTORY "${AWS_AUTH_LIB_DIR}")
+
+    find_package(OpenSSL REQUIRED)
+
+    set(AWS_IO_DEPENDS "aws-c-common_ep;aws-c-cal_ep")
+    if(CMAKE_SYSTEM_NAME STREQUAL "Linux")
+        paimon_add_s2n_project()
+        list(APPEND AWS_IO_DEPENDS s2n_ep)
+    endif()
+
+    paimon_add_aws_c_project(aws-c-common ${PAIMON_AWS_C_COMMON_BUILD_VERSION}
+                             ${PAIMON_AWS_C_COMMON_BUILD_SHA256_CHECKSUM} "")
+    paimon_add_aws_c_project(aws-c-sdkutils ${PAIMON_AWS_C_SDKUTILS_BUILD_VERSION}
+                             ${PAIMON_AWS_C_SDKUTILS_BUILD_SHA256_CHECKSUM}
+                             aws-c-common_ep)
+    paimon_add_aws_c_project(aws-c-cal ${PAIMON_AWS_C_CAL_BUILD_VERSION}
+                             ${PAIMON_AWS_C_CAL_BUILD_SHA256_CHECKSUM} aws-c-common_ep)
+    paimon_add_aws_c_project(aws-c-compression ${PAIMON_AWS_C_COMPRESSION_BUILD_VERSION}
+                             ${PAIMON_AWS_C_COMPRESSION_BUILD_SHA256_CHECKSUM}
+                             aws-c-common_ep)
+    paimon_add_aws_c_project(aws-c-io ${PAIMON_AWS_C_IO_BUILD_VERSION}
+                             ${PAIMON_AWS_C_IO_BUILD_SHA256_CHECKSUM} "${AWS_IO_DEPENDS}")
+    paimon_add_aws_c_project(aws-c-http ${PAIMON_AWS_C_HTTP_BUILD_VERSION}
+                             ${PAIMON_AWS_C_HTTP_BUILD_SHA256_CHECKSUM}
+                             "aws-c-common_ep;aws-c-io_ep;aws-c-compression_ep")
+    paimon_add_aws_c_project(aws-c-auth
+                             ${PAIMON_AWS_C_AUTH_BUILD_VERSION}
+                             ${PAIMON_AWS_C_AUTH_BUILD_SHA256_CHECKSUM}
+                             "aws-c-common_ep;aws-c-sdkutils_ep;aws-c-cal_ep;aws-c-io_ep;aws-c-http_ep"
+    )
+
+    find_package(Threads REQUIRED)
+    set(AWS_AUTH_PLATFORM_LIBS Threads::Threads ${CMAKE_DL_LIBS})
+    if(APPLE)
+        list(APPEND
+             AWS_AUTH_PLATFORM_LIBS
+             "-framework Security"
+             "-framework CoreFoundation"
+             "-framework Network")
+    endif()
+    set(AWS_AUTH_TLS_LIBS)
+    if(CMAKE_SYSTEM_NAME STREQUAL "Linux")
+        list(APPEND AWS_AUTH_TLS_LIBS "${AWS_AUTH_LIB_DIR}/libs2n.a")
+    endif()
+    set(AWS_AUTH_RUNTIME_LIBS
+        "${AWS_AUTH_LIB_DIR}/libaws-c-http.a"
+        "${AWS_AUTH_LIB_DIR}/libaws-c-io.a"
+        "${AWS_AUTH_LIB_DIR}/libaws-c-compression.a"
+        "${AWS_AUTH_LIB_DIR}/libaws-c-cal.a"
+        "${AWS_AUTH_LIB_DIR}/libaws-c-sdkutils.a"
+        "${AWS_AUTH_LIB_DIR}/libaws-c-common.a"
+        ${AWS_AUTH_TLS_LIBS})
+    if(NOT APPLE)
+        list(PREPEND AWS_AUTH_RUNTIME_LIBS "-Wl,--start-group")
+        list(APPEND AWS_AUTH_RUNTIME_LIBS "-Wl,--end-group")
+    endif()
+    # Keep OpenSSL after the static AWS archives. In embedded builds CMake can
+    # resolve these targets to static archives, where link order matters.
+    list(APPEND AWS_AUTH_RUNTIME_LIBS "${OPENSSL_SSL_LIBRARY}"
+         "${OPENSSL_CRYPTO_LIBRARY}")
+    add_library(aws_auth_minimal STATIC IMPORTED GLOBAL)
+    set_target_properties(aws_auth_minimal
+                          PROPERTIES IMPORTED_LOCATION
+                                     "${AWS_AUTH_LIB_DIR}/libaws-c-auth.a"
+                                     INTERFACE_INCLUDE_DIRECTORIES
+                                     "${AWS_AUTH_PREFIX}/include"
+                                     INTERFACE_LINK_LIBRARIES
+                                     "${AWS_AUTH_RUNTIME_LIBS};${AWS_AUTH_PLATFORM_LIBS}")
+    add_dependencies(aws_auth_minimal aws-c-auth_ep)
+
+    set(AWS_AUTH_INCLUDE_DIR
+        "${AWS_AUTH_PREFIX}/include"
+        PARENT_SCOPE)
+    set(AWS_AUTH_LIB_DIR
+        "${AWS_AUTH_LIB_DIR}"
+        PARENT_SCOPE)
+endfunction()

cmake_modules/ThirdpartyToolchain.cmake

@@ -49,6 +49,7 @@ foreach(_VERSION_ENTRY ${TOOLCHAIN_VERSIONS_TXT})
     if(NOT
        ((_VERSION_ENTRY MATCHES "^[^#][A-Za-z0-9-_]+_VERSION=")
         OR (_VERSION_ENTRY MATCHES "^[^#][A-Za-z0-9-_]+_CHECKSUM=")
+        OR (_VERSION_ENTRY MATCHES "^[^#][A-Za-z0-9-_]+_REVISION=")
         OR (_VERSION_ENTRY MATCHES "^[^#][A-Za-z0-9-_]+_PKG_NAME=")))
         continue()
     endif()
@@ -1877,6 +1878,10 @@ if(PAIMON_ENABLE_JINDO)
     build_jindosdk_c()
     build_jindosdk_nextarch()
 endif()
+if(PAIMON_ENABLE_S3)
+    include(BuildAwsAuth)
+    build_aws_auth()
+endif()
 if(PAIMON_ENABLE_LUMINA)
     build_lumina()
 endif()

cmake_modules/arrow.diff

@@ -415,6 +415,14 @@ diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/Thi
  # and crosscompiling emulator (for try_run() )
  if(CMAKE_CROSSCOMPILING_EMULATOR)
    string(REPLACE ";" ${EP_LIST_SEPARATOR} EP_CMAKE_CROSSCOMPILING_EMULATOR
+@@ -1720,6 +1725,7 @@ macro(build_thrift)
+       -DWITH_JAVASCRIPT=OFF
+       -DWITH_LIBEVENT=OFF
+       -DWITH_NODEJS=OFF
++      -DWITH_OPENSSL=OFF
+       -DWITH_PYTHON=OFF
+       -DWITH_QT5=OFF
+       -DWITH_ZLIB=OFF)
 diff --git a/cpp/cmake_modules/BuildUtils.cmake b/cpp/cmake_modules/BuildUtils.cmake
 --- a/cpp/cmake_modules/BuildUtils.cmake
 +++ b/cpp/cmake_modules/BuildUtils.cmake

src/paimon/fs/object_store/CMakeLists.txt

@@ -0,0 +1,41 @@
+# Copyright 2026-present Alibaba Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if(PAIMON_ENABLE_S3)
+    add_paimon_lib(paimon_object_store
+                   SOURCES
+                   http_client.cpp
+                   object_store_file_system.cpp
+                   DEPENDENCIES
+                   paimon_shared
+                   CURL::libcurl
+                   STATIC_LINK_LIBS
+                   CURL::libcurl
+                   fmt
+                   SHARED_LINK_LIBS
+                   paimon_shared
+                   SHARED_LINK_FLAGS
+                   ${PAIMON_VERSION_SCRIPT_FLAGS})
+
+    if(PAIMON_BUILD_TESTS)
+        add_paimon_test(object_store_file_system_test
+                        SOURCES
+                        object_store_file_system_test.cpp
+                        STATIC_LINK_LIBS
+                        paimon_shared
+                        test_utils_static
+                        paimon_object_store_static
+                        ${GTEST_LINK_TOOLCHAIN})
+    endif()
+endif()