PullRequest: 1 feat(zdfs): 新增zdfs文件系统支持

Merge branch 'yzh-dfs of git@code.alipay.com:antflink/paimon-cpp.git into main

https://code.alipay.com/antflink/paimon-cpp/pull_requests/1


Reviewed-by: 量子 <zhangchaoming.zcm@antgroup.com>

Author: Eyizoha

CMakeLists.txt

@@ -52,6 +52,7 @@ option(PAIMON_ENABLE_AVRO "Whether to enable avro file format" ON)
 option(PAIMON_ENABLE_ORC "Whether to enable orc file format" ON)
 option(PAIMON_ENABLE_LANCE "Whether to enable lance file format" OFF)
 option(PAIMON_ENABLE_JINDO "Whether to enable jindo file system" OFF)
+option(PAIMON_ENABLE_ZDFS "Whether to enable zdfs file system" OFF)
 option(PAIMON_ENABLE_LUMINA "Whether to enable lumina vector index" ON)
 option(PAIMON_ENABLE_LUCENE "Whether to enable lucene index" ON)
 
@@ -64,6 +65,12 @@ endif()
 if(PAIMON_ENABLE_JINDO)
     add_definitions(-DPAIMON_ENABLE_JINDO)
 endif()
+if(PAIMON_ENABLE_ZDFS)
+    if(PAIMON_USE_CXX11_ABI)
+        message(FATAL_ERROR "ZDFS cannot be enabled with C++11 ABI on")
+    endif()
+    add_definitions(-DPAIMON_ENABLE_ZDFS)
+endif()
 if(PAIMON_USE_CXX11_ABI)
     add_definitions(-D_GLIBCXX_USE_CXX11_ABI=1)
 else()
@@ -395,6 +402,11 @@ if(PAIMON_BUILD_TESTS)
         list(APPEND TEST_STATIC_LINK_LIBS paimon_jindo_file_system_shared)
         list(APPEND TEST_STATIC_LINK_LIBS "-Wl,--as-needed")
     endif()
+    if(PAIMON_ENABLE_ZDFS)
+        list(APPEND TEST_STATIC_LINK_LIBS "-Wl,--no-as-needed")
+        list(APPEND TEST_STATIC_LINK_LIBS paimon_zdfs_file_system_shared)
+        list(APPEND TEST_STATIC_LINK_LIBS "-Wl,--as-needed")
+    endif()
     if(PAIMON_ENABLE_LUMINA)
         list(APPEND TEST_STATIC_LINK_LIBS "-Wl,--no-as-needed")
         list(APPEND TEST_STATIC_LINK_LIBS paimon_lumina_index_shared)
@@ -426,6 +438,7 @@ config_summary_message()
 add_subdirectory(src/paimon)
 add_subdirectory(src/paimon/fs/local)
 add_subdirectory(src/paimon/fs/jindo)
+add_subdirectory(src/paimon/fs/zdfs)
 add_subdirectory(src/paimon/format/blob)
 add_subdirectory(src/paimon/format/orc)
 add_subdirectory(src/paimon/format/parquet)

PaimonConfig.cmake.in

@@ -120,6 +120,18 @@ set_target_properties(paimon_jindo_file_system_static PROPERTIES
     INTERFACE_INCLUDE_DIRECTORIES "@CMAKE_INSTALL_PREFIX@/include"
 )
 
+# paimon_zdfs_file_system
+add_library(paimon_zdfs_file_system_shared SHARED IMPORTED)
+set_target_properties(paimon_zdfs_file_system_shared PROPERTIES
+    IMPORTED_LOCATION "@CMAKE_INSTALL_PREFIX@/@CMAKE_INSTALL_LIBDIR@/libpaimon_zdfs_file_system.so"
+    INTERFACE_INCLUDE_DIRECTORIES "@CMAKE_INSTALL_PREFIX@/include"
+)
+add_library(paimon_zdfs_file_system_static STATIC IMPORTED)
+set_target_properties(paimon_zdfs_file_system_static PROPERTIES
+    IMPORTED_LOCATION "@CMAKE_INSTALL_PREFIX@/@CMAKE_INSTALL_LIBDIR@/libpaimon_zdfs_file_system.a"
+    INTERFACE_INCLUDE_DIRECTORIES "@CMAKE_INSTALL_PREFIX@/include"
+)
+
 # paimon_lance_file_format
 add_library(paimon_lance_file_format_shared SHARED IMPORTED)
 set_target_properties(paimon_lance_file_format_shared PROPERTIES

cmake_modules/ThirdpartyToolchain.cmake

@@ -160,6 +160,18 @@ else()
     endif()
 endif()
 
+if(DEFINED ENV{PAIMON_GFLAGS_URL})
+    set(GFLAGS_SOURCE_URL "$ENV{PAIMON_GFLAGS_URL}")
+else()
+    if(EXISTS "${THIRDPARTY_DIR}/${PAIMON_GFLAGS_PKG_NAME}")
+        set_urls(GFLAGS_SOURCE_URL "${THIRDPARTY_DIR}/${PAIMON_GFLAGS_PKG_NAME}")
+    else()
+        set_urls(GFLAGS_SOURCE_URL
+                 "${THIRDPARTY_MIRROR_URL}https://github.com/gflags/gflags/archive/refs/tags/${PAIMON_GFLAGS_BUILD_VERSION}.tar.gz"
+        )
+    endif()
+endif()
+
 if(DEFINED ENV{PAIMON_ZLIB_URL})
     set(ZLIB_SOURCE_URL "$ENV{PAIMON_ZLIB_URL}")
 else()
@@ -280,6 +292,18 @@ else()
     endif()
 endif()
 
+if(DEFINED ENV{PAIMON_ZDFS_URL})
+    set(ZDFS_SOURCE_URL "$ENV{PAIMON_ZDFS_URL}")
+else()
+    if(EXISTS "${THIRDPARTY_DIR}/${PAIMON_ZDFS_PKG_NAME}")
+        set_urls(ZDFS_SOURCE_URL "${THIRDPARTY_DIR}/${PAIMON_ZDFS_PKG_NAME}")
+    else()
+        set_urls(ZDFS_SOURCE_URL
+                 "http://beiming.alibaba-inc.com/beiming/services/zdfs/${PAIMON_ZDFS_BUILD_VERSION}/apps/zdfs-dfs-lib"
+        )
+    endif()
+endif()
+
 set(EP_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
 set(EP_C_FLAGS "${CMAKE_C_FLAGS}")
 string(REPLACE "-Wglobal-constructors" "" EP_CXX_FLAGS ${EP_CXX_FLAGS})
@@ -800,6 +824,50 @@ macro(build_jindosdk_nextarch)
     add_dependencies(jindosdk::nextarch jindosdk-nextarch_ep)
 endmacro()
 
+macro(build_zdfs)
+    message(STATUS "Building zdfs from precompiled package")
+
+    set(ZDFS_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/zdfs_ep-install")
+    set(ZDFS_HOME "${ZDFS_PREFIX}")
+    set(ZDFS_INCLUDE_DIR "${ZDFS_PREFIX}/include")
+    set(ZDFS_LIB_DIR "${ZDFS_PREFIX}/lib")
+    set(ZDFS_STATIC_LIB "${ZDFS_LIB_DIR}/libzdfs.a")
+    set(ZDFS_DYNAMIC_LIB "${ZDFS_LIB_DIR}/libzdfs.so")
+
+    # Extract and install zdfs from precompiled package
+    externalproject_add(zdfs_ep
+                        URL ${ZDFS_SOURCE_URL}
+                        URL_HASH "SHA256=${PAIMON_ZDFS_BUILD_SHA256_CHECKSUM}"
+                        ${THIRDPARTY_LOG_OPTIONS}
+                        CONFIGURE_COMMAND ""
+                        BUILD_COMMAND ""
+                        INSTALL_COMMAND bash -c
+                                        "cp -r <SOURCE_DIR>/include/* ${ZDFS_INCLUDE_DIR}"
+                        COMMAND bash -c "cp -r <SOURCE_DIR>/lib/libzdfs.* ${ZDFS_LIB_DIR}"
+                        BUILD_BYPRODUCTS "${ZDFS_STATIC_LIB}" "${ZDFS_DYNAMIC_LIB}")
+
+    # The include directory must exist before it is referenced by a target.
+    file(MAKE_DIRECTORY "${ZDFS_INCLUDE_DIR}")
+    file(MAKE_DIRECTORY "${ZDFS_LIB_DIR}")
+
+    add_library(zdfs::sdk_static STATIC IMPORTED)
+    set_target_properties(zdfs::sdk_static
+                          PROPERTIES IMPORTED_LOCATION "${ZDFS_STATIC_LIB}"
+                                     INTERFACE_INCLUDE_DIRECTORIES "${ZDFS_INCLUDE_DIR}")
+    add_dependencies(zdfs::sdk_static zdfs_ep)
+
+    add_library(zdfs::sdk_shared SHARED IMPORTED)
+    set_target_properties(zdfs::sdk_shared
+                          PROPERTIES IMPORTED_LOCATION "${ZDFS_DYNAMIC_LIB}"
+                                     INTERFACE_INCLUDE_DIRECTORIES "${ZDFS_INCLUDE_DIR}")
+    add_dependencies(zdfs::sdk_shared zdfs_ep)
+
+    install(DIRECTORY "${ZDFS_LIB_DIR}/"
+            DESTINATION ${CMAKE_INSTALL_LIBDIR}
+            FILES_MATCHING
+            PATTERN "libzdfs.*")
+endmacro()
+
 macro(build_protobuf)
     message(STATUS "Building protobuf from source")
     set(PROTOBUF_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/protobuf_ep-install")
@@ -1339,6 +1407,44 @@ macro(build_glog)
     endif()
 endmacro()
 
+macro(build_gflags)
+    message(STATUS "Building gflags from source")
+    set(GFLAGS_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/gflags_ep-install")
+    set(GFLAGS_INCLUDE_DIR "${GFLAGS_PREFIX}/include")
+    if(${UPPERCASE_BUILD_TYPE} STREQUAL "DEBUG")
+        set(GLOG_LIB_SUFFIX "_debug")
+    else()
+        set(GLOG_LIB_SUFFIX "")
+    endif()
+    set(GFLAGS_STATIC_LIB "${GFLAGS_PREFIX}/lib/libgflags${GLOG_LIB_SUFFIX}.a")
+    set(GFLAGS_CMAKE_CXX_FLAGS " -Wno-error ${EP_CXX_FLAGS}")
+    set(GFLAGS_CMAKE_C_FLAGS " -Wno-error ${EP_C_FLAGS}")
+
+    set(GFLAGS_CMAKE_ARGS
+            ${EP_COMMON_CMAKE_ARGS}
+            -DCMAKE_INSTALL_PREFIX=${GFLAGS_PREFIX}
+            -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
+            -DGFLAGS_NAMESPACE=gflags
+            -DCMAKE_CXX_FLAGS=${GFLAGS_CMAKE_CXX_FLAGS}
+            -DCMAKE_C_FLAGS=${GFLAGS_CMAKE_C_FLAGS})
+
+    externalproject_add(gflags_ep
+            URL ${GFLAGS_SOURCE_URL}
+            URL_HASH "SHA256=${PAIMON_GFLAGS_BUILD_SHA256_CHECKSUM}"
+            CMAKE_ARGS ${GFLAGS_CMAKE_ARGS}
+            BUILD_BYPRODUCTS "${GFLAGS_STATIC_LIB}")
+
+    file(MAKE_DIRECTORY "${GFLAGS_INCLUDE_DIR}")
+    add_library(gflags::gflags STATIC IMPORTED)
+    set_target_properties(gflags::gflags
+            PROPERTIES IMPORTED_LOCATION "${GFLAGS_STATIC_LIB}"
+            INTERFACE_INCLUDE_DIRECTORIES "${GFLAGS_INCLUDE_DIR}"
+            INTERFACE_LINK_DIRECTORIES "${GFLAGS_BUILD_DIR}/lib"
+            INTERFACE_COMPILE_DEFINITIONS "GFLAGS_USE_GFLAGS_EXPORT")
+
+    add_dependencies(gflags::gflags gflags_ep)
+endmacro()
+
 build_fmt()
 build_rapidjson()
 build_snappy()
@@ -1360,6 +1466,10 @@ if(PAIMON_ENABLE_JINDO)
     build_jindosdk_c()
     build_jindosdk_nextarch()
 endif()
+if(PAIMON_ENABLE_ZDFS)
+    build_gflags()
+    build_zdfs()
+endif()
 if(PAIMON_ENABLE_LUCENE)
     build_boost()
     build_lucene()

docs/source/build_system.rst

@@ -85,6 +85,7 @@ Paimon provides a set of built-in optional plugins that you can link to as neede
 
   - ``paimon_local_file_system_shared`` / ``paimon_local_file_system_static``
   - ``paimon_jindo_file_system_shared`` / ``paimon_jindo_file_system_static``
+  - ``paimon_zdfs_file_system_shared`` / ``paimon_zdfs_file_system_static``
 
 - Index plugins:
 

src/paimon/fs/zdfs/CMakeLists.txt

@@ -0,0 +1,33 @@
+option(PAIMON_ZDFS_USE_SHARED_SDK "Use shared zdfs sdk" OFF)
+
+if(PAIMON_ENABLE_ZDFS)
+    if(PAIMON_ZDFS_USE_SHARED_SDK)
+        set(zdfs_sdk_to_link "zdfs::sdk_shared")
+        set(zdfs_sdk_static_link "")
+        set(zdfs_sdk_shared_link "zdfs::sdk_shared")
+    else()
+        find_package(OpenSSL REQUIRED)
+        set(zdfs_sdk_to_link "zdfs::sdk_static" "OpenSSL::SSL" "OpenSSL::Crypto" "gflags::gflags")
+        set(zdfs_sdk_static_link "zdfs::sdk_static" "gflags::gflags")
+        set(zdfs_sdk_shared_link "OpenSSL::SSL" "OpenSSL::Crypto")
+    endif()
+
+    set(PAIMON_ZDFS_FILE_SYSTEM zdfs_file_status.cpp zdfs_file_system.cpp
+                                zdfs_file_system_factory.cpp zdfs_stream.cpp)
+
+    add_paimon_lib(paimon_zdfs_file_system
+                   SOURCES
+                   ${PAIMON_ZDFS_FILE_SYSTEM}
+                   EXTRA_INCLUDES
+                   ${ZDFS_INCLUDE_DIR}
+                   DEPENDENCIES
+                   paimon_shared
+                   ${zdfs_sdk_to_link}
+                   STATIC_LINK_LIBS
+                   ${zdfs_sdk_static_link}
+                   SHARED_LINK_LIBS
+                   paimon_shared
+                   ${zdfs_sdk_shared_link}
+                   SHARED_LINK_FLAGS
+                   ${PAIMON_VERSION_SCRIPT_FLAGS})
+endif()

src/paimon/fs/zdfs/zdfs_file_status.cpp

@@ -0,0 +1,46 @@
+//
+// Created by haohao.yzh on 2026/1/6.
+//
+
+#include "zdfs_file_status.h"
+
+namespace paimon {
+static std::string concatPath(const std::string &dir, const std::string &path) {
+    if (dir.empty()) {
+        return path;
+    }
+    if (dir.back() == '/') {
+        return dir + path;
+    }
+    return dir + "/" + path;
+}
+
+std::unique_ptr<ZdfsBasicFileStatus> ZdfsBasicFileStatus::Create(std::string path,
+                                                               const zdfs::EntryStat &entry_stat) {
+    return std::make_unique<ZdfsBasicFileStatus>(entry_stat.IsDir(), std::move(path));
+}
+
+std::unique_ptr<ZdfsBasicFileStatus> ZdfsBasicFileStatus::CreateSubfile(
+    const std::string &dir, const zdfs::EntryStat &entry_stat) {
+    return Create(concatPath(dir, entry_stat.path), entry_stat);
+}
+
+std::unique_ptr<ZdfsFileStatus> ZdfsFileStatus::Create(std::string path,
+                                                     const zdfs::EntryStat &entry_stat) {
+    if (entry_stat.IsDir()) {
+        const zdfs::DirStat &dirStat = entry_stat.dir;
+        return std::make_unique<ZdfsFileStatus>(true, std::move(path), dirStat.length,
+                                               dirStat.modify_time);
+    } else {
+        const zdfs::FileStat &fileStat = entry_stat.file;
+        return std::make_unique<ZdfsFileStatus>(false, std::move(path), fileStat.length,
+                                               fileStat.modify_time);
+    }
+}
+
+std::unique_ptr<ZdfsFileStatus> ZdfsFileStatus::CreateSubfile(const std::string &dir,
+                                                        const zdfs::EntryStat &entry_stat) {
+    return Create(concatPath(dir, entry_stat.path), entry_stat);
+}
+
+}  // namespace paimon

src/paimon/fs/zdfs/zdfs_file_status.h

@@ -0,0 +1,69 @@
+//
+// Created by haohao.yzh on 2026/1/6.
+//
+
+#pragma once
+
+#include <paimon/fs/file_system.h>
+#include <zdfs/zdfs.h>
+
+#include <utility>
+
+namespace paimon {
+class ZdfsBasicFileStatus : public BasicFileStatus {
+ public:
+    static std::unique_ptr<ZdfsBasicFileStatus> Create(std::string path,
+                                                      const zdfs::EntryStat &entry_stat);
+    static std::unique_ptr<ZdfsBasicFileStatus> CreateSubfile(const std::string &dir,
+                                                         const zdfs::EntryStat &entry_stat);
+
+    ZdfsBasicFileStatus(bool is_dir, std::string path) : is_dir_(is_dir), path_(std::move(path)) {}
+
+    bool IsDir() const override {
+        return is_dir_;
+    }
+
+    std::string GetPath() const override {
+        return path_;
+    }
+
+ private:
+    const bool is_dir_;
+    const std::string path_;
+};
+
+class ZdfsFileStatus : public FileStatus {
+ public:
+    static std::unique_ptr<ZdfsFileStatus> Create(std::string path,
+                                                 const zdfs::EntryStat &entry_stat);
+    static std::unique_ptr<ZdfsFileStatus> CreateSubfile(const std::string &dir,
+                                                    const zdfs::EntryStat &entry_stat);
+
+    ZdfsFileStatus(bool is_dir, std::string path, int64_t modification_time, uint64_t len)
+        : is_dir_(is_dir), path_(std::move(path)),
+        modification_time_(modification_time), len_(len) {}
+
+    uint64_t GetLen() const override {
+        return len_;
+    }
+
+    bool IsDir() const override {
+        return is_dir_;
+    }
+
+    std::string GetPath() const override {
+        return path_;
+    }
+
+    int64_t GetModificationTime() const override {
+        return modification_time_;
+    }
+
+ private:
+    const bool is_dir_;
+    const std::string path_;
+    const int64_t modification_time_;
+    const uint64_t len_;
+};
+
+}  // namespace paimon