fix: sync missing changes after repository migration (#154)

* fix(build): make package config relocatable (#306)

* fix: fix thread-safe problem for file store path factory (#323)

* feat(lumina): support null values in vector column during index building (#310)

* feat: add CreateReader api with field_name & index_type (#391)

* test: add tests for executor (#396)

* fix: fix IsThreadSafe() in UnionGlobalIndexReader (#395)

* fix: release global index writer before reader to avoid mem issue when write index failed (#398)

* fix: complete undefined shift fixes from #308

Migrate the two remaining hunks from alibaba/paimon-cpp@620adbe; the other hunks are already present in Apache.

* chore: remove remaining Lance references

* feat: remove support for FLOAT/DOUBLE type partition field (#333)

* refactor: remove unused path-based reader builder

---------

Co-authored-by: Yonghao Fang <yonghao.fyh@alibaba-inc.com>
Co-authored-by: lxy <38709059+lxy-9602@users.noreply.github.com>
Co-authored-by: lszskye <57179283+lszskye@users.noreply.github.com>

Author: 4 people

CMakeLists.txt

@@ -70,18 +70,12 @@ endif()
 if(PAIMON_USE_CXX11_ABI)
     add_definitions(-D_GLIBCXX_USE_CXX11_ABI=1)
 else()
-    # lance and lumina are provided in so file, cannot be recompiled with C++11 ABI off.
-    if(PAIMON_ENABLE_LANCE)
-        message(FATAL_ERROR "Lance cannot be enabled with C++11 ABI off")
-    endif()
+    # Lumina is provided as a shared library and cannot be recompiled with C++11 ABI off.
     if(PAIMON_ENABLE_LUMINA)
         message(FATAL_ERROR "Lumina cannot be enabled with C++11 ABI off")
     endif()
     add_definitions(-D_GLIBCXX_USE_CXX11_ABI=0)
 endif()
-if(PAIMON_ENABLE_LANCE)
-    add_definitions(-DPAIMON_ENABLE_LANCE)
-endif()
 if(PAIMON_ENABLE_LUMINA)
     add_definitions(-DPAIMON_ENABLE_LUMINA)
 endif()
@@ -100,9 +94,9 @@ set(PAIMON_SOURCE_DIR ${PROJECT_SOURCE_DIR})
 set(PAIMON_BINARY_DIR ${PROJECT_BINARY_DIR})
 set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/cmake_modules")
 set(BUILD_SUPPORT_DIR "${CMAKE_SOURCE_DIR}/build_support")
-set(PAIMON_CMAKE_INSTALL_DIR "${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME}")
 
 include(GNUInstallDirs)
+set(PAIMON_CMAKE_INSTALL_DIR "${CMAKE_INSTALL_LIBDIR}/cmake/Paimon")
 include(DefineOptions)
 
 if(PAIMON_OPTIONAL_INSTALL)
@@ -292,13 +286,6 @@ set(PAIMON_SHARED_PRIVATE_LINK_LIBS ${PAIMON_STATIC_LINK_LIBS})
 add_subdirectory(third_party/roaring_bitmap EXCLUDE_FROM_ALL)
 add_subdirectory(third_party/xxhash EXCLUDE_FROM_ALL)
 
-if(PAIMON_ENABLE_LANCE)
-    add_subdirectory(third_party/lance EXCLUDE_FROM_ALL)
-    link_directories(third_party/lance/lance_lib/target/release)
-    install(FILES ${PROJECT_SOURCE_DIR}/third_party/lance/lance_lib/target/release/liblance_lib_rc.so
-            DESTINATION ${CMAKE_INSTALL_LIBDIR})
-endif()
-
 list(APPEND PAIMON_LINK_LIBS ${CMAKE_DL_LIBS})
 list(APPEND PAIMON_SHARED_INSTALL_INTERFACE_LIBS ${CMAKE_DL_LIBS})
 
@@ -330,10 +317,6 @@ include_directories(${CMAKE_CURRENT_SOURCE_DIR}/include)
 include_directories("${CMAKE_SOURCE_DIR}/third_party/roaring_bitmap")
 include_directories("${CMAKE_SOURCE_DIR}/third_party/xxhash")
 
-if(PAIMON_ENABLE_LANCE)
-    include_directories("${CMAKE_SOURCE_DIR}/third_party/lance")
-endif()
-
 include_directories(SYSTEM ${ARROW_INCLUDE_DIR})
 include_directories(SYSTEM ${TBB_INCLUDE_DIR})
 
@@ -383,11 +366,6 @@ if(PAIMON_BUILD_TESTS)
         paimon_blob_file_format_shared
         "-Wl,--as-needed")
 
-    if(PAIMON_ENABLE_LANCE)
-        list(APPEND TEST_STATIC_LINK_LIBS "-Wl,--no-as-needed")
-        list(APPEND TEST_STATIC_LINK_LIBS paimon_lance_file_format_shared)
-        list(APPEND TEST_STATIC_LINK_LIBS "-Wl,--as-needed")
-    endif()
     if(PAIMON_ENABLE_ORC)
         list(APPEND TEST_STATIC_LINK_LIBS "-Wl,--no-as-needed")
         list(APPEND TEST_STATIC_LINK_LIBS paimon_orc_file_format_shared)
@@ -425,11 +403,11 @@ write_basic_package_version_file(
 
 configure_package_config_file("${CMAKE_CURRENT_SOURCE_DIR}/PaimonConfig.cmake.in"
                               "${CMAKE_CURRENT_BINARY_DIR}/PaimonConfig.cmake"
-                              INSTALL_DESTINATION lib/cmake/Paimon)
+                              INSTALL_DESTINATION ${PAIMON_CMAKE_INSTALL_DIR})
 
 install(FILES "${CMAKE_CURRENT_BINARY_DIR}/PaimonConfig.cmake"
               "${CMAKE_CURRENT_BINARY_DIR}/PaimonConfigVersion.cmake"
-        DESTINATION lib/cmake/Paimon)
+        DESTINATION ${PAIMON_CMAKE_INSTALL_DIR})
 
 config_summary_message()
 
@@ -442,13 +420,14 @@ add_subdirectory(src/paimon/format/blob)
 add_subdirectory(src/paimon/format/orc)
 add_subdirectory(src/paimon/format/parquet)
 add_subdirectory(src/paimon/format/avro)
-if(PAIMON_ENABLE_LANCE)
-    add_subdirectory(src/paimon/format/lance)
-endif()
 if(PAIMON_ENABLE_LUMINA)
     add_subdirectory(src/paimon/global_index/lumina)
 endif()
 add_subdirectory(src/paimon/global_index/lucene)
 add_subdirectory(src/paimon/testing/mock)
 add_subdirectory(src/paimon/testing/utils)
 add_subdirectory(test/inte)
+
+install(EXPORT PaimonTargets
+        NAMESPACE Paimon::
+        DESTINATION ${PAIMON_CMAKE_INSTALL_DIR})

PaimonConfig.cmake.in

@@ -14,134 +14,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# Main library
-add_library(paimon_shared SHARED IMPORTED)
-set_target_properties(paimon_shared PROPERTIES
-    IMPORTED_LOCATION "@CMAKE_INSTALL_PREFIX@/@CMAKE_INSTALL_LIBDIR@/libpaimon.so"
-    INTERFACE_INCLUDE_DIRECTORIES "@CMAKE_INSTALL_PREFIX@/include"
-)
-add_library(paimon_static STATIC IMPORTED)
-set_target_properties(paimon_static PROPERTIES
-    IMPORTED_LOCATION "@CMAKE_INSTALL_PREFIX@/@CMAKE_INSTALL_LIBDIR@/libpaimon.a"
-    INTERFACE_INCLUDE_DIRECTORIES "@CMAKE_INSTALL_PREFIX@/include"
-)
+@PACKAGE_INIT@
 
-# paimon_parquet_file_format
-add_library(paimon_parquet_file_format_shared SHARED IMPORTED)
-set_target_properties(paimon_parquet_file_format_shared PROPERTIES
-    IMPORTED_LOCATION "@CMAKE_INSTALL_PREFIX@/@CMAKE_INSTALL_LIBDIR@/libpaimon_parquet_file_format.so"
-    INTERFACE_INCLUDE_DIRECTORIES "@CMAKE_INSTALL_PREFIX@/include"
-)
-add_library(paimon_parquet_file_format_static STATIC IMPORTED)
-set_target_properties(paimon_parquet_file_format_static PROPERTIES
-    IMPORTED_LOCATION "@CMAKE_INSTALL_PREFIX@/@CMAKE_INSTALL_LIBDIR@/libpaimon_parquet_file_format.a"
-    INTERFACE_INCLUDE_DIRECTORIES "@CMAKE_INSTALL_PREFIX@/include"
-)
+include("${CMAKE_CURRENT_LIST_DIR}/PaimonTargets.cmake")
 
-# paimon_orc_file_format
-add_library(paimon_orc_file_format_shared SHARED IMPORTED)
-set_target_properties(paimon_orc_file_format_shared PROPERTIES
-    IMPORTED_LOCATION "@CMAKE_INSTALL_PREFIX@/@CMAKE_INSTALL_LIBDIR@/libpaimon_orc_file_format.so"
-    INTERFACE_INCLUDE_DIRECTORIES "@CMAKE_INSTALL_PREFIX@/include"
-)
-add_library(paimon_orc_file_format_static STATIC IMPORTED)
-set_target_properties(paimon_orc_file_format_static PROPERTIES
-    IMPORTED_LOCATION "@CMAKE_INSTALL_PREFIX@/@CMAKE_INSTALL_LIBDIR@/libpaimon_orc_file_format.a"
-    INTERFACE_INCLUDE_DIRECTORIES "@CMAKE_INSTALL_PREFIX@/include"
-)
-
-# paimon_local_file_system
-add_library(paimon_local_file_system_shared SHARED IMPORTED)
-set_target_properties(paimon_local_file_system_shared PROPERTIES
-    IMPORTED_LOCATION "@CMAKE_INSTALL_PREFIX@/@CMAKE_INSTALL_LIBDIR@/libpaimon_local_file_system.so"
-    INTERFACE_INCLUDE_DIRECTORIES "@CMAKE_INSTALL_PREFIX@/include"
-)
-add_library(paimon_local_file_system_static STATIC IMPORTED)
-set_target_properties(paimon_local_file_system_static PROPERTIES
-    IMPORTED_LOCATION "@CMAKE_INSTALL_PREFIX@/@CMAKE_INSTALL_LIBDIR@/libpaimon_local_file_system.a"
-    INTERFACE_INCLUDE_DIRECTORIES "@CMAKE_INSTALL_PREFIX@/include"
-)
-
-# paimon_avro_file_format
-add_library(paimon_avro_file_format_shared SHARED IMPORTED)
-set_target_properties(paimon_avro_file_format_shared PROPERTIES
-    IMPORTED_LOCATION "@CMAKE_INSTALL_PREFIX@/@CMAKE_INSTALL_LIBDIR@/libpaimon_avro_file_format.so"
-    INTERFACE_INCLUDE_DIRECTORIES "@CMAKE_INSTALL_PREFIX@/include"
-)
-add_library(paimon_avro_file_format_static STATIC IMPORTED)
-set_target_properties(paimon_avro_file_format_static PROPERTIES
-    IMPORTED_LOCATION "@CMAKE_INSTALL_PREFIX@/@CMAKE_INSTALL_LIBDIR@/libpaimon_avro_file_format.a"
-    INTERFACE_INCLUDE_DIRECTORIES "@CMAKE_INSTALL_PREFIX@/include"
-)
-
-# paimon_blob_file_format
-add_library(paimon_blob_file_format_shared SHARED IMPORTED)
-set_target_properties(paimon_blob_file_format_shared PROPERTIES
-    IMPORTED_LOCATION "@CMAKE_INSTALL_PREFIX@/@CMAKE_INSTALL_LIBDIR@/libpaimon_blob_file_format.so"
-    INTERFACE_INCLUDE_DIRECTORIES "@CMAKE_INSTALL_PREFIX@/include"
-)
-add_library(paimon_blob_file_format_static STATIC IMPORTED)
-set_target_properties(paimon_blob_file_format_static PROPERTIES
-    IMPORTED_LOCATION "@CMAKE_INSTALL_PREFIX@/@CMAKE_INSTALL_LIBDIR@/libpaimon_blob_file_format.a"
-    INTERFACE_INCLUDE_DIRECTORIES "@CMAKE_INSTALL_PREFIX@/include"
-)
-
-# paimon_file_index
-add_library(paimon_file_index_shared SHARED IMPORTED)
-set_target_properties(paimon_file_index_shared PROPERTIES
-    IMPORTED_LOCATION "@CMAKE_INSTALL_PREFIX@/@CMAKE_INSTALL_LIBDIR@/libpaimon_file_index.so"
-    INTERFACE_INCLUDE_DIRECTORIES "@CMAKE_INSTALL_PREFIX@/include"
-)
-add_library(paimon_file_index_static STATIC IMPORTED)
-set_target_properties(paimon_file_index_static PROPERTIES
-    IMPORTED_LOCATION "@CMAKE_INSTALL_PREFIX@/@CMAKE_INSTALL_LIBDIR@/libpaimon_file_index.a"
-    INTERFACE_INCLUDE_DIRECTORIES "@CMAKE_INSTALL_PREFIX@/include"
-)
-
-# paimon_global_index
-add_library(paimon_global_index_shared SHARED IMPORTED)
-set_target_properties(paimon_global_index_shared PROPERTIES
-    IMPORTED_LOCATION "@CMAKE_INSTALL_PREFIX@/@CMAKE_INSTALL_LIBDIR@/libpaimon_global_index.so"
-    INTERFACE_INCLUDE_DIRECTORIES "@CMAKE_INSTALL_PREFIX@/include"
-)
-add_library(paimon_global_index_static STATIC IMPORTED)
-set_target_properties(paimon_global_index_static PROPERTIES
-    IMPORTED_LOCATION "@CMAKE_INSTALL_PREFIX@/@CMAKE_INSTALL_LIBDIR@/libpaimon_global_index.a"
-    INTERFACE_INCLUDE_DIRECTORIES "@CMAKE_INSTALL_PREFIX@/include"
-)
-
-# paimon_jindo_file_system
-add_library(paimon_jindo_file_system_shared SHARED IMPORTED)
-set_target_properties(paimon_jindo_file_system_shared PROPERTIES
-    IMPORTED_LOCATION "@CMAKE_INSTALL_PREFIX@/@CMAKE_INSTALL_LIBDIR@/libpaimon_jindo_file_system.so"
-    INTERFACE_INCLUDE_DIRECTORIES "@CMAKE_INSTALL_PREFIX@/include"
-)
-add_library(paimon_jindo_file_system_static STATIC IMPORTED)
-set_target_properties(paimon_jindo_file_system_static PROPERTIES
-    IMPORTED_LOCATION "@CMAKE_INSTALL_PREFIX@/@CMAKE_INSTALL_LIBDIR@/libpaimon_jindo_file_system.a"
-    INTERFACE_INCLUDE_DIRECTORIES "@CMAKE_INSTALL_PREFIX@/include"
-)
-
-# paimon_lance_file_format
-add_library(paimon_lance_file_format_shared SHARED IMPORTED)
-set_target_properties(paimon_lance_file_format_shared PROPERTIES
-    IMPORTED_LOCATION "@CMAKE_INSTALL_PREFIX@/@CMAKE_INSTALL_LIBDIR@/libpaimon_lance_file_format.so"
-    INTERFACE_INCLUDE_DIRECTORIES "@CMAKE_INSTALL_PREFIX@/include"
-)
-add_library(paimon_lance_file_format_static STATIC IMPORTED)
-set_target_properties(paimon_lance_file_format_static PROPERTIES
-    IMPORTED_LOCATION "@CMAKE_INSTALL_PREFIX@/@CMAKE_INSTALL_LIBDIR@/libpaimon_lance_file_format.a"
-    INTERFACE_INCLUDE_DIRECTORIES "@CMAKE_INSTALL_PREFIX@/include"
-)
-
-# paimon_lumina_index
-add_library(paimon_lumina_index_shared SHARED IMPORTED)
-set_target_properties(paimon_lumina_index_shared PROPERTIES
-    IMPORTED_LOCATION "@CMAKE_INSTALL_PREFIX@/@CMAKE_INSTALL_LIBDIR@/libpaimon_lumina_index.so"
-    INTERFACE_INCLUDE_DIRECTORIES "@CMAKE_INSTALL_PREFIX@/include"
-)
-add_library(paimon_lumina_index_static STATIC IMPORTED)
-set_target_properties(paimon_lumina_index_static PROPERTIES
-    IMPORTED_LOCATION "@CMAKE_INSTALL_PREFIX@/@CMAKE_INSTALL_LIBDIR@/libpaimon_lumina_index.a"
-    INTERFACE_INCLUDE_DIRECTORIES "@CMAKE_INSTALL_PREFIX@/include"
-)
+check_required_components(Paimon)

ci/scripts/build_paimon.sh

@@ -21,7 +21,7 @@ source_dir=${1}
 enable_sanitizer=${2:-false}
 check_clang_tidy=${3:-false}
 build_type=${4:-Debug}
-build_dir=${1}/build
+build_dir="${source_dir}/build"
 
 # Display ccache status if available
 if command -v ccache &> /dev/null; then
@@ -32,8 +32,8 @@ else
     echo "=== ccache not found, compiling without cache acceleration ==="
 fi
 
-mkdir ${build_dir}
-pushd ${build_dir}
+mkdir -p "${build_dir}"
+pushd "${build_dir}"
 
 ENABLE_LUMINA="ON"
 if [[ "${CC:-}" == *"gcc-8"* ]] || [[ "${CXX:-}" == *"g++-8"* ]]; then
@@ -56,9 +56,9 @@ if [[ "${enable_sanitizer}" == "true" ]]; then
     )
 fi
 
-cmake "${CMAKE_ARGS[@]}" ${source_dir}
-cmake --build . -- -j$(nproc)
-ctest --output-on-failure -j $(nproc)
+cmake "${CMAKE_ARGS[@]}" "${source_dir}"
+cmake --build . -- -j "$(nproc)"
+ctest --output-on-failure -j "$(nproc)"
 
 if [[ "${check_clang_tidy}" == "true" ]]; then
     cmake --build . --target check-clang-tidy
@@ -72,4 +72,4 @@ fi
 
 popd
 
-rm -rf ${build_dir}
+rm -rf "${build_dir}"

cmake_modules/BuildUtils.cmake

@@ -150,7 +150,7 @@ function(add_paimon_lib LIB_NAME)
                             -Wl,--gc-sections)
 
         install(TARGETS ${LIB_NAME}_shared ${INSTALL_IS_OPTIONAL}
-                EXPORT ${LIB_NAME}_targets
+                EXPORT PaimonTargets
                 RUNTIME DESTINATION ${RUNTIME_INSTALL_DIR}
                 LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
                 ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
@@ -197,7 +197,7 @@ function(add_paimon_lib LIB_NAME)
                               PUBLIC "$<BUILD_INTERFACE:paimon_sanitizer_flags>")
 
         install(TARGETS ${LIB_NAME}_static ${INSTALL_IS_OPTIONAL}
-                EXPORT ${LIB_NAME}_targets
+                EXPORT PaimonTargets
                 RUNTIME DESTINATION ${RUNTIME_INSTALL_DIR}
                 LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
                 ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}

docs/source/build_system.rst

@@ -52,10 +52,10 @@ for data format and file system.
    find_package(Paimon REQUIRED)
 
    add_executable(my_example my_example.cc)
-   target_link_libraries(my_example PRIVATE arrow_shared
-                                            paimon_shared
-                                            paimon_parquet_file_format_shared
-                                            paimon_local_file_system_shared)
+   target_link_libraries(my_example PRIVATE Arrow::arrow_shared
+                                            Paimon::paimon_shared
+                                            Paimon::paimon_parquet_file_format_shared
+                                            Paimon::paimon_local_file_system_shared)
 
 Available variables and targets
 -------------------------------
@@ -64,10 +64,10 @@ The directive ``find_package(Paimon REQUIRED)`` instructs CMake to locate a
 Paimon C++ installation on your system. If successful, it sets ``Paimon_FOUND``
 to true if the Paimon C++ libraries were found.
 
-It also defines the following linkable targets (plain strings, not variables):
+It also defines the following linkable imported targets:
 
-* ``paimon_shared`` links to the Paimon shared libraries
-* ``paimon_static`` links to the Paimon static libraries
+* ``Paimon::paimon_shared`` links to the Paimon shared libraries
+* ``Paimon::paimon_static`` links to the Paimon static libraries
 
 In most cases, it is recommended to use the Paimon shared libraries.
 
@@ -78,21 +78,21 @@ Paimon provides a set of built-in optional plugins that you can link to as neede
 
 - File format plugins:
 
-  - ``paimon_parquet_file_format_shared`` / ``paimon_parquet_file_format_static``
-  - ``paimon_orc_file_format_shared`` / ``paimon_orc_file_format_static``
-  - ``paimon_avro_file_format_shared`` / ``paimon_avro_file_format_static``
-  - ``paimon_blob_file_format_shared`` / ``paimon_blob_file_format_static``
+  - ``Paimon::paimon_parquet_file_format_shared`` / ``Paimon::paimon_parquet_file_format_static``
+  - ``Paimon::paimon_orc_file_format_shared`` / ``Paimon::paimon_orc_file_format_static``
+  - ``Paimon::paimon_avro_file_format_shared`` / ``Paimon::paimon_avro_file_format_static``
+  - ``Paimon::paimon_blob_file_format_shared`` / ``Paimon::paimon_blob_file_format_static``
 
 - File system plugins:
 
-  - ``paimon_local_file_system_shared`` / ``paimon_local_file_system_static``
-  - ``paimon_jindo_file_system_shared`` / ``paimon_jindo_file_system_static``
+  - ``Paimon::paimon_local_file_system_shared`` / ``Paimon::paimon_local_file_system_static``
+  - ``Paimon::paimon_jindo_file_system_shared`` / ``Paimon::paimon_jindo_file_system_static``
 
 - Index plugins:
 
-  - ``paimon_file_index_shared`` / ``paimon_file_index_static``
-  - ``paimon_lumina_index_shared`` / ``paimon_lumina_index_static``
-  - ``paimon_lucene_index_shared`` / ``paimon_lucene_index_static``
+  - ``Paimon::paimon_file_index_shared`` / ``Paimon::paimon_file_index_static``
+  - ``Paimon::paimon_lumina_index_shared`` / ``Paimon::paimon_lumina_index_static``
+  - ``Paimon::paimon_lucene_index_shared`` / ``Paimon::paimon_lucene_index_static``
 
 .. note::
 

examples/CMakeLists.txt

@@ -26,20 +26,18 @@ find_package(Paimon CONFIG REQUIRED)
 add_executable(read_write_demo read_write_demo.cpp)
 add_executable(clean_demo clean_demo.cpp)
 
-include_directories(${PAIMON_INCLUDE_DIRS})
-
 set(CMAKE_EXE_LINKER_FLAGS "-Wl,--no-as-needed ${CMAKE_EXE_LINKER_FLAGS}")
 
 target_link_libraries(read_write_demo
-                      PRIVATE arrow_shared
-                              paimon_shared
-                              paimon_parquet_file_format_shared
-                              paimon_orc_file_format_shared
-                              paimon_local_file_system_shared)
+                      PRIVATE Arrow::arrow_shared
+                              Paimon::paimon_shared
+                              Paimon::paimon_parquet_file_format_shared
+                              Paimon::paimon_orc_file_format_shared
+                              Paimon::paimon_local_file_system_shared)
 
 target_link_libraries(clean_demo
-                      PRIVATE arrow_shared
-                              paimon_shared
-                              paimon_parquet_file_format_shared
-                              paimon_orc_file_format_shared
-                              paimon_local_file_system_shared)
+                      PRIVATE Arrow::arrow_shared
+                              Paimon::paimon_shared
+                              Paimon::paimon_parquet_file_format_shared
+                              Paimon::paimon_orc_file_format_shared
+                              Paimon::paimon_local_file_system_shared)