feat(index): introduce Lumina vector index and enable Lucene builds (#151)

Author: zjw1111

CMakeLists.txt

@@ -54,10 +54,6 @@ option(PAIMON_ENABLE_AVRO "Whether to enable avro file format" ON)
 option(PAIMON_ENABLE_ORC "Whether to enable orc file format" ON)
 option(PAIMON_ENABLE_JINDO "Whether to enable jindo file system" OFF)
 option(PAIMON_ENABLE_LUCENE "Whether to enable lucene index" OFF)
-if(PAIMON_ENABLE_LUCENE)
-    message(FATAL_ERROR "PAIMON_ENABLE_LUCENE is temporarily disabled. Please configure with -DPAIMON_ENABLE_LUCENE=OFF."
-    )
-endif()
 
 if(PAIMON_ENABLE_ORC)
     add_definitions(-DPAIMON_ENABLE_ORC)
@@ -302,12 +298,6 @@ endif()
 
 list(APPEND PAIMON_LINK_LIBS ${CMAKE_DL_LIBS})
 list(APPEND PAIMON_SHARED_INSTALL_INTERFACE_LIBS ${CMAKE_DL_LIBS})
-if(PAIMON_ENABLE_LUMINA)
-    add_subdirectory(third_party/lumina EXCLUDE_FROM_ALL)
-    link_directories(third_party/lumina/lib)
-    install(FILES ${PROJECT_SOURCE_DIR}/third_party/lumina/lib/liblumina.so
-            DESTINATION ${CMAKE_INSTALL_LIBDIR})
-endif()
 
 if(PAIMON_ENABLE_LUCENE)
     set(PAIMON_DICT_DEST "share/paimon/dict")
@@ -341,10 +331,6 @@ if(PAIMON_ENABLE_LANCE)
     include_directories("${CMAKE_SOURCE_DIR}/third_party/lance")
 endif()
 
-if(PAIMON_ENABLE_LUMINA)
-    include_directories("${CMAKE_SOURCE_DIR}/third_party/lumina/include")
-endif()
-
 include_directories(SYSTEM ${ARROW_INCLUDE_DIR})
 include_directories(SYSTEM ${TBB_INCLUDE_DIR})
 

ci/scripts/build_paimon.sh

@@ -35,12 +35,18 @@ fi
 mkdir ${build_dir}
 pushd ${build_dir}
 
+ENABLE_LUMINA="ON"
+if [[ "${CC:-}" == *"gcc-8"* ]] || [[ "${CXX:-}" == *"g++-8"* ]]; then
+    ENABLE_LUMINA="OFF"
+fi
+
 CMAKE_ARGS=(
     "-G Ninja"
     "-DCMAKE_BUILD_TYPE=${build_type}"
     "-DPAIMON_BUILD_TESTS=ON"
     "-DPAIMON_ENABLE_JINDO=ON"
-    "-DPAIMON_ENABLE_LUCENE=OFF"
+    "-DPAIMON_ENABLE_LUMINA=${ENABLE_LUMINA}"
+    "-DPAIMON_ENABLE_LUCENE=ON"
 )
 
 if [[ "${enable_sanitizer}" == "true" ]]; then

cmake_modules/ThirdpartyToolchain.cmake

@@ -281,6 +281,26 @@ else()
     endif()
 endif()
 
+if(DEFINED ENV{PAIMON_LUMINA_URL})
+    set(LUMINA_SOURCE_URL "$ENV{PAIMON_LUMINA_URL}")
+elseif(EXISTS "${THIRDPARTY_DIR}/${PAIMON_LUMINA_PKG_NAME}")
+    set_urls(LUMINA_SOURCE_URL "${THIRDPARTY_DIR}/${PAIMON_LUMINA_PKG_NAME}")
+else()
+    set_urls(LUMINA_SOURCE_URL
+             "https://paimon-cpp.oss-cn-beijing.aliyuncs.com/thirdparty/lumina/lumina_release-${PAIMON_LUMINA_BUILD_VERSION}.tar.gz"
+    )
+endif()
+
+if(DEFINED ENV{PAIMON_BOOST_URL})
+    set(BOOST_SOURCE_URL "$ENV{PAIMON_BOOST_URL}")
+elseif(EXISTS "${THIRDPARTY_DIR}/${PAIMON_BOOST_PKG_NAME}")
+    set_urls(BOOST_SOURCE_URL "${THIRDPARTY_DIR}/${PAIMON_BOOST_PKG_NAME}")
+else()
+    set_urls(BOOST_SOURCE_URL
+             "https://paimon-cpp.oss-cn-beijing.aliyuncs.com/thirdparty/boost/${PAIMON_BOOST_PKG_NAME}"
+    )
+endif()
+
 if(APPLE)
     set(JINDOSDK_C_DYNAMIC_LIB_NAME "jindosdk_c.${PAIMON_JINDOSDK_C_BUILD_VERSION}")
     set(JINDOSDK_C_DYNAMIC_LIB_FILE "lib${JINDOSDK_C_DYNAMIC_LIB_NAME}.dylib")
@@ -938,7 +958,7 @@ macro(build_boost)
     endif()
 
     externalproject_add(boost_ep
-                        URL "${THIRDPARTY_DIR}/boost/${PAIMON_BOOST_PKG_NAME}"
+                        URL ${BOOST_SOURCE_URL}
                         URL_HASH "SHA256=${PAIMON_BOOST_BUILD_SHA256_CHECKSUM}"
                         CONFIGURE_COMMAND ${BOOST_PREFIX}/src/boost_ep/bootstrap.sh
                                           --with-libraries=date_time,filesystem,iostreams,regex,system,thread,chrono,atomic
@@ -1196,6 +1216,42 @@ macro(build_jindosdk_c)
 
 endmacro()
 
+macro(build_lumina)
+    message(STATUS "Installing Lumina from precompiled package")
+
+    set(LUMINA_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/lumina_ep-install")
+    set(LUMINA_INCLUDE_DIR "${LUMINA_PREFIX}/include")
+    set(LUMINA_LIB_DIR "${LUMINA_PREFIX}/lib")
+    set(LUMINA_DYNAMIC_LIB "${LUMINA_LIB_DIR}/liblumina.so")
+
+    externalproject_add(lumina_ep
+                        URL ${LUMINA_SOURCE_URL}
+                        URL_HASH "SHA256=${PAIMON_LUMINA_BUILD_SHA256_CHECKSUM}"
+                        ${THIRDPARTY_LOG_OPTIONS}
+                        CONFIGURE_COMMAND ""
+                        BUILD_COMMAND ""
+                        INSTALL_COMMAND ${CMAKE_COMMAND} -E copy_directory
+                                        <SOURCE_DIR>/artifacts/cpp/linux-x86_64/install-root/usr/local/include
+                                        ${LUMINA_INCLUDE_DIR}
+                        COMMAND ${CMAKE_COMMAND} -E copy_if_different
+                                <SOURCE_DIR>/artifacts/cpp/linux-x86_64/install-root/usr/local/lib/liblumina.so
+                                ${LUMINA_DYNAMIC_LIB}
+                        BUILD_BYPRODUCTS "${LUMINA_DYNAMIC_LIB}")
+
+    file(MAKE_DIRECTORY "${LUMINA_INCLUDE_DIR}")
+    file(MAKE_DIRECTORY "${LUMINA_LIB_DIR}")
+
+    add_library(lumina::interface SHARED IMPORTED GLOBAL)
+    set_target_properties(lumina::interface
+                          PROPERTIES IMPORTED_LOCATION "${LUMINA_DYNAMIC_LIB}"
+                                     IMPORTED_NO_SONAME TRUE
+                                     INTERFACE_INCLUDE_DIRECTORIES
+                                     "${LUMINA_INCLUDE_DIR}")
+    add_dependencies(lumina::interface lumina_ep)
+
+    install(FILES "${LUMINA_DYNAMIC_LIB}" DESTINATION ${CMAKE_INSTALL_LIBDIR})
+endmacro()
+
 macro(build_jindosdk_nextarch)
     message(STATUS "Building jindosdk-nextarch from local source")
 
@@ -1817,6 +1873,9 @@ if(PAIMON_ENABLE_JINDO)
     build_jindosdk_c()
     build_jindosdk_nextarch()
 endif()
+if(PAIMON_ENABLE_LUMINA)
+    build_lumina()
+endif()
 if(PAIMON_ENABLE_LUCENE)
     build_boost()
     build_lucene()

docs/source/build_system.rst

@@ -91,6 +91,8 @@ Paimon provides a set of built-in optional plugins that you can link to as neede
 - Index plugins:
 
   - ``paimon_file_index_shared`` / ``paimon_file_index_static``
+  - ``paimon_lumina_index_shared`` / ``paimon_lumina_index_static``
+  - ``paimon_lucene_index_shared`` / ``paimon_lucene_index_static``
 
 .. note::
 

docs/source/building.rst

@@ -124,6 +124,8 @@ boolean flags to ``cmake``.
 * ``-DPAIMON_ENABLE_ORC=ON``: Paimon integration with Apache ORC
 * ``-DPAIMON_ENABLE_AVRO=ON``: Apache Avro libraries and Paimon integration
 * ``-DPAIMON_ENABLE_JINDO=ON``: Support for Alibaba Jindo filesystems
+* ``-DPAIMON_ENABLE_LUMINA=ON``: Support for the Lumina vector index.
+* ``-DPAIMON_ENABLE_LUCENE=ON``: Support for Lucene full-text search indexes
 
 Third-party dependency source
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

docs/source/user_guide/global_index.rst

@@ -26,6 +26,7 @@ Paimon supports multiple global index types:
 - **BTree Index**: An efficient index based on multi-level SST files for scalar column lookups.
 - **Range Bitmap Index**: A range bitmap index optimized for range predicates on ordered scalar columns. Extends the bitmap approach by encoding value ordering, enabling efficient less-than, greater-than, and range conditions.
 - **Lucene Index**: A full-text search index powered by Lucene++. Supports tokenized text search with multiple modes including match-all, match-any, phrase, prefix, and wildcard queries.
+- **Vector Index (Lumina)**: An approximate nearest neighbor (ANN) index powered by Lumina for vector similarity search with configurable distance metrics.
 
 Global indexes work on top of Data Evolution tables. To use global indexes, your table must have:
 
@@ -85,3 +86,10 @@ search modes including match-all, match-any, phrase, prefix, and wildcard querie
 - **Environment Variable**: ``PAIMON_JIEBA_DICT_DIR``
 
   - **Description**: Specifies the directory containing Jieba dictionary files for Chinese text tokenization. At runtime, the system first checks this environment variable; if not set, it falls back to the compile-time ``JIEBA_TEST_DICT_DIR`` macro (only available in test builds). If neither is available, will fail with an error.
+
+Vector Index (Lumina)
+---------------------
+
+An approximate nearest neighbor (ANN) index powered by Lumina for vector similarity search.
+It supports high-dimensional vector search with configurable distance metrics and encoding strategies.
+For more configurations, refer to the ``docs/reference`` directory in the Lumina release package.

src/paimon/global_index/lumina/CMakeLists.txt

@@ -0,0 +1,57 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if(PAIMON_ENABLE_LUMINA)
+    set(PAIMON_LUMINA_INDEX lumina_global_index.cpp lumina_global_index_factory.cpp)
+
+    add_paimon_lib(paimon_lumina_index
+                   SOURCES
+                   ${PAIMON_LUMINA_INDEX}
+                   DEPENDENCIES
+                   paimon_shared
+                   lumina::interface
+                   STATIC_LINK_LIBS
+                   arrow
+                   glog
+                   fmt
+                   dl
+                   Threads::Threads
+                   SHARED_LINK_LIBS
+                   lumina::interface
+                   paimon_shared
+                   SHARED_LINK_FLAGS
+                   ${PAIMON_VERSION_SCRIPT_FLAGS})
+
+    if(PAIMON_BUILD_TESTS)
+        add_paimon_test(lumina_index_test
+                        SOURCES
+                        lumina_api_test.cpp
+                        lumina_file_io_test.cpp
+                        lumina_global_index_test.cpp
+                        EXTRA_INCLUDES
+                        ${LUMINA_INCLUDE_DIR}
+                        STATIC_LINK_LIBS
+                        paimon_shared
+                        test_utils_static
+                        "-Wl,--whole-archive"
+                        paimon_local_file_system_static
+                        paimon_lumina_index_static
+                        "-Wl,--no-whole-archive"
+                        lumina::interface
+                        ${GTEST_LINK_TOOLCHAIN})
+    endif()
+
+endif()