Merge branch 'main' into feature/columnar-row-ref

Author: lucasfang

cmake_modules/ThirdpartyToolchain.cmake

@@ -292,6 +292,7 @@ macro(build_lucene)
     set(LUCENE_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/lucene_ep-install")
     set(LUCENE_CMAKE_ARGS
         ${EP_COMMON_CMAKE_ARGS}
+        "-DLUCENE_BUILD_SHARED=OFF"
         "-DENABLE_TEST=OFF"
         "-DCMAKE_C_FLAGS=-pthread"
         "-DCMAKE_CXX_FLAGS=-pthread"
@@ -303,7 +304,7 @@ macro(build_lucene)
         "-DBoost_THREAD_FOUND=TRUE"
         "-DCMAKE_INSTALL_PREFIX=${LUCENE_PREFIX}")
 
-    set(LUCENE_LIB "${LUCENE_PREFIX}/lib/liblucene++.so.0")
+    set(LUCENE_LIB "${LUCENE_PREFIX}/lib/liblucene++.a")
     externalproject_add(lucene_ep
                         ${EP_COMMON_OPTIONS}
                         URL ${LUCENE_SOURCE_URL}
@@ -323,13 +324,14 @@ macro(build_lucene)
     # The include directory must exist before it is referenced by a target.
     file(MAKE_DIRECTORY "${LUCENE_INCLUDE_DIR}")
     include_directories(SYSTEM ${LUCENE_INCLUDE_DIR} ${BOOST_INCLUDE_DIR})
-    add_library(lucene INTERFACE IMPORTED)
-    target_include_directories(lucene SYSTEM INTERFACE "${LUCENE_INCLUDE_DIR}")
-    target_compile_options(lucene INTERFACE -pthread)
+    add_library(lucene STATIC IMPORTED)
+    set_target_properties(lucene
+                          PROPERTIES IMPORTED_LOCATION "${LUCENE_LIB}"
+                                     INTERFACE_INCLUDE_DIRECTORIES
+                                     "${LUCENE_INCLUDE_DIR}")
 
     target_link_libraries(lucene
-                          INTERFACE "${LUCENE_LIB}"
-                                    boost_date_time
+                          INTERFACE boost_date_time
                                     boost_filesystem
                                     boost_regex
                                     boost_thread

include/paimon/file_store_write.h

@@ -47,6 +47,8 @@ class PAIMON_EXPORT FileStoreWrite {
     virtual ~FileStoreWrite() = default;
 
     /// Support write an input `RecordBatch` to internal buffer or file.
+    /// @note If a field in table schema is marked as non-nullable (`nullable = false`),
+    ///       the corresponding array in `batch` must have zero null entries.
     virtual Status Write(std::unique_ptr<RecordBatch>&& batch) = 0;
 
     /// Generate a list of commit messages with the latest generated data file meta

…tmap_vector_search_global_index_result.h …ndex/bitmap_scored_global_index_result.hinclude/paimon/global_index/bitmap_vector_search_global_index_result.h renamed to include/paimon/global_index/bitmap_scored_global_index_result.h include/paimon/global_index/bitmap_vector_search_global_index_result.h renamed to include/paimon/global_index/bitmap_scored_global_index_result.h

@@ -1,5 +1,5 @@
 /*
- * Copyright 2024-present Alibaba Inc.
+ * Copyright 2026-present Alibaba Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -26,25 +26,24 @@
 #include "paimon/visibility.h"
 
 namespace paimon {
-/// Represents a vector search global index result that combines a Roaring bitmap of candidate row
+/// Represents a scored global index result that combines a Roaring bitmap of candidate row
 /// ids with an array of associated relevance scores.
 ///
-/// **Important Ordering Note**: Despite inheriting from VectorSearchGlobalIndexResult, the results
-/// are
+/// **Important Ordering Note**: Inheriting from ScoredGlobalIndexResult, the results are
 /// **NOT sorted by score**. Instead, both the bitmap and the score vector are ordered by
 /// **ascending row id**. This design enables efficient merging and set operations while preserving
 /// row id-to-score mapping.
-class PAIMON_EXPORT BitmapVectorSearchGlobalIndexResult : public VectorSearchGlobalIndexResult {
+class PAIMON_EXPORT BitmapScoredGlobalIndexResult : public ScoredGlobalIndexResult {
  public:
-    BitmapVectorSearchGlobalIndexResult(RoaringBitmap64&& bitmap, std::vector<float>&& scores)
+    BitmapScoredGlobalIndexResult(RoaringBitmap64&& bitmap, std::vector<float>&& scores)
         : bitmap_(std::move(bitmap)), scores_(std::move(scores)) {
         assert(static_cast<size_t>(bitmap_.Cardinality()) == scores_.size());
     }
 
-    class VectorSearchIterator : public VectorSearchGlobalIndexResult::VectorSearchIterator {
+    class ScoredIterator : public ScoredGlobalIndexResult::ScoredIterator {
      public:
-        VectorSearchIterator(const RoaringBitmap64* bitmap, RoaringBitmap64::Iterator&& iter,
-                             const float* scores)
+        ScoredIterator(const RoaringBitmap64* bitmap, RoaringBitmap64::Iterator&& iter,
+                       const float* scores)
             : bitmap_(bitmap), iter_(std::move(iter)), scores_(scores) {}
 
         bool HasNext() const override {
@@ -66,8 +65,8 @@ class PAIMON_EXPORT BitmapVectorSearchGlobalIndexResult : public VectorSearchGlo
 
     Result<std::unique_ptr<GlobalIndexResult::Iterator>> CreateIterator() const override;
 
-    Result<std::unique_ptr<VectorSearchGlobalIndexResult::VectorSearchIterator>>
-    CreateVectorSearchIterator() const override;
+    Result<std::unique_ptr<ScoredGlobalIndexResult::ScoredIterator>> CreateScoredIterator()
+        const override;
 
     Result<std::shared_ptr<GlobalIndexResult>> And(
         const std::shared_ptr<GlobalIndexResult>& other) override;
@@ -90,7 +89,6 @@ class PAIMON_EXPORT BitmapVectorSearchGlobalIndexResult : public VectorSearchGlo
     const std::vector<float>& GetScores() const;
 
  private:
-    // TODO(xinyu.lxy): may use pair<int64_t, float>
     RoaringBitmap64 bitmap_;
     // ordered by row id
     std::vector<float> scores_;

include/paimon/global_index/global_index_reader.h

@@ -1,5 +1,5 @@
 /*
- * Copyright 2024-present Alibaba Inc.
+ * Copyright 2026-present Alibaba Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -21,6 +21,7 @@
 #include <vector>
 
 #include "paimon/global_index/global_index_result.h"
+#include "paimon/predicate/full_text_search.h"
 #include "paimon/predicate/function_visitor.h"
 #include "paimon/predicate/vector_search.h"
 #include "paimon/visibility.h"
@@ -40,9 +41,13 @@ class PAIMON_EXPORT GlobalIndexReader : public FunctionVisitor<std::shared_ptr<G
     /// VisitVectorSearch performs approximate vector similarity search.
     /// @warning `VisitVectorSearch` may return error status when it is incorrectly invoked (e.g.,
     /// BitmapGlobalIndexReader call `VisitVectorSearch`).
-    virtual Result<std::shared_ptr<VectorSearchGlobalIndexResult>> VisitVectorSearch(
+    virtual Result<std::shared_ptr<ScoredGlobalIndexResult>> VisitVectorSearch(
         const std::shared_ptr<VectorSearch>& vector_search) = 0;
 
+    /// VisitFullTextSearch performs full text search.
+    virtual Result<std::shared_ptr<GlobalIndexResult>> VisitFullTextSearch(
+        const std::shared_ptr<FullTextSearch>& full_text_search) = 0;
+
     /// @return true if the reader is thread-safe; false otherwise.
     virtual bool IsThreadSafe() const = 0;
 

include/paimon/global_index/global_index_result.h

@@ -1,5 +1,5 @@
 /*
- * Copyright 2024-present Alibaba Inc.
+ * Copyright 2026-present Alibaba Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -76,7 +76,7 @@ class PAIMON_EXPORT GlobalIndexResult : public std::enable_shared_from_this<Glob
     /// Serializes a GlobalIndexResult object into a byte array.
     ///
     /// @note This method only supports the following concrete implementations:
-    ///       - BitmapVectorSearchGlobalIndexResult
+    ///       - BitmapScoredGlobalIndexResult
     ///       - BitmapGlobalIndexResult
     ///
     /// @param global_index_result The GlobalIndexResult instance to serialize (must not be null).
@@ -91,7 +91,7 @@ class PAIMON_EXPORT GlobalIndexResult : public std::enable_shared_from_this<Glob
     ///
     /// @note The concrete type of the deserialized object is determined by metadata
     ///       embedded in the buffer. Currently, only the following types are supported:
-    ///       - BitmapVectorSearchGlobalIndexResult
+    ///       - BitmapScoredGlobalIndexResult
     ///       - BitmapGlobalIndexResult
     ///
     /// @param buffer Pointer to the serialized byte data (must not be null).
@@ -106,18 +106,18 @@ class PAIMON_EXPORT GlobalIndexResult : public std::enable_shared_from_this<Glob
     static constexpr int32_t VERSION = 1;
 };
 
-/// Represents the result of a vector search query against a global index.
+/// Represents the result with score of a query against a global index.
 /// This class encapsulates a set of search candidates (row id + score pairs) and provides
 /// an iterator interface to traverse them.
-class PAIMON_EXPORT VectorSearchGlobalIndexResult : public GlobalIndexResult {
+class PAIMON_EXPORT ScoredGlobalIndexResult : public GlobalIndexResult {
  public:
-    /// An iterator over the vector search results, returning (row_id, score) pairs.
+    /// An iterator over the scored results, returning (row_id, score) pairs.
     ///
     /// @note The results are **NOT sorted by score**. Instead, they are returned in **ascending
     ///       order of row_id**.
-    class VectorSearchIterator {
+    class ScoredIterator {
      public:
-        virtual ~VectorSearchIterator() = default;
+        virtual ~ScoredIterator() = default;
 
         /// Checks whether more row ids are available.
         virtual bool HasNext() const = 0;
@@ -132,7 +132,7 @@ class PAIMON_EXPORT VectorSearchGlobalIndexResult : public GlobalIndexResult {
         virtual std::pair<int64_t, float> NextWithScore() = 0;
     };
 
-    /// Creates a new iterator for traversing the vector search results.
-    virtual Result<std::unique_ptr<VectorSearchIterator>> CreateVectorSearchIterator() const = 0;
+    /// Creates a new iterator for traversing the scored results.
+    virtual Result<std::unique_ptr<ScoredIterator>> CreateScoredIterator() const = 0;
 };
 }  // namespace paimon

include/paimon/orphan_files_cleaner.h

@@ -184,6 +184,11 @@ class PAIMON_EXPORT OrphanFilesCleaner {
     /// files.
     virtual Result<std::set<std::string>> Clean() = 0;
 
+    /// Retrieve metrics related to orphan files cleaning operations.
+    ///
+    /// @return A shared pointer to a `Metrics` object containing cleaning metrics.
+    virtual std::shared_ptr<Metrics> GetMetrics() const = 0;
+
  protected:
     OrphanFilesCleaner() = default;
 };

include/paimon/record_batch.h

@@ -93,8 +93,17 @@ class PAIMON_EXPORT RecordBatch {
 /// various properties such as data, row kinds, partition information, and bucket id.
 class PAIMON_EXPORT RecordBatchBuilder {
  public:
-    /// Constructs a `RecordBatchBuilder` with Arrow data.
-    /// @param data Arrow array containing the record data.
+    /// Constructs a `RecordBatchBuilder` with Arrow data
+    ///
+    /// @note The `data` must conform to table schema:
+    ///       - Each array in `data` corresponds to a field in table schema.
+    ///       - If a field in table schema is marked as non-nullable (`nullable = false`),
+    ///         the corresponding array in `data` must have zero null entries.
+    ///
+    /// @note Consistency between `data` and table schema will be validated during the write
+    /// process.
+    ///
+    /// @param data ArrowArray struct containing the columnar data (via C Data Interface)
     explicit RecordBatchBuilder(::ArrowArray* data);
 
     ~RecordBatchBuilder();

src/paimon/CMakeLists.txt

@@ -52,7 +52,7 @@ set(PAIMON_COMMON_SRCS
     common/fs/file_system_factory.cpp
     common/global_config.cpp
     common/global_index/complete_index_score_batch_reader.cpp
-    common/global_index/bitmap_vector_search_global_index_result.cpp
+    common/global_index/bitmap_scored_global_index_result.cpp
     common/global_index/bitmap_global_index_result.cpp
     common/global_index/global_index_result.cpp
     common/global_index/global_indexer_factory.cpp
@@ -360,7 +360,7 @@ if(PAIMON_BUILD_TESTS)
                     common/global_index/global_index_result_test.cpp
                     common/global_index/global_indexer_factory_test.cpp
                     common/global_index/bitmap_global_index_result_test.cpp
-                    common/global_index/bitmap_vector_search_global_index_result_test.cpp
+                    common/global_index/bitmap_scored_global_index_result_test.cpp
                     common/global_index/bitmap/bitmap_global_index_test.cpp
                     common/io/byte_array_input_stream_test.cpp
                     common/io/data_input_output_stream_test.cpp