feat(lumina): support Lumina tag filtering (alibaba#404)

Author: lxy-9602

include/paimon/global_index/global_index_io_meta.h

@@ -16,11 +16,11 @@
 
 #pragma once
 
+#include <cstdint>
 #include <memory>
 #include <string>
 
 #include "paimon/memory/bytes.h"
-#include "paimon/utils/range.h"
 
 namespace paimon {
 /// Metadata describing a single file entry in a global index.

include/paimon/global_index/global_indexer.h

@@ -17,6 +17,7 @@
 #pragma once
 
 #include <memory>
+#include <optional>
 #include <string>
 #include <vector>
 
@@ -36,6 +37,9 @@ class PAIMON_EXPORT GlobalIndexer {
  public:
     virtual ~GlobalIndexer() = default;
 
+    /// Returns additional table fields required during index construction.
+    virtual Result<std::optional<std::vector<std::string>>> GetExtraFieldNames() const = 0;
+
     /// Creates a writer for building a global index on a specific field.
     ///
     /// @param field_name     Name of the field to be indexed.

include/paimon/predicate/vector_search.h

@@ -74,6 +74,10 @@ struct PAIMON_EXPORT VectorSearch {
     /// context-aware filtering at query time.
     /// @note All fields referenced in the predicate must have been materialized
     ///       in the index during build to ensure availability.
+    /// @note For tag-based vector indexes, fields referenced by the predicate
+    ///       must keep the same names and types as the fields used during index
+    ///       construction. Indexes built with tag fields must not be reused across
+    ///       schema evolution that renames or changes those tag fields.
     std::shared_ptr<Predicate> predicate;
     /// The distance metric to use for this query, if explicitly specified.
     /// If set, this value must match the distance type used by the index (e.g., EUCLIDEAN, COSINE).

src/paimon/common/global_index/bitmap/bitmap_global_index.cpp

@@ -19,6 +19,10 @@
 #include "paimon/common/global_index/wrap/file_index_writer_wrapper.h"
 
 namespace paimon {
+Result<std::optional<std::vector<std::string>>> BitmapGlobalIndex::GetExtraFieldNames() const {
+    return std::optional<std::vector<std::string>>(std::nullopt);
+}
+
 Result<std::shared_ptr<GlobalIndexWriter>> BitmapGlobalIndex::CreateWriter(
     const std::string& field_name, ::ArrowSchema* arrow_schema,
     const std::shared_ptr<GlobalIndexFileWriter>& file_writer,

src/paimon/common/global_index/bitmap/bitmap_global_index.h

@@ -17,6 +17,7 @@
 #pragma once
 
 #include <memory>
+#include <optional>
 #include <string>
 #include <vector>
 
@@ -29,6 +30,8 @@ class BitmapGlobalIndex : public GlobalIndexer {
  public:
     explicit BitmapGlobalIndex(const std::shared_ptr<BitmapFileIndex>& index) : index_(index) {}
 
+    Result<std::optional<std::vector<std::string>>> GetExtraFieldNames() const override;
+
     Result<std::shared_ptr<GlobalIndexWriter>> CreateWriter(
         const std::string& field_name, ::ArrowSchema* arrow_schema,
         const std::shared_ptr<GlobalIndexFileWriter>& file_writer,

src/paimon/common/global_index/btree/btree_global_indexer.cpp

@@ -54,6 +54,10 @@ Result<std::unique_ptr<BTreeGlobalIndexer>> BTreeGlobalIndexer::Create(
     return std::unique_ptr<BTreeGlobalIndexer>(new BTreeGlobalIndexer(cache_manager, options));
 }
 
+Result<std::optional<std::vector<std::string>>> BTreeGlobalIndexer::GetExtraFieldNames() const {
+    return std::optional<std::vector<std::string>>(std::nullopt);
+}
+
 Result<std::shared_ptr<GlobalIndexWriter>> BTreeGlobalIndexer::CreateWriter(
     const std::string& field_name, ::ArrowSchema* arrow_schema,
     const std::shared_ptr<GlobalIndexFileWriter>& file_writer,

src/paimon/common/global_index/btree/btree_global_indexer.h

@@ -53,6 +53,8 @@ class BTreeGlobalIndexer : public GlobalIndexer {
     static Result<std::unique_ptr<BTreeGlobalIndexer>> Create(
         const std::map<std::string, std::string>& options);
 
+    Result<std::optional<std::vector<std::string>>> GetExtraFieldNames() const override;
+
     Result<std::shared_ptr<GlobalIndexWriter>> CreateWriter(
         const std::string& field_name, ::ArrowSchema* arrow_schema,
         const std::shared_ptr<GlobalIndexFileWriter>& file_writer,

src/paimon/common/global_index/rangebitmap/range_bitmap_global_index.cpp

@@ -19,6 +19,10 @@
 #include "paimon/common/global_index/wrap/file_index_writer_wrapper.h"
 
 namespace paimon {
+Result<std::optional<std::vector<std::string>>> RangeBitmapGlobalIndex::GetExtraFieldNames() const {
+    return std::optional<std::vector<std::string>>(std::nullopt);
+}
+
 Result<std::shared_ptr<GlobalIndexWriter>> RangeBitmapGlobalIndex::CreateWriter(
     const std::string& field_name, ::ArrowSchema* arrow_schema,
     const std::shared_ptr<GlobalIndexFileWriter>& file_writer,

src/paimon/common/global_index/rangebitmap/range_bitmap_global_index.h

@@ -17,6 +17,7 @@
 #pragma once
 
 #include <memory>
+#include <optional>
 #include <string>
 #include <vector>
 
@@ -30,6 +31,8 @@ class RangeBitmapGlobalIndex : public GlobalIndexer {
     explicit RangeBitmapGlobalIndex(const std::shared_ptr<RangeBitmapFileIndex>& index)
         : index_(index) {}
 
+    Result<std::optional<std::vector<std::string>>> GetExtraFieldNames() const override;
+
     Result<std::shared_ptr<GlobalIndexWriter>> CreateWriter(
         const std::string& field_name, ::ArrowSchema* arrow_schema,
         const std::shared_ptr<GlobalIndexFileWriter>& file_writer,

src/paimon/common/reader/data_evolution_file_reader.cpp

@@ -23,6 +23,7 @@
 #include "paimon/common/reader/reader_utils.h"
 #include "paimon/common/utils/arrow/mem_utils.h"
 #include "paimon/common/utils/arrow/status_utils.h"
+
 namespace paimon {
 Result<std::unique_ptr<DataEvolutionFileReader>> DataEvolutionFileReader::Create(
     std::vector<std::unique_ptr<BatchReader>>&& readers,
@@ -165,11 +166,10 @@ Result<std::shared_ptr<arrow::Array>> DataEvolutionFileReader::NextBatchForSingl
     if (concat_array_vec.empty()) {
         return std::shared_ptr<arrow::Array>();
     }
-    if (concat_array_vec.size() == 1) {
-        // avoid data copy
+    if (concat_array_vec.size() == 1 && concat_array_vec[0]->offset() == 0) {
+        // Avoid data copy when the array is already normalized.
         return concat_array_vec[0];
     }
-    // TODO(xinyu.lxy) remove data copy for efficiency
     PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Array> concat_array,
                                       arrow::Concatenate(concat_array_vec, arrow_pool_.get()));
     assert(concat_array->length() == total_array_length);