fix little

Author: lxy-9602

include/paimon/global_index/global_index_io_meta.h

@@ -18,32 +18,23 @@
 
 #include <cstdint>
 #include <memory>
-#include <optional>
 #include <string>
-#include <vector>
 
 #include "paimon/memory/bytes.h"
-#include "paimon/utils/range.h"
 
 namespace paimon {
 /// Metadata describing a single file entry in a global index.
 struct PAIMON_EXPORT GlobalIndexIOMeta {
     GlobalIndexIOMeta(const std::string& _file_path, int64_t _file_size,
-                      const std::shared_ptr<Bytes>& _metadata,
-                      const std::optional<std::vector<int32_t>>& _extra_field_ids)
-        : file_path(_file_path),
-          file_size(_file_size),
-          metadata(_metadata),
-          extra_field_ids(_extra_field_ids) {}
+                      const std::shared_ptr<Bytes>& _metadata)
+        : file_path(_file_path), file_size(_file_size), metadata(_metadata) {}
 
     std::string file_path;
     int64_t file_size;
     /// Optional binary metadata associated with the file, such as serialized
     /// secondary index structures or inline index bytes.
     /// May be null if no additional metadata is available.
     std::shared_ptr<Bytes> metadata;
-    /// Optional table field ids materialized together with the indexed field.
-    std::optional<std::vector<int32_t>> extra_field_ids;
 };
 
 }  // namespace paimon

src/paimon/common/global_index/btree/btree_compatibility_test.cpp

@@ -115,8 +115,7 @@ class BTreeCompatibilityTest : public ::testing::Test {
         PAIMON_ASSIGN_OR_RAISE(auto file_status, fs_->GetFileStatus(bin_path));
         auto file_size = file_status->GetLen();
 
-        GlobalIndexIOMeta io_meta(bin_path, file_size, meta_bytes,
-                                  /*extra_field_ids=*/std::nullopt);
+        GlobalIndexIOMeta io_meta(bin_path, file_size, meta_bytes);
         std::vector<GlobalIndexIOMeta> metas = {io_meta};
 
         auto schema = arrow::schema({arrow::field("testField", arrow_type)});

src/paimon/common/global_index/btree/btree_file_meta_selector_test.cpp

@@ -48,12 +48,12 @@ class BTreeFileMetaSelectorTest : public ::testing::Test {
         // file6: only-nulls file (no keys, has_nulls=true)
         auto meta6 = std::make_shared<BTreeIndexMeta>(nullptr, nullptr, true);
         files_ = {
-            GlobalIndexIOMeta("file1", 1, meta1->Serialize(pool_.get()), std::nullopt),
-            GlobalIndexIOMeta("file2", 1, meta2->Serialize(pool_.get()), std::nullopt),
-            GlobalIndexIOMeta("file3", 1, meta3->Serialize(pool_.get()), std::nullopt),
-            GlobalIndexIOMeta("file4", 1, meta4->Serialize(pool_.get()), std::nullopt),
-            GlobalIndexIOMeta("file5", 1, meta5->Serialize(pool_.get()), std::nullopt),
-            GlobalIndexIOMeta("file6", 1, meta6->Serialize(pool_.get()), std::nullopt),
+            GlobalIndexIOMeta("file1", 1, meta1->Serialize(pool_.get())),
+            GlobalIndexIOMeta("file2", 1, meta2->Serialize(pool_.get())),
+            GlobalIndexIOMeta("file3", 1, meta3->Serialize(pool_.get())),
+            GlobalIndexIOMeta("file4", 1, meta4->Serialize(pool_.get())),
+            GlobalIndexIOMeta("file5", 1, meta5->Serialize(pool_.get())),
+            GlobalIndexIOMeta("file6", 1, meta6->Serialize(pool_.get())),
         };
     }
 

src/paimon/common/global_index/btree/btree_global_index_integration_test.cpp

@@ -1703,7 +1703,7 @@ TEST_P(BTreeGlobalIndexIntegrationTest, CreateReaderWithMultiFieldSchema) {
                                                   {BtreeDefs::kBtreeIndexCompression, GetParam()}};
     ASSERT_OK_AND_ASSIGN(auto indexer, BTreeGlobalIndexer::Create(options));
 
-    GlobalIndexIOMeta meta("fake_path", 100, nullptr, /*extra_field_ids=*/std::nullopt);
+    GlobalIndexIOMeta meta("fake_path", 100, nullptr);
     std::vector<GlobalIndexIOMeta> metas = {meta};
 
     ASSERT_NOK_WITH_MSG(indexer->CreateReader(c_schema.get(), file_reader, metas, pool_),

src/paimon/common/global_index/btree/btree_global_index_writer.cpp

@@ -199,8 +199,7 @@ Result<std::vector<GlobalIndexIOMeta>> BTreeGlobalIndexWriter::Finish() {
     // Create GlobalIndexIOMeta
     std::string file_path = file_writer_->ToPath(index_file_name_);
     PAIMON_ASSIGN_OR_RAISE(int64_t file_size, file_writer_->GetFileSize(index_file_name_));
-    GlobalIndexIOMeta io_meta(file_path, file_size, meta_bytes,
-                              /*extra_field_ids=*/std::nullopt);
+    GlobalIndexIOMeta io_meta(file_path, file_size, meta_bytes);
     return std::vector<GlobalIndexIOMeta>{io_meta};
 }
 

src/paimon/common/global_index/wrap/file_index_writer_wrapper.h

@@ -66,7 +66,7 @@ class FileIndexWriterWrapper : public GlobalIndexWriter {
         PAIMON_RETURN_NOT_OK(out->Flush());
         PAIMON_RETURN_NOT_OK(out->Close());
         GlobalIndexIOMeta meta(file_manager_->ToPath(file_name), /*file_size=*/bytes->size(),
-                               /*metadata=*/nullptr, /*extra_field_ids=*/std::nullopt);
+                               /*metadata=*/nullptr);
         return std::vector<GlobalIndexIOMeta>({meta});
     }
 

src/paimon/core/global_index/global_index_scan_impl.cpp

@@ -223,7 +223,7 @@ GlobalIndexIOMeta GlobalIndexScanImpl::ToGlobalIndexIOMeta(
     assert(index_meta->GetGlobalIndexMeta());
     const auto& global_index_meta = index_meta->GetGlobalIndexMeta().value();
     return {index_file_manager_->ToPath(index_meta), index_meta->FileSize(),
-            global_index_meta.index_meta, global_index_meta.extra_field_ids};
+            global_index_meta.index_meta};
 }
 
 Result<std::shared_ptr<GlobalIndexResult>> GlobalIndexScanImpl::Scan(

src/paimon/core/global_index/global_index_write_task.cpp

@@ -96,8 +96,13 @@ Result<std::vector<DataField>> GetExtraFields(const TableSchema& table_schema,
     extra_fields.reserve(extra_field_names.size());
     std::set<std::string> dedup_field_names;
     for (const auto& extra_field_name : extra_field_names) {
-        if (extra_field_name == field_name || !dedup_field_names.insert(extra_field_name).second) {
-            continue;
+        if (extra_field_name == field_name) {
+            return Status::Invalid(fmt::format(
+                "global index extra field {} must not be the indexed field", extra_field_name));
+        }
+        if (!dedup_field_names.insert(extra_field_name).second) {
+            return Status::Invalid(fmt::format("global index extra field {} must not be duplicated",
+                                               extra_field_name));
         }
         PAIMON_ASSIGN_OR_RAISE(DataField extra_field, table_schema.GetField(extra_field_name));
         extra_fields.push_back(extra_field);
@@ -174,11 +179,6 @@ Result<std::vector<GlobalIndexIOMeta>> BuildIndex(
             return Status::Invalid(
                 "array read from batch reader is not a struct array in GlobalIndexWriteTask");
         }
-        auto indexed_array = struct_array->GetFieldByName(field_name);
-        if (!indexed_array) {
-            return Status::Invalid(fmt::format(
-                "read array does not contain {} field in GlobalIndexWriteTask", field_name));
-        }
         auto row_id_array = struct_array->GetFieldByName(SpecialFields::RowId().Name());
         auto typed_row_id_array = std::dynamic_pointer_cast<arrow::Int64Array>(row_id_array);
         if (!typed_row_id_array) {
@@ -222,7 +222,8 @@ Result<std::vector<GlobalIndexIOMeta>> BuildIndex(
 Result<std::shared_ptr<CommitMessage>> ToCommitMessage(
     const std::string& index_type, int32_t field_id, const Range& range,
     const std::vector<GlobalIndexIOMeta>& global_index_io_metas, const BinaryRow& partition,
-    int32_t bucket, const std::shared_ptr<GlobalIndexFileManager>& file_manager) {
+    int32_t bucket, const std::shared_ptr<GlobalIndexFileManager>& file_manager,
+    const std::optional<std::vector<int32_t>>& extra_field_ids) {
     std::vector<std::shared_ptr<IndexFileMeta>> index_file_metas;
     index_file_metas.reserve(global_index_io_metas.size());
     bool is_external_path = file_manager->IsExternalPath();
@@ -235,8 +236,7 @@ Result<std::shared_ptr<CommitMessage>> ToCommitMessage(
         index_file_metas.push_back(std::make_shared<IndexFileMeta>(
             index_type, PathUtil::GetName(io_meta.file_path), io_meta.file_size, range.Count(),
             /*dv_ranges=*/std::nullopt, external_path,
-            GlobalIndexMeta(range.from, range.to, field_id, io_meta.extra_field_ids,
-                            io_meta.metadata)));
+            GlobalIndexMeta(range.from, range.to, field_id, extra_field_ids, io_meta.metadata)));
     }
     DataIncrement data_increment(std::move(index_file_metas));
     return std::make_shared<CommitMessageImpl>(partition, bucket,
@@ -313,13 +313,11 @@ Result<std::shared_ptr<CommitMessage>> GlobalIndexWriteTask::WriteIndex(
     PAIMON_ASSIGN_OR_RAISE(std::vector<GlobalIndexIOMeta> global_index_io_metas,
                            BuildIndex(field_name, range, writer_field_names, batch_reader.get(),
                                       global_index_writer.get()));
-    for (auto& io_meta : global_index_io_metas) {
-        io_meta.extra_field_ids = extra_field_ids;
-    }
 
     // generate commit message
     return ToCommitMessage(index_type, field.Id(), range, global_index_io_metas,
-                           data_split->Partition(), data_split->Bucket(), index_file_manager);
+                           data_split->Partition(), data_split->Bucket(), index_file_manager,
+                           extra_field_ids);
 }
 
 }  // namespace paimon

src/paimon/global_index/lucene/lucene_global_index_writer.cpp

@@ -237,7 +237,7 @@ Result<std::vector<GlobalIndexIOMeta>> LuceneGlobalIndexWriter::Finish() {
     PAIMON_RETURN_NOT_OK(RapidJsonUtil::ToJsonString(options_, &options_json));
     auto meta_bytes = std::make_shared<Bytes>(options_json, pool_.get());
     GlobalIndexIOMeta meta(file_writer_->ToPath(index_file_name), file_size,
-                           /*metadata=*/meta_bytes, /*extra_field_ids=*/std::nullopt);
+                           /*metadata=*/meta_bytes);
     return std::vector<GlobalIndexIOMeta>({meta});
 }