fix(btree): preserve empty keys in index metadata & disable global bitmap index (alibaba#436)

Author: lxy-9602

README.md

@@ -32,10 +32,17 @@ Paimon C++ is a high-performance C++ implementation of [Apache Paimon](https://p
 - **File systems**: file system abstraction with built-in local and Jindo file system support.
 - **File formats**: file format abstraction with built-in ORC, Parquet, and Avro support.
 - **Runtime utilities**: memory pool and thread pool abstractions with default implementations.
-- **AI-Oriented Features**: supports RowTracking and DataEvolution mode and provides Global Index capabilities including bitmap index, B-tree index, DiskANN-based vector search with Lumina, and Lucene-based full-text search.
+- **AI-Oriented Features**: supports RowTracking and DataEvolution mode and provides Global Index
+  capabilities including B-tree index, DiskANN-based vector search with Lumina, and Lucene-based
+  full-text search.
 - **Compatibility**: compatibility with Apache Paimon Java format and communication protocols,
   including commit messages, data splits, and manifests.
 
+> **Bitmap global index compatibility:** Java Paimon now uses a dedicated bitmap global index
+> format instead of the previously shared wrapped bitmap file index format.
+> Paimon C++ therefore currently treats the `bitmap` global index type as unsupported. The legacy
+> implementation remains in the codebase pending migration to the Java-compatible format.
+
 Note: Linux x86_64 and macOS arm64 builds are currently verified.
 
 ## Write And Commit Example

include/paimon/global_index/global_index_reader.h

@@ -47,7 +47,7 @@ class PAIMON_EXPORT GlobalIndexReader : public FunctionVisitor<std::shared_ptr<G
     /// @return true if the reader is thread-safe; false otherwise.
     virtual bool IsThreadSafe() const = 0;
 
-    /// @return An identifier representing the index type. (e.g., "bitmap", "lumina").
+    /// @return An identifier representing the index type. (e.g., "btree", "lumina").
     virtual std::string GetIndexType() const = 0;
 };
 

include/paimon/global_index/global_index_write_task.h

@@ -36,7 +36,7 @@ class PAIMON_EXPORT GlobalIndexWriteTask {
     ///
     /// @param table_path   Path to the table root directory where index files are stored.
     /// @param field_name   Name of the indexed column (must be present in the table schema).
-    /// @param index_type   Type of global index to build (e.g., "bitmap", "lumina").
+    /// @param index_type   Type of global index to build (e.g., "btree", "lumina").
     /// @param index_split  The indexed split containing the actual data (e.g., Parquet file) and
     //                      row id range [from, to] for data to build index.
     ///                     The range must be fully contained within the data covered

include/paimon/global_index/global_indexer_factory.h

@@ -32,7 +32,7 @@ class PAIMON_EXPORT GlobalIndexerFactory : public Factory {
  public:
     ~GlobalIndexerFactory() override = default;
 
-    /// Suffix used to distinguish global index identifiers (e.g., "bitmap-global").
+    /// Suffix used to distinguish global index identifiers (e.g., "btree-global").
     static const char GLOBAL_INDEX_IDENTIFIER_SUFFIX[];
 
     /// Creates a `GlobalIndexer` instance by looking up a registered factory using an identifier.
@@ -41,11 +41,11 @@ class PAIMON_EXPORT GlobalIndexerFactory : public Factory {
     /// (e.g., "-global") to form the full key used for factory lookup. This ensures namespace
     /// separation between file and global index types.
     ///
-    /// @param identifier The base name of the index type (e.g., "bitmap").
+    /// @param identifier The base name of the index type (e.g., "btree").
     /// @param options    Configuration parameters for the indexer.
     /// @return A `Result` containing a unique pointer to the created `GlobalIndexer`,
     ///         or an error if creation fails.
-    /// @return nullptr if no matching factory.
+    /// @return nullptr if no matching factory exists or the index type is temporarily unsupported.
     static Result<std::unique_ptr<GlobalIndexer>> Get(
         const std::string& identifier, const std::map<std::string, std::string>& options);
 

src/paimon/common/global_index/btree/btree_file_meta_selector.cpp

@@ -46,11 +46,7 @@ Result<std::vector<GlobalIndexIOMeta>> BTreeFileMetaSelector::VisitEqual(const L
         if (meta.OnlyNulls()) {
             return false;
         }
-        MemorySlice min_key_slice = WrapKeySlice(meta.FirstKey());
-        MemorySlice max_key_slice = WrapKeySlice(meta.LastKey());
-        PAIMON_ASSIGN_OR_RAISE(int32_t cmp_min, comparator_(literal_slice, min_key_slice));
-        PAIMON_ASSIGN_OR_RAISE(int32_t cmp_max, comparator_(literal_slice, max_key_slice));
-        return cmp_min >= 0 && cmp_max <= 0;
+        return Overlaps(meta, literal_slice, literal_slice);
     });
 }
 
@@ -67,8 +63,7 @@ Result<std::vector<GlobalIndexIOMeta>> BTreeFileMetaSelector::VisitLessThan(
         if (meta.OnlyNulls()) {
             return false;
         }
-        MemorySlice min_key_slice = WrapKeySlice(meta.FirstKey());
-        PAIMON_ASSIGN_OR_RAISE(int32_t cmp, comparator_(min_key_slice, literal_slice));
+        PAIMON_ASSIGN_OR_RAISE(int32_t cmp, CompareFirstKey(meta, literal_slice));
         return cmp < 0;
     });
 }
@@ -81,8 +76,7 @@ Result<std::vector<GlobalIndexIOMeta>> BTreeFileMetaSelector::VisitLessOrEqual(
         if (meta.OnlyNulls()) {
             return false;
         }
-        MemorySlice min_key_slice = WrapKeySlice(meta.FirstKey());
-        PAIMON_ASSIGN_OR_RAISE(int32_t cmp, comparator_(min_key_slice, literal_slice));
+        PAIMON_ASSIGN_OR_RAISE(int32_t cmp, CompareFirstKey(meta, literal_slice));
         return cmp <= 0;
     });
 }
@@ -95,8 +89,7 @@ Result<std::vector<GlobalIndexIOMeta>> BTreeFileMetaSelector::VisitGreaterThan(
         if (meta.OnlyNulls()) {
             return false;
         }
-        MemorySlice max_key_slice = WrapKeySlice(meta.LastKey());
-        PAIMON_ASSIGN_OR_RAISE(int32_t cmp, comparator_(max_key_slice, literal_slice));
+        PAIMON_ASSIGN_OR_RAISE(int32_t cmp, CompareLastKey(meta, literal_slice));
         return cmp > 0;
     });
 }
@@ -109,8 +102,7 @@ Result<std::vector<GlobalIndexIOMeta>> BTreeFileMetaSelector::VisitGreaterOrEqua
         if (meta.OnlyNulls()) {
             return false;
         }
-        MemorySlice max_key_slice = WrapKeySlice(meta.LastKey());
-        PAIMON_ASSIGN_OR_RAISE(int32_t cmp, comparator_(max_key_slice, literal_slice));
+        PAIMON_ASSIGN_OR_RAISE(int32_t cmp, CompareLastKey(meta, literal_slice));
         return cmp >= 0;
     });
 }
@@ -127,12 +119,9 @@ Result<std::vector<GlobalIndexIOMeta>> BTreeFileMetaSelector::VisitIn(
         if (meta.OnlyNulls()) {
             return false;
         }
-        MemorySlice min_key_slice = WrapKeySlice(meta.FirstKey());
-        MemorySlice max_key_slice = WrapKeySlice(meta.LastKey());
         for (const auto& literal_slice : literal_slices) {
-            PAIMON_ASSIGN_OR_RAISE(int32_t cmp_min, comparator_(literal_slice, min_key_slice));
-            PAIMON_ASSIGN_OR_RAISE(int32_t cmp_max, comparator_(literal_slice, max_key_slice));
-            if (cmp_min >= 0 && cmp_max <= 0) {
+            PAIMON_ASSIGN_OR_RAISE(bool overlaps, Overlaps(meta, literal_slice, literal_slice));
+            if (overlaps) {
                 return true;
             }
         }
@@ -176,6 +165,39 @@ Result<std::vector<GlobalIndexIOMeta>> BTreeFileMetaSelector::Filter(
     return result;
 }
 
+Result<bool> BTreeFileMetaSelector::Overlaps(const BTreeIndexMeta& meta, const MemorySlice& from,
+                                             const MemorySlice& to) const {
+    if (meta.FirstKey()) {
+        PAIMON_ASSIGN_OR_RAISE(int32_t cmp, comparator_(to, WrapKeySlice(meta.FirstKey())));
+        if (cmp < 0) {
+            return false;
+        }
+    }
+    if (meta.LastKey()) {
+        PAIMON_ASSIGN_OR_RAISE(int32_t cmp, comparator_(from, WrapKeySlice(meta.LastKey())));
+        if (cmp > 0) {
+            return false;
+        }
+    }
+    return true;
+}
+
+Result<int32_t> BTreeFileMetaSelector::CompareFirstKey(const BTreeIndexMeta& meta,
+                                                       const MemorySlice& literal) const {
+    if (!meta.FirstKey()) {
+        return -1;
+    }
+    return comparator_(WrapKeySlice(meta.FirstKey()), literal);
+}
+
+Result<int32_t> BTreeFileMetaSelector::CompareLastKey(const BTreeIndexMeta& meta,
+                                                      const MemorySlice& literal) const {
+    if (!meta.LastKey()) {
+        return 1;
+    }
+    return comparator_(WrapKeySlice(meta.LastKey()), literal);
+}
+
 MemorySlice BTreeFileMetaSelector::WrapKeySlice(const std::shared_ptr<Bytes>& key) {
     return MemorySlice::Wrap(MemorySegment::WrapView(key->data(), key->size()));
 }

src/paimon/common/global_index/btree/btree_file_meta_selector.h

@@ -56,6 +56,13 @@ class BTreeFileMetaSelector : public FunctionVisitor<std::vector<GlobalIndexIOMe
 
     Result<std::vector<GlobalIndexIOMeta>> Filter(const MetaPredicate& predicate) const;
 
+    Result<bool> Overlaps(const BTreeIndexMeta& meta, const MemorySlice& from,
+                          const MemorySlice& to) const;
+
+    Result<int32_t> CompareFirstKey(const BTreeIndexMeta& meta, const MemorySlice& literal) const;
+
+    Result<int32_t> CompareLastKey(const BTreeIndexMeta& meta, const MemorySlice& literal) const;
+
     Result<MemorySlice> SerializeLiteral(const Literal& literal) const;
 
     /// Create a non-owning MemorySlice view over the raw bytes of a key,

src/paimon/common/global_index/btree/btree_file_meta_selector_test.cpp

@@ -225,4 +225,43 @@ TEST_F(BTreeFileMetaSelectorTest, TestOnlyNullsFileExcludedFromRangeQueries) {
     ASSERT_EQ(names.count("file6"), 1u);
 }
 
+TEST_F(BTreeFileMetaSelectorTest, TestEmptyStringKeyDoesNotCrash) {
+    std::shared_ptr<MemoryPool> pool = GetDefaultPool();
+    std::shared_ptr<arrow::DataType> key_type = arrow::utf8();
+    auto serialize = [&](const char* value, size_t size) {
+        Literal literal(FieldType::STRING, value, size);
+        EXPECT_OK_AND_ASSIGN(std::shared_ptr<Bytes> result,
+                             KeySerializer::SerializeKey(literal, key_type, pool.get()));
+        return result;
+    };
+
+    auto empty_meta =
+        std::make_shared<BTreeIndexMeta>(serialize("", 0), serialize("www.example.com", 15), false);
+    auto normal_meta =
+        std::make_shared<BTreeIndexMeta>(serialize("aaa.com", 7), serialize("zzz.com", 7), false);
+    auto null_meta = std::make_shared<BTreeIndexMeta>(nullptr, nullptr, true);
+    std::vector<GlobalIndexIOMeta> files = {
+        GlobalIndexIOMeta("file_empty", 1, empty_meta->Serialize(pool.get())),
+        GlobalIndexIOMeta("file_normal", 1, normal_meta->Serialize(pool.get())),
+        GlobalIndexIOMeta("file_nulls", 1, null_meta->Serialize(pool.get())),
+    };
+
+    BTreeFileMetaSelector selector(files, key_type, pool);
+
+    ASSERT_OK_AND_ASSIGN(std::vector<GlobalIndexIOMeta> result,
+                         selector.VisitEqual(Literal(FieldType::STRING, "www.example.com", 15)));
+    CheckResult(result, {"file_empty", "file_normal"});
+
+    ASSERT_OK_AND_ASSIGN(result, selector.VisitLessThan(Literal(FieldType::STRING, "bbb.com", 7)));
+    CheckResult(result, {"file_empty", "file_normal"});
+
+    ASSERT_OK_AND_ASSIGN(
+        result, selector.VisitGreaterThan(Literal(FieldType::STRING, "www.example.com", 15)));
+    CheckResult(result, {"file_normal"});
+
+    ASSERT_OK_AND_ASSIGN(result, selector.VisitIn({Literal(FieldType::STRING, "", 0),
+                                                   Literal(FieldType::STRING, "zzz.com", 7)}));
+    CheckResult(result, {"file_empty", "file_normal"});
+}
+
 }  // namespace paimon::test

src/paimon/common/global_index/btree/btree_global_index_integration_test.cpp

@@ -20,6 +20,7 @@
 #include "paimon/common/factories/io_hook.h"
 #include "paimon/common/global_index/btree/btree_global_index_writer.h"
 #include "paimon/common/global_index/btree/btree_global_indexer.h"
+#include "paimon/common/global_index/btree/btree_index_meta.h"
 #include "paimon/common/global_index/btree/lazy_filtered_btree_reader.h"
 #include "paimon/common/options/memory_size.h"
 #include "paimon/common/utils/scope_guard.h"
@@ -457,6 +458,50 @@ TEST_P(BTreeGlobalIndexIntegrationTest, WriteAndReadStringData) {
     }
 }
 
+TEST_P(BTreeGlobalIndexIntegrationTest, WriteEmptyStringKeyMetadata) {
+    auto file_writer = std::make_shared<FakeGlobalIndexFileWriter>(fs_, base_path_);
+    auto field = arrow::field("str_field", arrow::utf8());
+    auto c_schema = CreateArrowSchema(field);
+
+    std::map<std::string, std::string> options = {{BtreeDefs::kBtreeIndexBlockSize, "128"},
+                                                  {BtreeDefs::kBtreeIndexCompression, GetParam()}};
+    ASSERT_OK_AND_ASSIGN(auto indexer, BTreeGlobalIndexer::Create(options));
+    ASSERT_OK_AND_ASSIGN(auto writer,
+                         indexer->CreateWriter("str_field", c_schema.get(), file_writer, pool_));
+    auto array = arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_({field}), R"([
+        [null],
+        [""],
+        ["abc"]
+    ])")
+                     .ValueOrDie();
+
+    ArrowArray c_array;
+    ASSERT_TRUE(arrow::ExportArray(*array, &c_array).ok());
+    std::vector<int64_t> row_ids(array->length());
+    std::iota(row_ids.begin(), row_ids.end(), 0);
+    ASSERT_OK(writer->AddBatch(&c_array, std::move(row_ids)));
+    ASSERT_OK_AND_ASSIGN(auto metas, writer->Finish());
+    ASSERT_EQ(metas.size(), 1);
+
+    std::shared_ptr<BTreeIndexMeta> meta =
+        BTreeIndexMeta::Deserialize(metas[0].metadata, pool_.get());
+    ASSERT_TRUE(meta->FirstKey());
+    ASSERT_EQ(meta->FirstKey()->size(), 0);
+    ASSERT_TRUE(meta->LastKey());
+    ASSERT_EQ(std::string(meta->LastKey()->data(), meta->LastKey()->size()), "abc");
+    ASSERT_TRUE(meta->HasNulls());
+    ASSERT_FALSE(meta->OnlyNulls());
+
+    auto file_reader = std::make_shared<FakeGlobalIndexFileReader>(fs_, base_path_);
+    c_schema = CreateArrowSchema(field);
+    ASSERT_OK_AND_ASSIGN(auto reader,
+                         indexer->CreateReader(c_schema.get(), file_reader, metas, pool_));
+
+    Literal empty(FieldType::STRING, "", 0);
+    ASSERT_OK_AND_ASSIGN(auto result, reader->VisitEqual(empty));
+    CheckResult(result, {1});
+}
+
 TEST_P(BTreeGlobalIndexIntegrationTest, WriteAndReadBigIntData) {
     auto file_writer = std::make_shared<FakeGlobalIndexFileWriter>(fs_, base_path_);
     auto field = arrow::field("bigint_field", arrow::int64());

src/paimon/common/global_index/btree/btree_index_meta.cpp

@@ -19,48 +19,71 @@
 #include "paimon/common/memory/memory_slice_output.h"
 
 namespace paimon {
+namespace {
+
+std::shared_ptr<Bytes> ReadKey(MemorySliceInput* input, int32_t key_length, MemoryPool* pool) {
+    if (key_length == 0) {
+        return std::make_shared<Bytes>(0, pool);
+    }
+    return input->ReadSliceView(key_length).CopyBytes(pool);
+}
+
+}  // namespace
 
 std::shared_ptr<BTreeIndexMeta> BTreeIndexMeta::Deserialize(const std::shared_ptr<Bytes>& meta,
                                                             paimon::MemoryPool* pool) {
-    auto slice = MemorySlice::Wrap(meta);
-    auto input = slice.ToInput();
-    auto first_key_len = input.ReadInt();
-    std::shared_ptr<Bytes> first_key;
-    if (first_key_len) {
-        first_key = input.ReadSliceView(first_key_len).CopyBytes(pool);
-    }
-    auto last_key_len = input.ReadInt();
-    std::shared_ptr<Bytes> last_key;
-    if (last_key_len) {
-        last_key = input.ReadSliceView(last_key_len).CopyBytes(pool);
+    MemorySlice slice = MemorySlice::Wrap(meta);
+    MemorySliceInput input = slice.ToInput();
+    int32_t first_key_len = input.ReadInt();
+    std::shared_ptr<Bytes> first_key = ReadKey(&input, first_key_len, pool);
+    int32_t last_key_len = input.ReadInt();
+    std::shared_ptr<Bytes> last_key = ReadKey(&input, last_key_len, pool);
+    bool has_nulls = input.ReadByte() == static_cast<int8_t>(1);
+
+    if (input.Available() >= 2) {
+        int8_t format_version = input.ReadByte();
+        if (format_version == kFormatVersionWithNullFlags) {
+            int8_t null_key_flags = input.ReadByte();
+            if ((null_key_flags & kFirstKeyIsNull) != 0) {
+                first_key.reset();
+            }
+            if ((null_key_flags & kLastKeyIsNull) != 0) {
+                last_key.reset();
+            }
+        }
+    } else if (first_key_len == 0 && last_key_len == 0 && has_nulls) {
+        // Legacy metadata used zero length for null keys. Both empty boundaries plus a null bitmap
+        // identify an all-null file; a single empty boundary remains a valid serialized key.
+        first_key.reset();
+        last_key.reset();
     }
-    auto has_nulls = input.ReadByte() == static_cast<int8_t>(1);
     return std::make_shared<BTreeIndexMeta>(first_key, last_key, has_nulls);
 }
 
 std::shared_ptr<Bytes> BTreeIndexMeta::Serialize(paimon::MemoryPool* pool) const {
-    // Calculate total size: first_key_len(4) + first_key + last_key_len(4) + last_key +
-    // has_nulls(1)
     int32_t first_key_size = first_key_ ? first_key_->size() : 0;
     int32_t last_key_size = last_key_ ? last_key_->size() : 0;
     int32_t total_size = Size();
-
     MemorySliceOutput output(total_size, pool);
+    int8_t null_key_flags = 0;
 
-    // Write first_key_len and first_key
     output.WriteValue(first_key_size);
     if (first_key_) {
         output.WriteBytes(first_key_);
+    } else {
+        null_key_flags |= kFirstKeyIsNull;
     }
 
-    // Write last_key_len and last_key
     output.WriteValue(last_key_size);
     if (last_key_) {
         output.WriteBytes(last_key_);
+    } else {
+        null_key_flags |= kLastKeyIsNull;
     }
 
-    // Write has_nulls
     output.WriteValue(static_cast<int8_t>(has_nulls_ ? 1 : 0));
+    output.WriteValue(kFormatVersionWithNullFlags);
+    output.WriteValue(null_key_flags);
 
     return output.ToSlice().GetOrCreateHeapMemory(pool);
 }