feat: implement RangeBitmapGlobalIndex for global range-bitmap index support (alibaba#199)

Author: lxy-9602

src/paimon/CMakeLists.txt

@@ -396,6 +396,8 @@ if(PAIMON_BUILD_TESTS)
                     common/global_index/bitmap_global_index_result_test.cpp
                     common/global_index/bitmap_scored_global_index_result_test.cpp
                     common/global_index/bitmap/bitmap_global_index_test.cpp
+                    common/global_index/rangebitmap/range_bitmap_global_index_test.cpp
+                    common/global_index/wrap/file_index_reader_wrapper_test.cpp
                     common/io/byte_array_input_stream_test.cpp
                     common/io/data_input_output_stream_test.cpp
                     common/io/buffered_input_stream_test.cpp

src/paimon/common/file_index/rangebitmap/range_bitmap_file_index.h

@@ -28,13 +28,14 @@
 #include "paimon/predicate/literal.h"
 #include "paimon/result.h"
 #include "paimon/status.h"
+#include "paimon/visibility.h"
 
 namespace paimon {
 
 class RangeBitmapFileIndexWriter;
 class RangeBitmapFileIndexReader;
 
-class RangeBitmapFileIndex final : public FileIndexer {
+class PAIMON_EXPORT RangeBitmapFileIndex final : public FileIndexer {
  public:
     explicit RangeBitmapFileIndex(const std::map<std::string, std::string>& options);
 

src/paimon/common/global_index/CMakeLists.txt

@@ -12,14 +12,17 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-set(PAIMON_GLOBAL_INDEX_SRC bitmap/bitmap_global_index.cpp
-                            bitmap/bitmap_global_index_factory.cpp)
+set(PAIMON_GLOBAL_INDEX_SRC
+    bitmap/bitmap_global_index.cpp bitmap/bitmap_global_index_factory.cpp
+    rangebitmap/range_bitmap_global_index.cpp
+    rangebitmap/range_bitmap_global_index_factory.cpp)
 
 add_paimon_lib(paimon_global_index
                SOURCES
                ${PAIMON_GLOBAL_INDEX_SRC}
                DEPENDENCIES
                paimon_shared
+               paimon_file_index_shared
                STATIC_LINK_LIBS
                arrow
                fmt

src/paimon/common/global_index/bitmap/bitmap_global_index.cpp

@@ -17,9 +17,6 @@
 
 #include "paimon/common/global_index/wrap/file_index_reader_wrapper.h"
 #include "paimon/common/global_index/wrap/file_index_writer_wrapper.h"
-#include "paimon/file_index/bitmap_index_result.h"
-#include "paimon/global_index/bitmap_global_index_result.h"
-#include "paimon/utils/roaring_bitmap64.h"
 
 namespace paimon {
 Result<std::shared_ptr<GlobalIndexWriter>> BitmapGlobalIndex::CreateWriter(
@@ -47,31 +44,9 @@ Result<std::shared_ptr<GlobalIndexReader>> BitmapGlobalIndex::CreateReader(
         index_->CreateReader(arrow_schema, /*start=*/0, meta.file_size, in, pool));
     auto transform = [range_end = meta.range_end](const std::shared_ptr<FileIndexResult>& result)
         -> Result<std::shared_ptr<GlobalIndexResult>> {
-        return ToGlobalIndexResult(range_end, result);
+        return FileIndexReaderWrapper::ToGlobalIndexResult(range_end, result);
     };
     return std::make_shared<BitmapGlobalIndexReader>(reader, transform);
 }
 
-Result<std::shared_ptr<GlobalIndexResult>> BitmapGlobalIndex::ToGlobalIndexResult(
-    int64_t range_end, const std::shared_ptr<FileIndexResult>& result) {
-    if (auto remain = std::dynamic_pointer_cast<Remain>(result)) {
-        return std::make_shared<BitmapGlobalIndexResult>([range_end]() -> Result<RoaringBitmap64> {
-            RoaringBitmap64 bitmap;
-            bitmap.AddRange(0, range_end + 1);
-            return bitmap;
-        });
-    } else if (auto skip = std::dynamic_pointer_cast<Skip>(result)) {
-        return std::make_shared<BitmapGlobalIndexResult>(
-            []() -> Result<RoaringBitmap64> { return RoaringBitmap64(); });
-    } else if (auto bitmap_result = std::dynamic_pointer_cast<BitmapIndexResult>(result)) {
-        return std::make_shared<BitmapGlobalIndexResult>(
-            [bitmap_result]() -> Result<RoaringBitmap64> {
-                PAIMON_ASSIGN_OR_RAISE(const RoaringBitmap32* bitmap, bitmap_result->GetBitmap());
-                return RoaringBitmap64(*bitmap);
-            });
-    }
-    return Status::Invalid(
-        "invalid FileIndexResult, supposed to be Remain or Skip or BitmapIndexResult");
-}
-
 }  // namespace paimon

src/paimon/common/global_index/bitmap/bitmap_global_index.h

@@ -39,10 +39,6 @@ class BitmapGlobalIndex : public GlobalIndexer {
         const std::vector<GlobalIndexIOMeta>& files,
         const std::shared_ptr<MemoryPool>& pool) const override;
 
- private:
-    static Result<std::shared_ptr<GlobalIndexResult>> ToGlobalIndexResult(
-        int64_t range_end, const std::shared_ptr<FileIndexResult>& result);
-
  private:
     std::shared_ptr<BitmapFileIndex> index_;
 };

src/paimon/common/global_index/bitmap/bitmap_global_index_test.cpp

@@ -30,41 +30,20 @@
 #include "paimon/common/utils/path_util.h"
 #include "paimon/common/utils/string_utils.h"
 #include "paimon/core/global_index/global_index_file_manager.h"
-#include "paimon/core/index/index_path_factory.h"
 #include "paimon/data/timestamp.h"
 #include "paimon/file_index/bitmap_index_result.h"
 #include "paimon/fs/local/local_file_system.h"
 #include "paimon/global_index/bitmap_global_index_result.h"
 #include "paimon/global_index/global_index_result.h"
+#include "paimon/testing/mock/mock_index_path_factory.h"
 #include "paimon/testing/utils/testharness.h"
+
 namespace paimon::test {
 class BitmapGlobalIndexTest : public ::testing::Test {
  public:
     void SetUp() override {}
     void TearDown() override {}
 
-    class FakeIndexPathFactory : public IndexPathFactory {
-     public:
-        explicit FakeIndexPathFactory(const std::string& index_path) : index_path_(index_path) {}
-        std::string NewPath() const override {
-            assert(false);
-            return "";
-        }
-        std::string ToPath(const std::shared_ptr<IndexFileMeta>& file) const override {
-            assert(false);
-            return "";
-        }
-        std::string ToPath(const std::string& file_name) const override {
-            return PathUtil::JoinPath(index_path_, file_name);
-        }
-        bool IsExternalPath() const override {
-            return false;
-        }
-
-     private:
-        std::string index_path_;
-    };
-
     std::unique_ptr<::ArrowSchema> CreateArrowSchema(
         const std::shared_ptr<arrow::DataType>& data_type) const {
         auto schema = arrow::schema({arrow::field("f0", data_type)});
@@ -83,7 +62,7 @@ class BitmapGlobalIndexTest : public ::testing::Test {
         auto file_index = std::make_shared<BitmapFileIndex>(options);
         auto global_index = std::make_shared<BitmapGlobalIndex>(file_index);
 
-        auto path_factory = std::make_shared<FakeIndexPathFactory>(index_root);
+        auto path_factory = std::make_shared<MockIndexPathFactory>(index_root);
         auto file_writer = std::make_shared<GlobalIndexFileManager>(fs_, path_factory);
 
         PAIMON_ASSIGN_OR_RAISE(
@@ -125,7 +104,7 @@ class BitmapGlobalIndexTest : public ::testing::Test {
         auto file_index = std::make_shared<BitmapFileIndex>(std::map<std::string, std::string>());
         auto global_index = std::make_shared<BitmapGlobalIndex>(file_index);
 
-        auto path_factory = std::make_shared<FakeIndexPathFactory>(index_root);
+        auto path_factory = std::make_shared<MockIndexPathFactory>(index_root);
         auto file_reader = std::make_shared<GlobalIndexFileManager>(fs_, path_factory);
         EXPECT_OK_AND_ASSIGN(
             auto global_index_reader,
@@ -139,42 +118,6 @@ class BitmapGlobalIndexTest : public ::testing::Test {
     std::shared_ptr<FileSystem> fs_ = std::make_shared<LocalFileSystem>();
 };
 
-TEST_F(BitmapGlobalIndexTest, TestToGlobalIndexResult) {
-    {
-        ASSERT_OK_AND_ASSIGN(auto global_result, BitmapGlobalIndex::ToGlobalIndexResult(
-                                                     /*range_end=*/5l, FileIndexResult::Remain()));
-        CheckResult(global_result, {0l, 1l, 2l, 3l, 4l, 5l});
-    }
-    {
-        ASSERT_OK_AND_ASSIGN(auto global_result, BitmapGlobalIndex::ToGlobalIndexResult(
-                                                     /*range_end=*/5l, FileIndexResult::Skip()));
-        CheckResult(global_result, {});
-    }
-    {
-        auto bitmap_supplier = []() -> Result<RoaringBitmap32> {
-            return RoaringBitmap32::From({1, 4, 2147483647});
-        };
-        auto file_result = std::make_shared<BitmapIndexResult>(bitmap_supplier);
-        ASSERT_OK_AND_ASSIGN(auto global_result, BitmapGlobalIndex::ToGlobalIndexResult(
-                                                     /*range_end=*/2147483647l, file_result));
-        CheckResult(global_result, {1l, 4l, 2147483647l});
-    }
-    {
-        class FakeFileIndexResult : public FileIndexResult {
-            Result<bool> IsRemain() const override {
-                return true;
-            }
-            std::string ToString() const override {
-                return "fake file index result";
-            }
-        };
-        auto file_result = std::make_shared<FakeFileIndexResult>();
-        ASSERT_NOK_WITH_MSG(
-            BitmapGlobalIndex::ToGlobalIndexResult(/*range_end=*/10l, file_result),
-            "invalid FileIndexResult, supposed to be Remain or Skip or BitmapIndexResult");
-    }
-}
-
 TEST_F(BitmapGlobalIndexTest, TestStringType) {
     auto test_root_dir = UniqueTestDirectory::Create();
     ASSERT_TRUE(test_root_dir);

src/paimon/common/global_index/rangebitmap/range_bitmap_global_index.cpp

@@ -0,0 +1,52 @@
+/*
+ * Copyright 2026-present Alibaba Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "paimon/common/global_index/rangebitmap/range_bitmap_global_index.h"
+
+#include "paimon/common/global_index/wrap/file_index_reader_wrapper.h"
+#include "paimon/common/global_index/wrap/file_index_writer_wrapper.h"
+
+namespace paimon {
+Result<std::shared_ptr<GlobalIndexWriter>> RangeBitmapGlobalIndex::CreateWriter(
+    const std::string& field_name, ::ArrowSchema* arrow_schema,
+    const std::shared_ptr<GlobalIndexFileWriter>& file_writer,
+    const std::shared_ptr<MemoryPool>& pool) const {
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<FileIndexWriter> writer,
+                           index_->CreateWriter(arrow_schema, pool));
+    return std::make_shared<FileIndexWriterWrapper>(
+        /*index_type=*/"range-bitmap", file_writer, writer);
+}
+
+Result<std::shared_ptr<GlobalIndexReader>> RangeBitmapGlobalIndex::CreateReader(
+    ::ArrowSchema* arrow_schema, const std::shared_ptr<GlobalIndexFileReader>& file_reader,
+    const std::vector<GlobalIndexIOMeta>& files, const std::shared_ptr<MemoryPool>& pool) const {
+    if (files.size() != 1) {
+        return Status::Invalid(
+            "invalid GlobalIndexIOMeta for RangeBitmapGlobalIndex, exist multiple metas");
+    }
+    const auto& meta = files[0];
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<InputStream> in,
+                           file_reader->GetInputStream(meta.file_path));
+    PAIMON_ASSIGN_OR_RAISE(
+        std::shared_ptr<FileIndexReader> reader,
+        index_->CreateReader(arrow_schema, /*start=*/0, meta.file_size, in, pool));
+    auto transform = [range_end = meta.range_end](const std::shared_ptr<FileIndexResult>& result)
+        -> Result<std::shared_ptr<GlobalIndexResult>> {
+        return FileIndexReaderWrapper::ToGlobalIndexResult(range_end, result);
+    };
+    return std::make_shared<RangeBitmapGlobalIndexReader>(reader, transform);
+}
+
+}  // namespace paimon

src/paimon/common/global_index/rangebitmap/range_bitmap_global_index.h

@@ -0,0 +1,65 @@
+/*
+ * Copyright 2026-present Alibaba Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "paimon/common/file_index/rangebitmap/range_bitmap_file_index.h"
+#include "paimon/common/global_index/wrap/file_index_reader_wrapper.h"
+#include "paimon/global_index/global_indexer.h"
+
+namespace paimon {
+class RangeBitmapGlobalIndex : public GlobalIndexer {
+ public:
+    explicit RangeBitmapGlobalIndex(const std::shared_ptr<RangeBitmapFileIndex>& index)
+        : index_(index) {}
+
+    Result<std::shared_ptr<GlobalIndexWriter>> CreateWriter(
+        const std::string& field_name, ::ArrowSchema* arrow_schema,
+        const std::shared_ptr<GlobalIndexFileWriter>& file_writer,
+        const std::shared_ptr<MemoryPool>& pool) const override;
+
+    Result<std::shared_ptr<GlobalIndexReader>> CreateReader(
+        ::ArrowSchema* arrow_schema, const std::shared_ptr<GlobalIndexFileReader>& file_reader,
+        const std::vector<GlobalIndexIOMeta>& files,
+        const std::shared_ptr<MemoryPool>& pool) const override;
+
+ private:
+    std::shared_ptr<RangeBitmapFileIndex> index_;
+};
+
+class RangeBitmapGlobalIndexReader : public FileIndexReaderWrapper {
+ public:
+    RangeBitmapGlobalIndexReader(const std::shared_ptr<FileIndexReader>& reader,
+                                 const std::function<Result<std::shared_ptr<GlobalIndexResult>>(
+                                     const std::shared_ptr<FileIndexResult>&)>& transform)
+        : FileIndexReaderWrapper(reader, transform) {}
+
+    static inline const char kIdentifier[] = "range-bitmap";
+
+    bool IsThreadSafe() const override {
+        return false;
+    }
+
+    std::string GetIndexType() const override {
+        return kIdentifier;
+    }
+};
+
+}  // namespace paimon

src/paimon/common/global_index/rangebitmap/range_bitmap_global_index_factory.cpp

@@ -0,0 +1,36 @@
+/*
+ * Copyright 2026-present Alibaba Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "paimon/common/global_index/rangebitmap/range_bitmap_global_index_factory.h"
+
+#include <utility>
+
+#include "paimon/common/file_index/rangebitmap/range_bitmap_file_index.h"
+#include "paimon/common/global_index/rangebitmap/range_bitmap_global_index.h"
+
+namespace paimon {
+
+const char RangeBitmapGlobalIndexFactory::IDENTIFIER[] = "range-bitmap-global";
+
+Result<std::unique_ptr<GlobalIndexer>> RangeBitmapGlobalIndexFactory::Create(
+    const std::map<std::string, std::string>& options) const {
+    auto range_bitmap_file_index = std::make_shared<RangeBitmapFileIndex>(options);
+    return std::make_unique<RangeBitmapGlobalIndex>(range_bitmap_file_index);
+}
+
+REGISTER_PAIMON_FACTORY(RangeBitmapGlobalIndexFactory);
+
+}  // namespace paimon

src/paimon/common/global_index/rangebitmap/range_bitmap_global_index_factory.h

@@ -0,0 +1,39 @@
+/*
+ * Copyright 2026-present Alibaba Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <map>
+#include <memory>
+#include <string>
+
+#include "paimon/global_index/global_indexer_factory.h"
+
+namespace paimon {
+/// Factory for creating range-bitmap global indexers.
+class RangeBitmapGlobalIndexFactory : public GlobalIndexerFactory {
+ public:
+    static const char IDENTIFIER[];
+
+    const char* Identifier() const override {
+        return IDENTIFIER;
+    }
+
+    Result<std::unique_ptr<GlobalIndexer>> Create(
+        const std::map<std::string, std::string>& options) const override;
+};
+
+}  // namespace paimon