feat: support external path for global index (alibaba#52)

Author: lszskye

include/paimon/defs.h

@@ -284,6 +284,9 @@ struct PAIMON_EXPORT Options {
     static const char BLOB_AS_DESCRIPTOR[];
     /// "global-index.enabled" - Whether to enable global index for scan. Default value is "true".
     static const char GLOBAL_INDEX_ENABLED[];
+    /// "global-index.external-path" - Global index root directory, if not set, the global index
+    /// files will be stored under the index directory.
+    static const char GLOBAL_INDEX_EXTERNAL_PATH[];
 };
 
 static constexpr int64_t BATCH_WRITE_COMMIT_IDENTIFIER = std::numeric_limits<int64_t>::max();

include/paimon/global_index/global_index_io_meta.h

@@ -25,14 +25,14 @@
 namespace paimon {
 /// Metadata describing a single file entry in a global index.
 struct PAIMON_EXPORT GlobalIndexIOMeta {
-    GlobalIndexIOMeta(const std::string& _file_name, int64_t _file_size, int64_t _range_end,
+    GlobalIndexIOMeta(const std::string& _file_path, int64_t _file_size, int64_t _range_end,
                       const std::shared_ptr<Bytes>& _metadata)
-        : file_name(_file_name),
+        : file_path(_file_path),
           file_size(_file_size),
           range_end(_range_end),
           metadata(_metadata) {}
 
-    std::string file_name;
+    std::string file_path;
     int64_t file_size;
     /// The inclusive range end covered by this file (i.e., the last local row id).
     int64_t range_end;

include/paimon/global_index/io/global_index_file_writer.h

@@ -36,6 +36,9 @@ class PAIMON_EXPORT GlobalIndexFileWriter {
 
     /// Get the file size of input file name.
     virtual Result<int64_t> GetFileSize(const std::string& file_name) const = 0;
+
+    /// Get the index file path of input file name.
+    virtual std::string ToPath(const std::string& file_name) const = 0;
 };
 
 }  // namespace paimon

src/paimon/common/defs.cpp

@@ -80,4 +80,5 @@ const char Options::DATA_EVOLUTION_ENABLED[] = "data-evolution.enabled";
 const char Options::PARTITION_GENERATE_LEGACY_NAME[] = "partition.legacy-name";
 const char Options::BLOB_AS_DESCRIPTOR[] = "blob-as-descriptor";
 const char Options::GLOBAL_INDEX_ENABLED[] = "global-index.enabled";
+const char Options::GLOBAL_INDEX_EXTERNAL_PATH[] = "global-index.external-path";
 }  // namespace paimon

src/paimon/common/global_index/bitmap/bitmap_global_index.cpp

@@ -41,7 +41,7 @@ Result<std::shared_ptr<GlobalIndexReader>> BitmapGlobalIndex::CreateReader(
     }
     const auto& meta = files[0];
     PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<InputStream> in,
-                           file_reader->GetInputStream(meta.file_name));
+                           file_reader->GetInputStream(meta.file_path));
     PAIMON_ASSIGN_OR_RAISE(
         std::shared_ptr<FileIndexReader> reader,
         index_->CreateReader(arrow_schema, /*start=*/0, meta.file_size, in, pool));

src/paimon/common/global_index/bitmap/bitmap_global_index_test.cpp

@@ -100,8 +100,9 @@ class BitmapGlobalIndexTest : public ::testing::Test {
         PAIMON_ASSIGN_OR_RAISE(auto result_metas, global_writer->Finish());
         // check meta
         EXPECT_EQ(result_metas.size(), 1);
-        EXPECT_TRUE(StringUtils::StartsWith(result_metas[0].file_name, "bitmap-global-index-"));
-        EXPECT_TRUE(StringUtils::EndsWith(result_metas[0].file_name, ".index"));
+        auto file_name = PathUtil::GetName(result_metas[0].file_path);
+        EXPECT_TRUE(StringUtils::StartsWith(file_name, "bitmap-global-index-"));
+        EXPECT_TRUE(StringUtils::EndsWith(file_name, ".index"));
         EXPECT_EQ(result_metas[0].range_end, expected_range.to);
         EXPECT_FALSE(result_metas[0].metadata);
         return result_metas[0];

src/paimon/common/global_index/wrap/file_index_writer_wrapper.h

@@ -72,7 +72,8 @@ class FileIndexWriterWrapper : public GlobalIndexWriter {
         }
         PAIMON_RETURN_NOT_OK(out->Flush());
         PAIMON_RETURN_NOT_OK(out->Close());
-        GlobalIndexIOMeta meta(file_name, /*file_size=*/bytes->size(), /*range_end=*/count_ - 1,
+        GlobalIndexIOMeta meta(file_manager_->ToPath(file_name), /*file_size=*/bytes->size(),
+                               /*range_end=*/count_ - 1,
                                /*metadata=*/nullptr);
         return std::vector<GlobalIndexIOMeta>({meta});
     }

src/paimon/core/core_options.cpp

@@ -303,6 +303,7 @@ struct CoreOptions::Impl {
     bool data_evolution_enabled = false;
     bool legacy_partition_name_enabled = true;
     bool global_index_enabled = true;
+    std::optional<std::string> global_index_external_path;
 };
 
 // Parse configurations from a map and return a populated CoreOptions object
@@ -470,6 +471,15 @@ Result<CoreOptions> CoreOptions::FromMap(
     // Parse global-index.enabled
     PAIMON_RETURN_NOT_OK(
         parser.Parse<bool>(Options::GLOBAL_INDEX_ENABLED, &impl->global_index_enabled));
+
+    // Parse global_index.external-path
+    std::string global_index_external_path;
+    PAIMON_RETURN_NOT_OK(
+        parser.ParseString(Options::GLOBAL_INDEX_EXTERNAL_PATH, &global_index_external_path));
+    if (!global_index_external_path.empty()) {
+        impl->global_index_external_path = global_index_external_path;
+    }
+
     return options;
 }
 
@@ -746,4 +756,23 @@ bool CoreOptions::LegacyPartitionNameEnabled() const {
 bool CoreOptions::GlobalIndexEnabled() const {
     return impl_->global_index_enabled;
 }
+
+std::optional<std::string> CoreOptions::GetGlobalIndexExternalPath() const {
+    return impl_->global_index_external_path;
+}
+
+Result<std::optional<std::string>> CoreOptions::CreateGlobalIndexExternalPath() const {
+    std::optional<std::string> global_index_external_path = GetGlobalIndexExternalPath();
+    if (global_index_external_path == std::nullopt) {
+        return global_index_external_path;
+    }
+    std::string tmp_path = global_index_external_path.value();
+    StringUtils::Trim(&tmp_path);
+    PAIMON_ASSIGN_OR_RAISE(Path path, PathUtil::ToPath(tmp_path));
+    if (path.scheme.empty()) {
+        return Status::Invalid(fmt::format("scheme is null, path is {}", tmp_path));
+    }
+    return std::optional<std::string>(path.ToString());
+}
+
 }  // namespace paimon

src/paimon/core/core_options.h

@@ -102,7 +102,6 @@ class PAIMON_EXPORT CoreOptions {
     std::optional<std::string> GetScanFallbackBranch() const;
     std::string GetBranch() const;
 
-    std::optional<std::string> GetDataFileExternalPaths() const;
     ExternalPathStrategy GetExternalPathStrategy() const;
     Result<std::vector<std::string>> CreateExternalPaths() const;
     bool EnableAdaptivePrefetchStrategy() const;
@@ -117,8 +116,14 @@ class PAIMON_EXPORT CoreOptions {
     bool LegacyPartitionNameEnabled() const;
 
     bool GlobalIndexEnabled() const;
+    Result<std::optional<std::string>> CreateGlobalIndexExternalPath() const;
+
     const std::map<std::string, std::string>& ToMap() const;
 
+ private:
+    std::optional<std::string> GetDataFileExternalPaths() const;
+    std::optional<std::string> GetGlobalIndexExternalPath() const;
+
  private:
     struct Impl;
 

src/paimon/core/core_options_test.cpp

@@ -87,6 +87,7 @@ TEST(CoreOptionsTest, TestDefaultValue) {
     ASSERT_FALSE(core_options.DataEvolutionEnabled());
     ASSERT_TRUE(core_options.LegacyPartitionNameEnabled());
     ASSERT_TRUE(core_options.GlobalIndexEnabled());
+    ASSERT_FALSE(core_options.GetGlobalIndexExternalPath());
 }
 
 TEST(CoreOptionsTest, TestFromMap) {
@@ -144,6 +145,7 @@ TEST(CoreOptionsTest, TestFromMap) {
         {Options::DATA_EVOLUTION_ENABLED, "true"},
         {Options::PARTITION_GENERATE_LEGACY_NAME, "false"},
         {Options::GLOBAL_INDEX_ENABLED, "false"},
+        {Options::GLOBAL_INDEX_EXTERNAL_PATH, "FILE:///tmp/global_index/"},
     };
 
     ASSERT_OK_AND_ASSIGN(CoreOptions core_options, CoreOptions::FromMap(options));
@@ -212,6 +214,8 @@ TEST(CoreOptionsTest, TestFromMap) {
     ASSERT_TRUE(core_options.DataEvolutionEnabled());
     ASSERT_FALSE(core_options.LegacyPartitionNameEnabled());
     ASSERT_FALSE(core_options.GlobalIndexEnabled());
+    ASSERT_TRUE(core_options.GetGlobalIndexExternalPath());
+    ASSERT_EQ(core_options.GetGlobalIndexExternalPath().value(), "FILE:///tmp/global_index/");
 }
 
 TEST(CoreOptionsTest, TestInvalidCase) {
@@ -273,6 +277,25 @@ TEST(CoreOptionsTest, TestInvalidCreateExternalPath) {
     }
 }
 
+TEST(CoreOptionsTest, TestCreateGlobalIndexExternalPath) {
+    std::map<std::string, std::string> options = {
+        {Options::GLOBAL_INDEX_EXTERNAL_PATH, " FILE:///tmp/index1"},
+    };
+    ASSERT_OK_AND_ASSIGN(CoreOptions core_options, CoreOptions::FromMap(options));
+    ASSERT_OK_AND_ASSIGN(std::optional<std::string> external_path,
+                         core_options.CreateGlobalIndexExternalPath());
+    ASSERT_EQ("FILE:/tmp/index1", external_path.value());
+}
+
+TEST(CoreOptionsTest, TestInvalidCreateGlobalIndexExternalPath) {
+    std::map<std::string, std::string> options = {
+        {Options::GLOBAL_INDEX_EXTERNAL_PATH, "/tmp/index1"},
+    };
+    ASSERT_OK_AND_ASSIGN(CoreOptions core_options, CoreOptions::FromMap(options));
+    ASSERT_NOK_WITH_MSG(core_options.CreateGlobalIndexExternalPath(),
+                        "scheme is null, path is /tmp/index1");
+}
+
 TEST(CoreOptionsTest, TestFileSystem) {
     {
         auto mock_fs = std::make_shared<MockFileSystem>();