Merge branch 'main' into feature/spill-to-disk

Author: lucasfang

include/paimon/catalog/identifier.h

@@ -16,8 +16,10 @@
 
 #pragma once
 
+#include <optional>
 #include <string>
 
+#include "paimon/result.h"
 #include "paimon/type_fwd.h"
 #include "paimon/visibility.h"
 
@@ -27,18 +29,32 @@ namespace paimon {
 class PAIMON_EXPORT Identifier {
  public:
     static const char kUnknownDatabase[];
+    static const char kSystemTableSplitter[];
+    static const char kSystemBranchPrefix[];
+    static const char kDefaultMainBranch[];
 
     explicit Identifier(const std::string& table);
     Identifier(const std::string& database, const std::string& table);
 
     bool operator==(const Identifier& other);
     const std::string& GetDatabaseName() const;
     const std::string& GetTableName() const;
+    Result<std::string> GetDataTableName() const;
+    Result<std::optional<std::string>> GetBranchName() const;
+    Result<std::string> GetBranchNameOrDefault() const;
+    Result<std::optional<std::string>> GetSystemTableName() const;
+    Result<bool> IsSystemTable() const;
     std::string ToString() const;
 
  private:
+    Status SplitTableName() const;
+
     const std::string database_;
     const std::string table_;
+    mutable bool parsed_ = false;
+    mutable std::string data_table_;
+    mutable std::optional<std::string> branch_;
+    mutable std::optional<std::string> system_table_;
 };
 
 }  // namespace paimon

include/paimon/schema/schema.h

@@ -57,6 +57,16 @@ class PAIMON_EXPORT Schema {
     /// failure.
     virtual Result<FieldType> GetFieldType(const std::string& field_name) const = 0;
 
+    /// Get an optional comment describing the schema object.
+    /// @return The comment if set, or std::nullopt otherwise.
+    virtual std::optional<std::string> Comment() const = 0;
+};
+
+/// Schema contract for data tables.
+class PAIMON_EXPORT DataSchema : public Schema {
+ public:
+    ~DataSchema() override = default;
+
     /// Get the unique identifier of this table schema.
     /// @return The schema id
     virtual int64_t Id() const = 0;
@@ -86,10 +96,12 @@ class PAIMON_EXPORT Schema {
     /// Get the table-level options associated with this schema.
     /// @return A reference to the map of option key-value pairs (e.g., file format, filesystem).
     virtual const std::map<std::string, std::string>& Options() const = 0;
+};
 
-    /// Get an optional comment describing the table.
-    /// @return The table comment if set, or std::nullopt otherwise.
-    virtual std::optional<std::string> Comment() const = 0;
+/// Schema contract for system tables.
+class PAIMON_EXPORT SystemSchema : public Schema {
+ public:
+    ~SystemSchema() override = default;
 };
 
 }  // namespace paimon

include/paimon/table/source/table_read.h

@@ -66,6 +66,10 @@ class PAIMON_EXPORT TableRead {
  protected:
     explicit TableRead(const std::shared_ptr<MemoryPool>& memory_pool);
 
+    std::shared_ptr<MemoryPool> GetMemoryPool() const {
+        return pool_;
+    }
+
  private:
     std::shared_ptr<MemoryPool> pool_;
 };

include/paimon/utils/roaring_bitmap64.h

@@ -74,6 +74,19 @@ class PAIMON_EXPORT RoaringBitmap64 {
     /// @param x value added to bitmap
     void Add(int64_t x);
 
+    /// Bulk-insert `n` values into the bitmap.
+    ///
+    /// Compared to repeatedly calling `Add`, this implementation:
+    ///   1. Buckets the input values by their high-32 bits in a single pass.
+    ///   2. Feeds each bucket to the inner 32-bit Roaring's true-batch
+    ///      `addMany(uint32_t*)` path, which performs container-level bulk
+    ///      insertion.
+    ///
+    /// This avoids the per-value `std::map` lookup of the 64-bit wrapper and
+    /// the per-value insertion overhead inside the 32-bit array container.
+    /// Values may be unsorted; ordering is handled internally.
+    void AddMany(size_t n, const int64_t* values);
+
     /// @param x value added to bitmap
     /// @return false if contain x; true if not contain x
     bool CheckedAdd(int64_t x);

src/paimon/CMakeLists.txt

@@ -314,6 +314,11 @@ set(PAIMON_CORE_SRCS
     core/table/source/table_read.cpp
     core/table/source/table_scan.cpp
     core/table/source/data_evolution_batch_scan.cpp
+    core/table/system/options_system_table.cpp
+    core/table/system/system_table.cpp
+    core/table/system/system_table_read.cpp
+    core/table/system/system_table_scan.cpp
+    core/table/system/system_table_schema.cpp
     core/tag/tag.cpp
     core/utils/field_mapping.cpp
     core/utils/file_store_path_factory.cpp

src/paimon/common/global_index/btree/btree_defs.h

@@ -41,6 +41,16 @@ struct BtreeDefs {
     static inline const char kBtreeIndexHighPriorityPoolRatio[] =
         "btree-index.high-priority-pool-ratio";
 
+    /// "btree-index.read-buffer-size" - Optional. Specifies the read buffer size for the B-tree
+    /// index. This setting can be tuned based on query patterns:
+    ///   - For range queries (e.g., `VisitLessThan`, `VisitGreaterOrEqual`), increasing the buffer
+    ///   size (e.g., to 1MB) may improve I/O bandwidth and sequential read performance.
+    ///   - For point queries (e.g., `VisitEqual`), buffering can introduce negative effects due to
+    ///   read amplification; it is recommended to leave this option unset.
+    ///
+    /// If specified, read block with `BufferedInputStream`.
+    static inline const char kBtreeIndexReadBufferSize[] = "btree-index.read-buffer-size";
+
     static inline const char kDefaultBtreeIndexBlockSize[] = "64KB";
     static inline const char kDefaultBtreeIndexCompression[] = "none";
     static inline const char kDefaultBtreeIndexCacheSize[] = "128MB";