Skip to content

Commit c2d6ada

Browse files
SteNicholasSteNicholas
authored
feat: support totalBuckets of PartitionStatistics (#107)
1 parent b3da562 commit c2d6ada

8 files changed

Lines changed: 65 additions & 33 deletions

‎src/paimon/core/catalog/commit_table_request_test.cpp‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -42,10 +42,11 @@ TEST(CommitTableRequestTest, TestSimple) {
4242
/*statistics=*/std::nullopt, /*properties=*/std::nullopt, /*next_row_id=*/0);
4343
std::vector<PartitionStatistics> partition_statistics = {
4444
PartitionStatistics(/*spec=*/{{"f1", "20"}}, /*record_count=*/1, /*file_size_in_bytes=*/541,
45-
/*file_count=*/1, /*last_file_creation_time=*/1724090888743),
45+
/*file_count=*/1, /*last_file_creation_time=*/1724090888743,
46+
/*total_buckets=*/-1),
4647
PartitionStatistics(/*spec=*/{{"f1", "10"}}, /*record_count=*/4,
4748
/*file_size_in_bytes=*/1118, /*file_count=*/2,
48-
/*last_file_creation_time=*/1724090888727)};
49+
/*last_file_creation_time=*/1724090888727, /*total_buckets=*/-1)};
4950
std::string expected_request_str = R"({
5051
"snapshot": {
5152
"version": 3,
@@ -74,7 +75,8 @@ TEST(CommitTableRequestTest, TestSimple) {
7475
"recordCount": 1,
7576
"fileSizeInBytes": 541,
7677
"fileCount": 1,
77-
"lastFileCreationTime": 1724090888743
78+
"lastFileCreationTime": 1724090888743,
79+
"totalBuckets": -1
7880
},
7981
{
8082
"spec": {
@@ -83,7 +85,8 @@ TEST(CommitTableRequestTest, TestSimple) {
8385
"recordCount": 4,
8486
"fileSizeInBytes": 1118,
8587
"fileCount": 2,
86-
"lastFileCreationTime": 1724090888727
88+
"lastFileCreationTime": 1724090888727,
89+
"totalBuckets": -1
8790
}
8891
]
8992
})";

‎src/paimon/core/manifest/partition_entry.cpp‎

Lines changed: 14 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -29,28 +29,31 @@
2929
namespace paimon {
3030
PartitionEntry::PartitionEntry(const BinaryRow& partition, int64_t record_count,
3131
int64_t file_size_in_bytes, int64_t file_count,
32-
int64_t last_file_creation_time)
32+
int64_t last_file_creation_time, int32_t total_buckets)
3333
: partition_(partition),
3434
record_count_(record_count),
3535
file_size_in_bytes_(file_size_in_bytes),
3636
file_count_(file_count),
37-
last_file_creation_time_(last_file_creation_time) {}
37+
last_file_creation_time_(last_file_creation_time),
38+
total_buckets_(total_buckets) {}
3839

3940
PartitionEntry PartitionEntry::Merge(const PartitionEntry& entry) const {
40-
return PartitionEntry(partition_, record_count_ + entry.record_count_,
41-
file_size_in_bytes_ + entry.file_size_in_bytes_,
42-
file_count_ + entry.file_count_,
43-
std::max(last_file_creation_time_, entry.last_file_creation_time_));
41+
return PartitionEntry(
42+
partition_, record_count_ + entry.record_count_,
43+
file_size_in_bytes_ + entry.file_size_in_bytes_, file_count_ + entry.file_count_,
44+
std::max(last_file_creation_time_, entry.last_file_creation_time_), entry.total_buckets_);
4445
}
4546

4647
Result<PartitionEntry> PartitionEntry::FromDataFile(const BinaryRow& partition,
4748
const FileKind& kind,
48-
const std::shared_ptr<DataFileMeta>& file) {
49+
const std::shared_ptr<DataFileMeta>& file,
50+
const int32_t total_buckets) {
4951
int64_t record_count = kind == FileKind::Delete() ? -file->row_count : file->row_count;
5052
int64_t file_size_in_bytes = kind == FileKind::Delete() ? -file->file_size : file->file_size;
5153
int64_t file_count = kind == FileKind::Delete() ? -1 : 1;
5254
PAIMON_ASSIGN_OR_RAISE(int64_t utc_millis, file->CreationTimeEpochMillis());
53-
return PartitionEntry(partition, record_count, file_size_in_bytes, file_count, utc_millis);
55+
return PartitionEntry(partition, record_count, file_size_in_bytes, file_count, utc_millis,
56+
total_buckets);
5457
}
5558

5659
Status PartitionEntry::Merge(const std::vector<ManifestEntry>& from,
@@ -75,7 +78,7 @@ Result<PartitionStatistics> PartitionEntry::ToPartitionStatistics(
7578
PAIMON_ASSIGN_OR_RAISE(part_values, partition_computer->GeneratePartitionVector(partition_));
7679
std::map<std::string, std::string> part_values_map(part_values.begin(), part_values.end());
7780
return PartitionStatistics(part_values_map, record_count_, file_size_in_bytes_, file_count_,
78-
last_file_creation_time_);
81+
last_file_creation_time_, total_buckets_);
7982
}
8083

8184
bool PartitionEntry::operator==(const PartitionEntry& other) const {
@@ -84,7 +87,8 @@ bool PartitionEntry::operator==(const PartitionEntry& other) const {
8487
}
8588
return partition_ == other.partition_ && record_count_ == other.record_count_ &&
8689
file_size_in_bytes_ == other.file_size_in_bytes_ && file_count_ == other.file_count_ &&
87-
last_file_creation_time_ == other.last_file_creation_time_;
90+
last_file_creation_time_ == other.last_file_creation_time_ &&
91+
total_buckets_ == other.total_buckets_;
8892
}
8993

9094
} // namespace paimon

‎src/paimon/core/manifest/partition_entry.h‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ class BinaryRowPartitionComputer;
3939
class PartitionEntry {
4040
public:
4141
PartitionEntry(const BinaryRow& partition, int64_t record_count, int64_t file_size_in_bytes,
42-
int64_t file_count, int64_t last_file_creation_time);
42+
int64_t file_count, int64_t last_file_creation_time, int32_t total_buckets);
4343

4444
const BinaryRow& Partition() const {
4545
return partition_;
@@ -58,14 +58,18 @@ class PartitionEntry {
5858
int64_t LastFileCreationTime() const {
5959
return last_file_creation_time_;
6060
}
61+
int32_t TotalBuckets() const {
62+
return total_buckets_;
63+
}
6164
PartitionEntry Merge(const PartitionEntry& entry) const;
6265

6366
static Result<PartitionEntry> FromManifestEntry(const ManifestEntry& entry) {
64-
return FromDataFile(entry.Partition(), entry.Kind(), entry.File());
67+
return FromDataFile(entry.Partition(), entry.Kind(), entry.File(), entry.TotalBuckets());
6568
}
6669

6770
static Result<PartitionEntry> FromDataFile(const BinaryRow& partition, const FileKind& kind,
68-
const std::shared_ptr<DataFileMeta>& file);
71+
const std::shared_ptr<DataFileMeta>& file,
72+
int32_t total_buckets);
6973

7074
Result<PartitionStatistics> ToPartitionStatistics(
7175
const BinaryRowPartitionComputer* partition_computer) const;
@@ -81,5 +85,6 @@ class PartitionEntry {
8185
int64_t file_size_in_bytes_;
8286
int64_t file_count_;
8387
int64_t last_file_creation_time_;
88+
int32_t total_buckets_;
8489
};
8590
} // namespace paimon

‎src/paimon/core/manifest/partition_entry_test.cpp‎

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -66,8 +66,10 @@ TEST_F(PartitionEntryTest, TestSimple) {
6666
/*total_buckets=*/2, GetDataFileMeta(Timestamp(1500, 123)))));
6767

6868
auto res = partition_entry1.Merge(partition_entry2);
69-
auto expected_partition_entry = PartitionEntry(BinaryRow::EmptyRow(), /*record_count=*/10,
70-
/*file_size_in_bytes=*/20, 2, -28798500);
69+
auto expected_partition_entry =
70+
PartitionEntry(BinaryRow::EmptyRow(), /*record_count=*/10,
71+
/*file_size_in_bytes=*/20, /*file_count=*/2,
72+
/*last_file_creation_time=*/-28798500, /*total_buckets=*/2);
7173
ASSERT_EQ(res, expected_partition_entry);
7274
}
7375
{
@@ -82,8 +84,10 @@ TEST_F(PartitionEntryTest, TestSimple) {
8284
/*total_buckets=*/2, GetDataFileMeta(Timestamp(1500, 123)))));
8385

8486
auto res = partition_entry1.Merge(partition_entry2);
85-
auto expected_partition_entry = PartitionEntry(BinaryRow::EmptyRow(), /*record_count=*/0,
86-
/*file_size_in_bytes=*/0, 0, -28798500);
87+
auto expected_partition_entry =
88+
PartitionEntry(BinaryRow::EmptyRow(), /*record_count=*/0,
89+
/*file_size_in_bytes=*/0, /*file_count=*/0,
90+
/*last_file_creation_time=*/-28798500, /*total_buckets=*/2);
8791
ASSERT_EQ(res, expected_partition_entry);
8892
}
8993
}
@@ -108,10 +112,10 @@ TEST_F(PartitionEntryTest, TestMerge) {
108112
std::unordered_map<BinaryRow, PartitionEntry> expected_partition_entry;
109113
expected_partition_entry.emplace(
110114
std::piecewise_construct, std::forward_as_tuple(partition0),
111-
std::forward_as_tuple(PartitionEntry(partition0, 10, 20, 2, -28798500)));
115+
std::forward_as_tuple(PartitionEntry(partition0, 10, 20, 2, -28798500, 2)));
112116
expected_partition_entry.emplace(
113117
std::piecewise_construct, std::forward_as_tuple(partition1),
114-
std::forward_as_tuple(PartitionEntry(partition1, 5, 10, 1, -28800000)));
118+
std::forward_as_tuple(PartitionEntry(partition1, 5, 10, 1, -28800000, 2)));
115119
ASSERT_EQ(to, expected_partition_entry);
116120
}
117121

‎src/paimon/core/operation/append_only_file_store_scan_test.cpp‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -111,9 +111,11 @@ TEST(AppendOnlyFileStoreScanTest, TestReadPartitionEntries) {
111111

112112
std::vector<PartitionEntry> expected_partition_entries = {
113113
PartitionEntry(GenerateRow(10), /*record_count=*/9, /*file_size_in_bytes=*/1183,
114-
/*file_count=*/2, /*last_file_creation_time=*/1721643834472l - 28800000l),
114+
/*file_count=*/2, /*last_file_creation_time=*/1721643834472l - 28800000l,
115+
/*total_buckets=*/2),
115116
PartitionEntry(GenerateRow(20), /*record_count=*/2, /*file_size_in_bytes=*/1047,
116-
/*file_count=*/2, /*last_file_creation_time=*/1721643267404l - 28800000l)};
117+
/*file_count=*/2, /*last_file_creation_time=*/1721643267404l - 28800000l,
118+
/*total_buckets=*/2)};
117119
auto ComparePartitionEntryByPartition = [](const PartitionEntry& lhs,
118120
const PartitionEntry& rhs) -> bool {
119121
return lhs.Partition().GetInt(0) < rhs.Partition().GetInt(0);

‎src/paimon/core/operation/file_store_commit_impl_test.cpp‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -344,10 +344,12 @@ TEST_F(FileStoreCommitImplTest, TestRESTCatalogCommit) {
344344
std::vector<PartitionStatistics> expected_partition_statistics = {
345345
PartitionStatistics(/*spec=*/{{"f1", "20"}}, /*record_count=*/1, /*file_size_in_bytes=*/541,
346346
/*file_count=*/1,
347-
/*last_file_creation_time=*/1724090888743l - 28800000l),
347+
/*last_file_creation_time=*/1724090888743l - 28800000l,
348+
/*total_buckets=*/-1),
348349
PartitionStatistics(/*spec=*/{{"f1", "10"}}, /*record_count=*/4,
349350
/*file_size_in_bytes=*/1118, /*file_count=*/2,
350-
/*last_file_creation_time=*/1724090888727l - 28800000l)};
351+
/*last_file_creation_time=*/1724090888727l - 28800000l,
352+
/*total_buckets=*/-1)};
351353
CommitTableRequest expected_commit_table_request(expected_snapshot,
352354
expected_partition_statistics);
353355
ASSERT_TRUE(commit_table_request.TEST_Equal(expected_commit_table_request));

‎src/paimon/core/partition/partition_statistics.h‎

Lines changed: 15 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -34,12 +34,13 @@ class PartitionStatistics : public Jsonizable<PartitionStatistics> {
3434
public:
3535
using SpecType = std::map<std::string, std::string>;
3636
PartitionStatistics(const SpecType& spec, int64_t record_count, int64_t file_size_in_bytes,
37-
int64_t file_count, int64_t last_file_creation_time)
37+
int64_t file_count, int64_t last_file_creation_time, int32_t total_buckets)
3838
: spec_(spec),
3939
record_count_(record_count),
4040
file_size_in_bytes_(file_size_in_bytes),
4141
file_count_(file_count),
42-
last_file_creation_time_(last_file_creation_time) {}
42+
last_file_creation_time_(last_file_creation_time),
43+
total_buckets_(total_buckets) {}
4344

4445
const SpecType& Spec() const {
4546
return spec_;
@@ -56,6 +57,9 @@ class PartitionStatistics : public Jsonizable<PartitionStatistics> {
5657
int64_t LastFileCreationTime() const {
5758
return last_file_creation_time_;
5859
}
60+
int32_t TotalBuckets() const {
61+
return total_buckets_;
62+
}
5963

6064
rapidjson::Value ToJson(rapidjson::Document::AllocatorType* allocator) const
6165
noexcept(false) override {
@@ -72,6 +76,8 @@ class PartitionStatistics : public Jsonizable<PartitionStatistics> {
7276
obj.AddMember(rapidjson::StringRef(FIELD_LAST_FILE_CREATION_TIME),
7377
RapidJsonUtil::SerializeValue(last_file_creation_time_, allocator).Move(),
7478
*allocator);
79+
obj.AddMember(rapidjson::StringRef(FIELD_TOTAL_BUCKETS),
80+
RapidJsonUtil::SerializeValue(total_buckets_, allocator).Move(), *allocator);
7581
return obj;
7682
}
7783

@@ -83,12 +89,14 @@ class PartitionStatistics : public Jsonizable<PartitionStatistics> {
8389
file_count_ = RapidJsonUtil::DeserializeKeyValue<int64_t>(obj, FIELD_FILE_COUNT);
8490
last_file_creation_time_ =
8591
RapidJsonUtil::DeserializeKeyValue<int64_t>(obj, FIELD_LAST_FILE_CREATION_TIME);
92+
total_buckets_ = RapidJsonUtil::DeserializeKeyValue<int32_t>(obj, FIELD_TOTAL_BUCKETS);
8693
}
8794

8895
bool operator==(const PartitionStatistics& rhs) const {
8996
return record_count_ == rhs.record_count_ &&
9097
file_size_in_bytes_ == rhs.file_size_in_bytes_ && file_count_ == rhs.file_count_ &&
91-
last_file_creation_time_ == rhs.last_file_creation_time_ && spec_ == rhs.spec_;
98+
last_file_creation_time_ == rhs.last_file_creation_time_ &&
99+
total_buckets_ == rhs.total_buckets_ && spec_ == rhs.spec_;
92100
}
93101

94102
std::string ToString() const {
@@ -102,7 +110,8 @@ class PartitionStatistics : public Jsonizable<PartitionStatistics> {
102110
}
103111
oss << "}, recordCount=" << record_count_ << ", fileSizeInBytes=" << file_size_in_bytes_
104112
<< ", fileCount=" << file_count_
105-
<< ", lastFileCreationTime=" << last_file_creation_time_ << "}";
113+
<< ", lastFileCreationTime=" << last_file_creation_time_
114+
<< ", totalBuckets=" << total_buckets_ << "}";
106115
return oss.str();
107116
}
108117

@@ -115,12 +124,14 @@ class PartitionStatistics : public Jsonizable<PartitionStatistics> {
115124
static constexpr const char* FIELD_FILE_SIZE_IN_BYTES = "fileSizeInBytes";
116125
static constexpr const char* FIELD_FILE_COUNT = "fileCount";
117126
static constexpr const char* FIELD_LAST_FILE_CREATION_TIME = "lastFileCreationTime";
127+
static constexpr const char* FIELD_TOTAL_BUCKETS = "totalBuckets";
118128

119129
SpecType spec_;
120130
int64_t record_count_ = 0;
121131
int64_t file_size_in_bytes_ = 0;
122132
int64_t file_count_ = 0;
123133
int64_t last_file_creation_time_ = 0;
134+
int32_t total_buckets_ = 0;
124135
};
125136

126137
} // namespace paimon

‎src/paimon/core/partition/partition_statistics_test.cpp‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -43,15 +43,16 @@ TEST_F(PartitionStatisticsTest, TestJsonizable) {
4343
"recordCount": 4,
4444
"fileSizeInBytes": 1118,
4545
"fileCount": 2,
46-
"lastFileCreationTime": 1724090888727
46+
"lastFileCreationTime": 1724090888727,
47+
"totalBuckets": 1
4748
})";
4849

4950
ASSERT_OK_AND_ASSIGN(PartitionStatistics partition_statistics,
5051
PartitionStatistics::FromJsonString(json_str));
5152

5253
PartitionStatistics expected_partition_statistics(
5354
/*spec=*/{{"f1", "10"}, {"f2", "20"}}, /*record_count=*/4, /*file_size_in_bytes=*/1118,
54-
/*file_count=*/2, /*last_file_creation_time=*/1724090888727);
55+
/*file_count=*/2, /*last_file_creation_time=*/1724090888727, /*total_buckets=*/1);
5556
ASSERT_EQ(expected_partition_statistics, partition_statistics);
5657

5758
ASSERT_OK_AND_ASSIGN(std::string new_json_str, partition_statistics.ToJsonString());

0 commit comments

Comments
 (0)