Skip to content

Commit 4bcf482

Browse files
authored
Merge branch 'main' into fts-support-jieba
2 parents 1495b84 + 00cb53b commit 4bcf482

14 files changed

Lines changed: 150 additions & 51 deletions

File tree

src/paimon/common/memory/bytes.cpp

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -59,9 +59,16 @@ Bytes& Bytes::operator=(Bytes&& other) noexcept {
5959
if (&other == this) {
6060
return *this;
6161
}
62-
this->~Bytes();
63-
std::memcpy(this, &other, sizeof(*this));
64-
new (&other) Bytes();
62+
if (data_ != nullptr) {
63+
assert(pool_);
64+
pool_->Free(data_, size_);
65+
}
66+
pool_ = other.pool_;
67+
data_ = other.data_;
68+
size_ = other.size_;
69+
other.pool_ = nullptr;
70+
other.data_ = nullptr;
71+
other.size_ = 0;
6572
return *this;
6673
}
6774

src/paimon/common/memory/bytes_test.cpp

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -82,4 +82,73 @@ TEST(BytesTest, TestCompare) {
8282
ASSERT_LT(*bytes1, *bytes2);
8383
ASSERT_FALSE(*bytes1 < *bytes1);
8484
}
85+
86+
// Test to verify that move assignment correctly handles memory and prevents double-free.
87+
// Before the fix, the old implementation used memcpy + destructor which caused:
88+
// 1. The target's original memory was freed in destructor
89+
// 2. After memcpy, both source and target pointed to same memory
90+
// 3. When source was "reset" via placement new, it became empty
91+
// 4. But if move assignment was called again on the same target, the memcpy'd
92+
// pointer would be freed again (double-free) or memory accounting would be wrong.
93+
TEST(BytesTest, TestMoveAssignmentNoDoubleFree) {
94+
auto pool = paimon::GetMemoryPool();
95+
96+
// Create three Bytes objects on stack
97+
Bytes a("aaaa", pool.get()); // 4 bytes
98+
Bytes b("bb", pool.get()); // 2 bytes
99+
Bytes c("cccccc", pool.get()); // 6 bytes
100+
ASSERT_EQ(12, pool->CurrentUsage()); // 4 + 2 + 6 = 12
101+
102+
// First move: b = std::move(a)
103+
// Should free b's original memory (2 bytes), transfer a's memory to b
104+
b = std::move(a);
105+
ASSERT_EQ(10, pool->CurrentUsage()); // 4 + 6 = 10 (b's 2 bytes freed)
106+
ASSERT_EQ("aaaa", std::string(b.data(), b.size()));
107+
// Moved-from object is expected to be empty by Bytes' contract.
108+
ASSERT_EQ(nullptr, a.data()); // NOLINT(bugprone-use-after-move, clang-analyzer-cplusplus.Move)
109+
ASSERT_EQ(0, a.size()); // NOLINT(bugprone-use-after-move, clang-analyzer-cplusplus.Move)
110+
111+
// Second move: b = std::move(c)
112+
// Should free b's current memory (4 bytes from a), transfer c's memory to b
113+
// This is where the old implementation would cause issues:
114+
// - Old code would call destructor on b, freeing the 4 bytes
115+
// - Then memcpy c into b, making b point to c's 6-byte buffer
116+
// - Memory accounting would be wrong because Free was called on wrong data
117+
b = std::move(c);
118+
ASSERT_EQ(6, pool->CurrentUsage()); // Only c's 6 bytes remain (now owned by b)
119+
ASSERT_EQ("cccccc", std::string(b.data(), b.size()));
120+
// Moved-from object is expected to be empty by Bytes' contract.
121+
ASSERT_EQ(nullptr, c.data()); // NOLINT(bugprone-use-after-move, clang-analyzer-cplusplus.Move)
122+
ASSERT_EQ(0, c.size()); // NOLINT(bugprone-use-after-move, clang-analyzer-cplusplus.Move)
123+
124+
// Self-assignment should be safe. Use an alias to avoid -Wself-move.
125+
Bytes* self = &b;
126+
b = std::move(*self);
127+
ASSERT_EQ(6, pool->CurrentUsage());
128+
ASSERT_EQ("cccccc", std::string(b.data(), b.size()));
129+
}
130+
131+
// Test move assignment with heap-allocated Bytes to verify no double-free
132+
// when combining unique_ptr semantics with move assignment
133+
TEST(BytesTest, TestMoveAssignmentHeapAllocated) {
134+
auto pool = paimon::GetMemoryPool();
135+
136+
auto bytes1 = Bytes::AllocateBytes("hello", pool.get()); // 5 bytes + sizeof(Bytes)
137+
auto bytes2 = Bytes::AllocateBytes("world!", pool.get()); // 6 bytes + sizeof(Bytes)
138+
size_t expected = 5 + 6 + 2 * sizeof(Bytes);
139+
ASSERT_EQ(expected, pool->CurrentUsage());
140+
141+
// Move the content of bytes1 into bytes2's Bytes object
142+
// This should free "world!" (6 bytes) and transfer "hello" ownership
143+
*bytes2 = std::move(*bytes1);
144+
expected = 5 + 2 * sizeof(Bytes); // "world!" freed, "hello" transferred
145+
ASSERT_EQ(expected, pool->CurrentUsage());
146+
ASSERT_EQ("hello", std::string(bytes2->data(), bytes2->size()));
147+
ASSERT_EQ(nullptr, bytes1->data());
148+
149+
// Reset bytes2, which should free "hello"
150+
bytes2.reset();
151+
expected = sizeof(Bytes); // Only bytes1's empty Bytes struct remains
152+
ASSERT_EQ(expected, pool->CurrentUsage());
153+
}
85154
} // namespace paimon::test

src/paimon/core/catalog/commit_table_request_test.cpp

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -42,10 +42,11 @@ TEST(CommitTableRequestTest, TestSimple) {
4242
/*statistics=*/std::nullopt, /*properties=*/std::nullopt, /*next_row_id=*/0);
4343
std::vector<PartitionStatistics> partition_statistics = {
4444
PartitionStatistics(/*spec=*/{{"f1", "20"}}, /*record_count=*/1, /*file_size_in_bytes=*/541,
45-
/*file_count=*/1, /*last_file_creation_time=*/1724090888743),
45+
/*file_count=*/1, /*last_file_creation_time=*/1724090888743,
46+
/*total_buckets=*/-1),
4647
PartitionStatistics(/*spec=*/{{"f1", "10"}}, /*record_count=*/4,
4748
/*file_size_in_bytes=*/1118, /*file_count=*/2,
48-
/*last_file_creation_time=*/1724090888727)};
49+
/*last_file_creation_time=*/1724090888727, /*total_buckets=*/-1)};
4950
std::string expected_request_str = R"({
5051
"snapshot": {
5152
"version": 3,
@@ -74,7 +75,8 @@ TEST(CommitTableRequestTest, TestSimple) {
7475
"recordCount": 1,
7576
"fileSizeInBytes": 541,
7677
"fileCount": 1,
77-
"lastFileCreationTime": 1724090888743
78+
"lastFileCreationTime": 1724090888743,
79+
"totalBuckets": -1
7880
},
7981
{
8082
"spec": {
@@ -83,7 +85,8 @@ TEST(CommitTableRequestTest, TestSimple) {
8385
"recordCount": 4,
8486
"fileSizeInBytes": 1118,
8587
"fileCount": 2,
86-
"lastFileCreationTime": 1724090888727
88+
"lastFileCreationTime": 1724090888727,
89+
"totalBuckets": -1
8790
}
8891
]
8992
})";

src/paimon/core/manifest/partition_entry.cpp

Lines changed: 14 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -29,28 +29,31 @@
2929
namespace paimon {
3030
PartitionEntry::PartitionEntry(const BinaryRow& partition, int64_t record_count,
3131
int64_t file_size_in_bytes, int64_t file_count,
32-
int64_t last_file_creation_time)
32+
int64_t last_file_creation_time, int32_t total_buckets)
3333
: partition_(partition),
3434
record_count_(record_count),
3535
file_size_in_bytes_(file_size_in_bytes),
3636
file_count_(file_count),
37-
last_file_creation_time_(last_file_creation_time) {}
37+
last_file_creation_time_(last_file_creation_time),
38+
total_buckets_(total_buckets) {}
3839

3940
PartitionEntry PartitionEntry::Merge(const PartitionEntry& entry) const {
40-
return PartitionEntry(partition_, record_count_ + entry.record_count_,
41-
file_size_in_bytes_ + entry.file_size_in_bytes_,
42-
file_count_ + entry.file_count_,
43-
std::max(last_file_creation_time_, entry.last_file_creation_time_));
41+
return PartitionEntry(
42+
partition_, record_count_ + entry.record_count_,
43+
file_size_in_bytes_ + entry.file_size_in_bytes_, file_count_ + entry.file_count_,
44+
std::max(last_file_creation_time_, entry.last_file_creation_time_), entry.total_buckets_);
4445
}
4546

4647
Result<PartitionEntry> PartitionEntry::FromDataFile(const BinaryRow& partition,
4748
const FileKind& kind,
48-
const std::shared_ptr<DataFileMeta>& file) {
49+
const std::shared_ptr<DataFileMeta>& file,
50+
const int32_t total_buckets) {
4951
int64_t record_count = kind == FileKind::Delete() ? -file->row_count : file->row_count;
5052
int64_t file_size_in_bytes = kind == FileKind::Delete() ? -file->file_size : file->file_size;
5153
int64_t file_count = kind == FileKind::Delete() ? -1 : 1;
5254
PAIMON_ASSIGN_OR_RAISE(int64_t utc_millis, file->CreationTimeEpochMillis());
53-
return PartitionEntry(partition, record_count, file_size_in_bytes, file_count, utc_millis);
55+
return PartitionEntry(partition, record_count, file_size_in_bytes, file_count, utc_millis,
56+
total_buckets);
5457
}
5558

5659
Status PartitionEntry::Merge(const std::vector<ManifestEntry>& from,
@@ -75,7 +78,7 @@ Result<PartitionStatistics> PartitionEntry::ToPartitionStatistics(
7578
PAIMON_ASSIGN_OR_RAISE(part_values, partition_computer->GeneratePartitionVector(partition_));
7679
std::map<std::string, std::string> part_values_map(part_values.begin(), part_values.end());
7780
return PartitionStatistics(part_values_map, record_count_, file_size_in_bytes_, file_count_,
78-
last_file_creation_time_);
81+
last_file_creation_time_, total_buckets_);
7982
}
8083

8184
bool PartitionEntry::operator==(const PartitionEntry& other) const {
@@ -84,7 +87,8 @@ bool PartitionEntry::operator==(const PartitionEntry& other) const {
8487
}
8588
return partition_ == other.partition_ && record_count_ == other.record_count_ &&
8689
file_size_in_bytes_ == other.file_size_in_bytes_ && file_count_ == other.file_count_ &&
87-
last_file_creation_time_ == other.last_file_creation_time_;
90+
last_file_creation_time_ == other.last_file_creation_time_ &&
91+
total_buckets_ == other.total_buckets_;
8892
}
8993

9094
} // namespace paimon

src/paimon/core/manifest/partition_entry.h

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ class BinaryRowPartitionComputer;
3939
class PartitionEntry {
4040
public:
4141
PartitionEntry(const BinaryRow& partition, int64_t record_count, int64_t file_size_in_bytes,
42-
int64_t file_count, int64_t last_file_creation_time);
42+
int64_t file_count, int64_t last_file_creation_time, int32_t total_buckets);
4343

4444
const BinaryRow& Partition() const {
4545
return partition_;
@@ -58,14 +58,18 @@ class PartitionEntry {
5858
int64_t LastFileCreationTime() const {
5959
return last_file_creation_time_;
6060
}
61+
int32_t TotalBuckets() const {
62+
return total_buckets_;
63+
}
6164
PartitionEntry Merge(const PartitionEntry& entry) const;
6265

6366
static Result<PartitionEntry> FromManifestEntry(const ManifestEntry& entry) {
64-
return FromDataFile(entry.Partition(), entry.Kind(), entry.File());
67+
return FromDataFile(entry.Partition(), entry.Kind(), entry.File(), entry.TotalBuckets());
6568
}
6669

6770
static Result<PartitionEntry> FromDataFile(const BinaryRow& partition, const FileKind& kind,
68-
const std::shared_ptr<DataFileMeta>& file);
71+
const std::shared_ptr<DataFileMeta>& file,
72+
int32_t total_buckets);
6973

7074
Result<PartitionStatistics> ToPartitionStatistics(
7175
const BinaryRowPartitionComputer* partition_computer) const;
@@ -81,5 +85,6 @@ class PartitionEntry {
8185
int64_t file_size_in_bytes_;
8286
int64_t file_count_;
8387
int64_t last_file_creation_time_;
88+
int32_t total_buckets_;
8489
};
8590
} // namespace paimon

src/paimon/core/manifest/partition_entry_test.cpp

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -66,8 +66,10 @@ TEST_F(PartitionEntryTest, TestSimple) {
6666
/*total_buckets=*/2, GetDataFileMeta(Timestamp(1500, 123)))));
6767

6868
auto res = partition_entry1.Merge(partition_entry2);
69-
auto expected_partition_entry = PartitionEntry(BinaryRow::EmptyRow(), /*record_count=*/10,
70-
/*file_size_in_bytes=*/20, 2, -28798500);
69+
auto expected_partition_entry =
70+
PartitionEntry(BinaryRow::EmptyRow(), /*record_count=*/10,
71+
/*file_size_in_bytes=*/20, /*file_count=*/2,
72+
/*last_file_creation_time=*/-28798500, /*total_buckets=*/2);
7173
ASSERT_EQ(res, expected_partition_entry);
7274
}
7375
{
@@ -82,8 +84,10 @@ TEST_F(PartitionEntryTest, TestSimple) {
8284
/*total_buckets=*/2, GetDataFileMeta(Timestamp(1500, 123)))));
8385

8486
auto res = partition_entry1.Merge(partition_entry2);
85-
auto expected_partition_entry = PartitionEntry(BinaryRow::EmptyRow(), /*record_count=*/0,
86-
/*file_size_in_bytes=*/0, 0, -28798500);
87+
auto expected_partition_entry =
88+
PartitionEntry(BinaryRow::EmptyRow(), /*record_count=*/0,
89+
/*file_size_in_bytes=*/0, /*file_count=*/0,
90+
/*last_file_creation_time=*/-28798500, /*total_buckets=*/2);
8791
ASSERT_EQ(res, expected_partition_entry);
8892
}
8993
}
@@ -108,10 +112,10 @@ TEST_F(PartitionEntryTest, TestMerge) {
108112
std::unordered_map<BinaryRow, PartitionEntry> expected_partition_entry;
109113
expected_partition_entry.emplace(
110114
std::piecewise_construct, std::forward_as_tuple(partition0),
111-
std::forward_as_tuple(PartitionEntry(partition0, 10, 20, 2, -28798500)));
115+
std::forward_as_tuple(PartitionEntry(partition0, 10, 20, 2, -28798500, 2)));
112116
expected_partition_entry.emplace(
113117
std::piecewise_construct, std::forward_as_tuple(partition1),
114-
std::forward_as_tuple(PartitionEntry(partition1, 5, 10, 1, -28800000)));
118+
std::forward_as_tuple(PartitionEntry(partition1, 5, 10, 1, -28800000, 2)));
115119
ASSERT_EQ(to, expected_partition_entry);
116120
}
117121

src/paimon/core/operation/append_only_file_store_scan_test.cpp

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -111,9 +111,11 @@ TEST(AppendOnlyFileStoreScanTest, TestReadPartitionEntries) {
111111

112112
std::vector<PartitionEntry> expected_partition_entries = {
113113
PartitionEntry(GenerateRow(10), /*record_count=*/9, /*file_size_in_bytes=*/1183,
114-
/*file_count=*/2, /*last_file_creation_time=*/1721643834472l - 28800000l),
114+
/*file_count=*/2, /*last_file_creation_time=*/1721643834472l - 28800000l,
115+
/*total_buckets=*/2),
115116
PartitionEntry(GenerateRow(20), /*record_count=*/2, /*file_size_in_bytes=*/1047,
116-
/*file_count=*/2, /*last_file_creation_time=*/1721643267404l - 28800000l)};
117+
/*file_count=*/2, /*last_file_creation_time=*/1721643267404l - 28800000l,
118+
/*total_buckets=*/2)};
117119
auto ComparePartitionEntryByPartition = [](const PartitionEntry& lhs,
118120
const PartitionEntry& rhs) -> bool {
119121
return lhs.Partition().GetInt(0) < rhs.Partition().GetInt(0);

src/paimon/core/operation/file_store_commit_impl_test.cpp

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -344,10 +344,12 @@ TEST_F(FileStoreCommitImplTest, TestRESTCatalogCommit) {
344344
std::vector<PartitionStatistics> expected_partition_statistics = {
345345
PartitionStatistics(/*spec=*/{{"f1", "20"}}, /*record_count=*/1, /*file_size_in_bytes=*/541,
346346
/*file_count=*/1,
347-
/*last_file_creation_time=*/1724090888743l - 28800000l),
347+
/*last_file_creation_time=*/1724090888743l - 28800000l,
348+
/*total_buckets=*/-1),
348349
PartitionStatistics(/*spec=*/{{"f1", "10"}}, /*record_count=*/4,
349350
/*file_size_in_bytes=*/1118, /*file_count=*/2,
350-
/*last_file_creation_time=*/1724090888727l - 28800000l)};
351+
/*last_file_creation_time=*/1724090888727l - 28800000l,
352+
/*total_buckets=*/-1)};
351353
CommitTableRequest expected_commit_table_request(expected_snapshot,
352354
expected_partition_statistics);
353355
ASSERT_TRUE(commit_table_request.TEST_Equal(expected_commit_table_request));

src/paimon/core/partition/partition_statistics.h

Lines changed: 15 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -34,12 +34,13 @@ class PartitionStatistics : public Jsonizable<PartitionStatistics> {
3434
public:
3535
using SpecType = std::map<std::string, std::string>;
3636
PartitionStatistics(const SpecType& spec, int64_t record_count, int64_t file_size_in_bytes,
37-
int64_t file_count, int64_t last_file_creation_time)
37+
int64_t file_count, int64_t last_file_creation_time, int32_t total_buckets)
3838
: spec_(spec),
3939
record_count_(record_count),
4040
file_size_in_bytes_(file_size_in_bytes),
4141
file_count_(file_count),
42-
last_file_creation_time_(last_file_creation_time) {}
42+
last_file_creation_time_(last_file_creation_time),
43+
total_buckets_(total_buckets) {}
4344

4445
const SpecType& Spec() const {
4546
return spec_;
@@ -56,6 +57,9 @@ class PartitionStatistics : public Jsonizable<PartitionStatistics> {
5657
int64_t LastFileCreationTime() const {
5758
return last_file_creation_time_;
5859
}
60+
int32_t TotalBuckets() const {
61+
return total_buckets_;
62+
}
5963

6064
rapidjson::Value ToJson(rapidjson::Document::AllocatorType* allocator) const
6165
noexcept(false) override {
@@ -72,6 +76,8 @@ class PartitionStatistics : public Jsonizable<PartitionStatistics> {
7276
obj.AddMember(rapidjson::StringRef(FIELD_LAST_FILE_CREATION_TIME),
7377
RapidJsonUtil::SerializeValue(last_file_creation_time_, allocator).Move(),
7478
*allocator);
79+
obj.AddMember(rapidjson::StringRef(FIELD_TOTAL_BUCKETS),
80+
RapidJsonUtil::SerializeValue(total_buckets_, allocator).Move(), *allocator);
7581
return obj;
7682
}
7783

@@ -83,12 +89,14 @@ class PartitionStatistics : public Jsonizable<PartitionStatistics> {
8389
file_count_ = RapidJsonUtil::DeserializeKeyValue<int64_t>(obj, FIELD_FILE_COUNT);
8490
last_file_creation_time_ =
8591
RapidJsonUtil::DeserializeKeyValue<int64_t>(obj, FIELD_LAST_FILE_CREATION_TIME);
92+
total_buckets_ = RapidJsonUtil::DeserializeKeyValue<int32_t>(obj, FIELD_TOTAL_BUCKETS);
8693
}
8794

8895
bool operator==(const PartitionStatistics& rhs) const {
8996
return record_count_ == rhs.record_count_ &&
9097
file_size_in_bytes_ == rhs.file_size_in_bytes_ && file_count_ == rhs.file_count_ &&
91-
last_file_creation_time_ == rhs.last_file_creation_time_ && spec_ == rhs.spec_;
98+
last_file_creation_time_ == rhs.last_file_creation_time_ &&
99+
total_buckets_ == rhs.total_buckets_ && spec_ == rhs.spec_;
92100
}
93101

94102
std::string ToString() const {
@@ -102,7 +110,8 @@ class PartitionStatistics : public Jsonizable<PartitionStatistics> {
102110
}
103111
oss << "}, recordCount=" << record_count_ << ", fileSizeInBytes=" << file_size_in_bytes_
104112
<< ", fileCount=" << file_count_
105-
<< ", lastFileCreationTime=" << last_file_creation_time_ << "}";
113+
<< ", lastFileCreationTime=" << last_file_creation_time_
114+
<< ", totalBuckets=" << total_buckets_ << "}";
106115
return oss.str();
107116
}
108117

@@ -115,12 +124,14 @@ class PartitionStatistics : public Jsonizable<PartitionStatistics> {
115124
static constexpr const char* FIELD_FILE_SIZE_IN_BYTES = "fileSizeInBytes";
116125
static constexpr const char* FIELD_FILE_COUNT = "fileCount";
117126
static constexpr const char* FIELD_LAST_FILE_CREATION_TIME = "lastFileCreationTime";
127+
static constexpr const char* FIELD_TOTAL_BUCKETS = "totalBuckets";
118128

119129
SpecType spec_;
120130
int64_t record_count_ = 0;
121131
int64_t file_size_in_bytes_ = 0;
122132
int64_t file_count_ = 0;
123133
int64_t last_file_creation_time_ = 0;
134+
int32_t total_buckets_ = 0;
124135
};
125136

126137
} // namespace paimon

src/paimon/core/partition/partition_statistics_test.cpp

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -43,15 +43,16 @@ TEST_F(PartitionStatisticsTest, TestJsonizable) {
4343
"recordCount": 4,
4444
"fileSizeInBytes": 1118,
4545
"fileCount": 2,
46-
"lastFileCreationTime": 1724090888727
46+
"lastFileCreationTime": 1724090888727,
47+
"totalBuckets": 1
4748
})";
4849

4950
ASSERT_OK_AND_ASSIGN(PartitionStatistics partition_statistics,
5051
PartitionStatistics::FromJsonString(json_str));
5152

5253
PartitionStatistics expected_partition_statistics(
5354
/*spec=*/{{"f1", "10"}, {"f2", "20"}}, /*record_count=*/4, /*file_size_in_bytes=*/1118,
54-
/*file_count=*/2, /*last_file_creation_time=*/1724090888727);
55+
/*file_count=*/2, /*last_file_creation_time=*/1724090888727, /*total_buckets=*/1);
5556
ASSERT_EQ(expected_partition_statistics, partition_statistics);
5657

5758
ASSERT_OK_AND_ASSIGN(std::string new_json_str, partition_statistics.ToJsonString());

0 commit comments

Comments
 (0)