Skip to content

Commit 6b0bc7b

Browse files
authored
Merge branch 'main' into support_avro_write
2 parents 1ddfbfc + 00fd06d commit 6b0bc7b

18 files changed

Lines changed: 171 additions & 36 deletions

include/paimon/file_index/file_index_reader.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,8 @@ class PAIMON_EXPORT FileIndexReader : public FunctionVisitor<std::shared_ptr<Fil
6060
Result<std::shared_ptr<FileIndexResult>> VisitEndsWith(const Literal& suffix) override;
6161

6262
Result<std::shared_ptr<FileIndexResult>> VisitContains(const Literal& literal) override;
63+
64+
Result<std::shared_ptr<FileIndexResult>> VisitLike(const Literal& literal) override;
6365
};
6466

6567
} // namespace paimon

include/paimon/predicate/function_visitor.h

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -71,5 +71,8 @@ class PAIMON_EXPORT FunctionVisitor {
7171

7272
/// Evaluates whether string values contain the given substring.
7373
virtual Result<T> VisitContains(const Literal& literal) = 0;
74+
75+
/// Evaluates whether string values like the given string.
76+
virtual Result<T> VisitLike(const Literal& literal) = 0;
7477
};
7578
} // namespace paimon

src/paimon/common/file_index/empty/empty_file_index_reader.h

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,9 @@ class EmptyFileIndexReader : public FileIndexReader {
4646
Result<std::shared_ptr<FileIndexResult>> VisitContains(const Literal& literal) override {
4747
return FileIndexResult::Skip();
4848
}
49+
Result<std::shared_ptr<FileIndexResult>> VisitLike(const Literal& literal) override {
50+
return FileIndexResult::Skip();
51+
}
4952
Result<std::shared_ptr<FileIndexResult>> VisitLessThan(const Literal& literal) override {
5053
return FileIndexResult::Skip();
5154
}

src/paimon/common/file_index/empty/empty_file_index_reader_test.cpp

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ TEST(EmptyFileIndexReaderTest, TestSimple) {
3131
ASSERT_FALSE(reader.VisitStartsWith(lit0).value()->IsRemain().value());
3232
ASSERT_FALSE(reader.VisitEndsWith(lit0).value()->IsRemain().value());
3333
ASSERT_FALSE(reader.VisitContains(lit0).value()->IsRemain().value());
34+
ASSERT_FALSE(reader.VisitLike(lit0).value()->IsRemain().value());
3435
ASSERT_FALSE(reader.VisitLessThan(lit0).value()->IsRemain().value());
3536
ASSERT_FALSE(reader.VisitGreaterOrEqual(lit0).value()->IsRemain().value());
3637
ASSERT_FALSE(reader.VisitLessOrEqual(lit0).value()->IsRemain().value());

src/paimon/common/file_index/file_index_reader.cpp

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,10 @@ Result<std::shared_ptr<FileIndexResult>> FileIndexReader::VisitContains(const Li
4141
return FileIndexResult::Remain();
4242
}
4343

44+
Result<std::shared_ptr<FileIndexResult>> FileIndexReader::VisitLike(const Literal& literal) {
45+
return FileIndexResult::Remain();
46+
}
47+
4448
Result<std::shared_ptr<FileIndexResult>> FileIndexReader::VisitLessThan(const Literal& literal) {
4549
return FileIndexResult::Remain();
4650
}

src/paimon/common/global_index/wrap/file_index_reader_wrapper.h

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -114,6 +114,12 @@ class FileIndexReaderWrapper : public GlobalIndexReader {
114114
return transform_(file_index_result);
115115
}
116116

117+
Result<std::shared_ptr<GlobalIndexResult>> VisitLike(const Literal& literal) override {
118+
PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<FileIndexResult> file_index_result,
119+
reader_->VisitLike(literal));
120+
return transform_(file_index_result);
121+
}
122+
117123
Result<std::shared_ptr<VectorSearchGlobalIndexResult>> VisitVectorSearch(
118124
const std::shared_ptr<VectorSearch>& vector_search) override {
119125
return Status::Invalid(

src/paimon/common/memory/bytes.cpp

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -59,9 +59,16 @@ Bytes& Bytes::operator=(Bytes&& other) noexcept {
5959
if (&other == this) {
6060
return *this;
6161
}
62-
this->~Bytes();
63-
std::memcpy(this, &other, sizeof(*this));
64-
new (&other) Bytes();
62+
if (data_ != nullptr) {
63+
assert(pool_);
64+
pool_->Free(data_, size_);
65+
}
66+
pool_ = other.pool_;
67+
data_ = other.data_;
68+
size_ = other.size_;
69+
other.pool_ = nullptr;
70+
other.data_ = nullptr;
71+
other.size_ = 0;
6572
return *this;
6673
}
6774

src/paimon/common/memory/bytes_test.cpp

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -82,4 +82,73 @@ TEST(BytesTest, TestCompare) {
8282
ASSERT_LT(*bytes1, *bytes2);
8383
ASSERT_FALSE(*bytes1 < *bytes1);
8484
}
85+
86+
// Test to verify that move assignment correctly handles memory and prevents double-free.
87+
// Before the fix, the old implementation used memcpy + destructor which caused:
88+
// 1. The target's original memory was freed in destructor
89+
// 2. After memcpy, both source and target pointed to same memory
90+
// 3. When source was "reset" via placement new, it became empty
91+
// 4. But if move assignment was called again on the same target, the memcpy'd
92+
// pointer would be freed again (double-free) or memory accounting would be wrong.
93+
TEST(BytesTest, TestMoveAssignmentNoDoubleFree) {
94+
auto pool = paimon::GetMemoryPool();
95+
96+
// Create three Bytes objects on stack
97+
Bytes a("aaaa", pool.get()); // 4 bytes
98+
Bytes b("bb", pool.get()); // 2 bytes
99+
Bytes c("cccccc", pool.get()); // 6 bytes
100+
ASSERT_EQ(12, pool->CurrentUsage()); // 4 + 2 + 6 = 12
101+
102+
// First move: b = std::move(a)
103+
// Should free b's original memory (2 bytes), transfer a's memory to b
104+
b = std::move(a);
105+
ASSERT_EQ(10, pool->CurrentUsage()); // 4 + 6 = 10 (b's 2 bytes freed)
106+
ASSERT_EQ("aaaa", std::string(b.data(), b.size()));
107+
// Moved-from object is expected to be empty by Bytes' contract.
108+
ASSERT_EQ(nullptr, a.data()); // NOLINT(bugprone-use-after-move, clang-analyzer-cplusplus.Move)
109+
ASSERT_EQ(0, a.size()); // NOLINT(bugprone-use-after-move, clang-analyzer-cplusplus.Move)
110+
111+
// Second move: b = std::move(c)
112+
// Should free b's current memory (4 bytes from a), transfer c's memory to b
113+
// This is where the old implementation would cause issues:
114+
// - Old code would call destructor on b, freeing the 4 bytes
115+
// - Then memcpy c into b, making b point to c's 6-byte buffer
116+
// - Memory accounting would be wrong because Free was called on wrong data
117+
b = std::move(c);
118+
ASSERT_EQ(6, pool->CurrentUsage()); // Only c's 6 bytes remain (now owned by b)
119+
ASSERT_EQ("cccccc", std::string(b.data(), b.size()));
120+
// Moved-from object is expected to be empty by Bytes' contract.
121+
ASSERT_EQ(nullptr, c.data()); // NOLINT(bugprone-use-after-move, clang-analyzer-cplusplus.Move)
122+
ASSERT_EQ(0, c.size()); // NOLINT(bugprone-use-after-move, clang-analyzer-cplusplus.Move)
123+
124+
// Self-assignment should be safe. Use an alias to avoid -Wself-move.
125+
Bytes* self = &b;
126+
b = std::move(*self);
127+
ASSERT_EQ(6, pool->CurrentUsage());
128+
ASSERT_EQ("cccccc", std::string(b.data(), b.size()));
129+
}
130+
131+
// Test move assignment with heap-allocated Bytes to verify no double-free
132+
// when combining unique_ptr semantics with move assignment
133+
TEST(BytesTest, TestMoveAssignmentHeapAllocated) {
134+
auto pool = paimon::GetMemoryPool();
135+
136+
auto bytes1 = Bytes::AllocateBytes("hello", pool.get()); // 5 bytes + sizeof(Bytes)
137+
auto bytes2 = Bytes::AllocateBytes("world!", pool.get()); // 6 bytes + sizeof(Bytes)
138+
size_t expected = 5 + 6 + 2 * sizeof(Bytes);
139+
ASSERT_EQ(expected, pool->CurrentUsage());
140+
141+
// Move the content of bytes1 into bytes2's Bytes object
142+
// This should free "world!" (6 bytes) and transfer "hello" ownership
143+
*bytes2 = std::move(*bytes1);
144+
expected = 5 + 2 * sizeof(Bytes); // "world!" freed, "hello" transferred
145+
ASSERT_EQ(expected, pool->CurrentUsage());
146+
ASSERT_EQ("hello", std::string(bytes2->data(), bytes2->size()));
147+
ASSERT_EQ(nullptr, bytes1->data());
148+
149+
// Reset bytes2, which should free "hello"
150+
bytes2.reset();
151+
expected = sizeof(Bytes); // Only bytes1's empty Bytes struct remains
152+
ASSERT_EQ(expected, pool->CurrentUsage());
153+
}
85154
} // namespace paimon::test

src/paimon/core/catalog/commit_table_request_test.cpp

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -42,10 +42,11 @@ TEST(CommitTableRequestTest, TestSimple) {
4242
/*statistics=*/std::nullopt, /*properties=*/std::nullopt, /*next_row_id=*/0);
4343
std::vector<PartitionStatistics> partition_statistics = {
4444
PartitionStatistics(/*spec=*/{{"f1", "20"}}, /*record_count=*/1, /*file_size_in_bytes=*/541,
45-
/*file_count=*/1, /*last_file_creation_time=*/1724090888743),
45+
/*file_count=*/1, /*last_file_creation_time=*/1724090888743,
46+
/*total_buckets=*/-1),
4647
PartitionStatistics(/*spec=*/{{"f1", "10"}}, /*record_count=*/4,
4748
/*file_size_in_bytes=*/1118, /*file_count=*/2,
48-
/*last_file_creation_time=*/1724090888727)};
49+
/*last_file_creation_time=*/1724090888727, /*total_buckets=*/-1)};
4950
std::string expected_request_str = R"({
5051
"snapshot": {
5152
"version": 3,
@@ -74,7 +75,8 @@ TEST(CommitTableRequestTest, TestSimple) {
7475
"recordCount": 1,
7576
"fileSizeInBytes": 541,
7677
"fileCount": 1,
77-
"lastFileCreationTime": 1724090888743
78+
"lastFileCreationTime": 1724090888743,
79+
"totalBuckets": -1
7880
},
7981
{
8082
"spec": {
@@ -83,7 +85,8 @@ TEST(CommitTableRequestTest, TestSimple) {
8385
"recordCount": 4,
8486
"fileSizeInBytes": 1118,
8587
"fileCount": 2,
86-
"lastFileCreationTime": 1724090888727
88+
"lastFileCreationTime": 1724090888727,
89+
"totalBuckets": -1
8790
}
8891
]
8992
})";

src/paimon/core/manifest/partition_entry.cpp

Lines changed: 14 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -29,28 +29,31 @@
2929
namespace paimon {
3030
PartitionEntry::PartitionEntry(const BinaryRow& partition, int64_t record_count,
3131
int64_t file_size_in_bytes, int64_t file_count,
32-
int64_t last_file_creation_time)
32+
int64_t last_file_creation_time, int32_t total_buckets)
3333
: partition_(partition),
3434
record_count_(record_count),
3535
file_size_in_bytes_(file_size_in_bytes),
3636
file_count_(file_count),
37-
last_file_creation_time_(last_file_creation_time) {}
37+
last_file_creation_time_(last_file_creation_time),
38+
total_buckets_(total_buckets) {}
3839

3940
PartitionEntry PartitionEntry::Merge(const PartitionEntry& entry) const {
40-
return PartitionEntry(partition_, record_count_ + entry.record_count_,
41-
file_size_in_bytes_ + entry.file_size_in_bytes_,
42-
file_count_ + entry.file_count_,
43-
std::max(last_file_creation_time_, entry.last_file_creation_time_));
41+
return PartitionEntry(
42+
partition_, record_count_ + entry.record_count_,
43+
file_size_in_bytes_ + entry.file_size_in_bytes_, file_count_ + entry.file_count_,
44+
std::max(last_file_creation_time_, entry.last_file_creation_time_), entry.total_buckets_);
4445
}
4546

4647
Result<PartitionEntry> PartitionEntry::FromDataFile(const BinaryRow& partition,
4748
const FileKind& kind,
48-
const std::shared_ptr<DataFileMeta>& file) {
49+
const std::shared_ptr<DataFileMeta>& file,
50+
const int32_t total_buckets) {
4951
int64_t record_count = kind == FileKind::Delete() ? -file->row_count : file->row_count;
5052
int64_t file_size_in_bytes = kind == FileKind::Delete() ? -file->file_size : file->file_size;
5153
int64_t file_count = kind == FileKind::Delete() ? -1 : 1;
5254
PAIMON_ASSIGN_OR_RAISE(int64_t utc_millis, file->CreationTimeEpochMillis());
53-
return PartitionEntry(partition, record_count, file_size_in_bytes, file_count, utc_millis);
55+
return PartitionEntry(partition, record_count, file_size_in_bytes, file_count, utc_millis,
56+
total_buckets);
5457
}
5558

5659
Status PartitionEntry::Merge(const std::vector<ManifestEntry>& from,
@@ -75,7 +78,7 @@ Result<PartitionStatistics> PartitionEntry::ToPartitionStatistics(
7578
PAIMON_ASSIGN_OR_RAISE(part_values, partition_computer->GeneratePartitionVector(partition_));
7679
std::map<std::string, std::string> part_values_map(part_values.begin(), part_values.end());
7780
return PartitionStatistics(part_values_map, record_count_, file_size_in_bytes_, file_count_,
78-
last_file_creation_time_);
81+
last_file_creation_time_, total_buckets_);
7982
}
8083

8184
bool PartitionEntry::operator==(const PartitionEntry& other) const {
@@ -84,7 +87,8 @@ bool PartitionEntry::operator==(const PartitionEntry& other) const {
8487
}
8588
return partition_ == other.partition_ && record_count_ == other.record_count_ &&
8689
file_size_in_bytes_ == other.file_size_in_bytes_ && file_count_ == other.file_count_ &&
87-
last_file_creation_time_ == other.last_file_creation_time_;
90+
last_file_creation_time_ == other.last_file_creation_time_ &&
91+
total_buckets_ == other.total_buckets_;
8892
}
8993

9094
} // namespace paimon

0 commit comments

Comments
 (0)