Skip to content

Commit 9666fb4

Browse files
author
duanyan.duan
committed
fix(blob): allow blob files with different schema ids in a bunch
BlobBunch::Add required every file in a bunch to share the same schema id. After schema evolution that adds/drops OTHER columns, a single blob field's contiguous files can carry different schema ids (an old appended file plus a compacted file), and MergeRangesAndSort groups them into one bunch by row id, raising a spurious "All files in a blob bunch should have the same schema id." Paimon Java (SpecialFieldBunch.add) guards this check with if (!isBlobFile(file)) so blob files are exempt -- a blob column's on-disk layout is schema-independent and the bunch is read with the first file's schema regardless. The guard was dropped when porting to C++. Restore it and add a regression unit test.
1 parent ffbda4f commit 9666fb4

2 files changed

Lines changed: 36 additions & 4 deletions

File tree

src/paimon/core/operation/data_evolution_split_read.cpp

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -94,8 +94,18 @@ Status DataEvolutionSplitRead::BlobBunch::Add(const std::shared_ptr<DataFileMeta
9494
}
9595
}
9696
if (!files_.empty()) {
97-
if (file->schema_id != files_[0]->schema_id) {
98-
return Status::Invalid("All files in a blob bunch should have the same schema id.");
97+
// Mirror Paimon Java (SpecialFieldBunch.add): the schema id must agree only for
98+
// non-blob (vector-store) files. Blob files are allowed to span schema ids -- a
99+
// blob column's on-disk layout is schema-independent, so after schema evolution
100+
// (which adds/drops OTHER columns) the same blob field can have contiguous files
101+
// written under different schema ids that MergeRangesAndSort groups into one
102+
// bunch. The Java guard `if (!isBlobFile(file))` around this check was dropped
103+
// when porting, which turned that legitimate case into a spurious error.
104+
if (!BlobUtils::IsBlobFile(file->file_name)) {
105+
if (file->schema_id != files_[0]->schema_id) {
106+
return Status::Invalid(
107+
"All files in a blob bunch should have the same schema id.");
108+
}
99109
}
100110
if (file->write_cols != files_[0]->write_cols) {
101111
return Status::Invalid(

src/paimon/core/operation/data_evolution_split_read_test.cpp

Lines changed: 24 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -84,12 +84,12 @@ class DataEvolutionSplitReadTest : public ::testing::Test {
8484
std::shared_ptr<DataFileMeta> CreateBlobFile(
8585
const std::string& file_name, int64_t first_row_id, int64_t row_count,
8686
int64_t max_sequence_number,
87-
const std::optional<std::vector<std::string>>& write_cols) const {
87+
const std::optional<std::vector<std::string>>& write_cols, int64_t schema_id = 0) const {
8888
return DataFileMeta::ForAppend(file_name + ".blob", /*file_size=*/row_count,
8989
/*row_count=*/row_count,
9090
/*row_stats=*/SimpleStats::EmptyStats(),
9191
/*min_sequence_number=*/0, max_sequence_number,
92-
/*schema_id=*/0, FileSource::Append(),
92+
schema_id, FileSource::Append(),
9393
/*value_stats_cols=*/std::nullopt,
9494
/*external_path=*/std::nullopt, first_row_id, write_cols)
9595
.value();
@@ -247,6 +247,28 @@ TEST_F(DataEvolutionSplitReadTest, TestAddBlobFileWithDifferentWriteCols) {
247247
"All files in a blob bunch should have the same write columns.");
248248
}
249249

250+
TEST_F(DataEvolutionSplitReadTest, TestAddBlobFilesWithDifferentSchemaId) {
251+
// A blob field written under schema 0 for the first rows and under schema 1 for the
252+
// following rows (after schema evolution that adds/drops OTHER columns) must still be
253+
// groupable into a single bunch: a blob column's on-disk layout is schema-independent.
254+
// Same write cols, contiguous row ids, different schema id -> both added OK.
255+
auto blob_entry =
256+
CreateBlobFile("blob1", /*first_row_id=*/0, /*row_count=*/100,
257+
/*max_sequence_number=*/1,
258+
/*write_cols=*/std::optional<std::vector<std::string>>({"blob_col"}),
259+
/*schema_id=*/0);
260+
auto blob_tail =
261+
CreateBlobFile("blob2", /*first_row_id=*/100, /*row_count=*/200,
262+
/*max_sequence_number=*/1,
263+
/*write_cols=*/std::optional<std::vector<std::string>>({"blob_col"}),
264+
/*schema_id=*/1);
265+
auto blob_bunch = std::make_shared<DataEvolutionSplitRead::BlobBunch>(
266+
INT64_MAX, /*has_row_ids_selection=*/false);
267+
ASSERT_OK(blob_bunch->Add(blob_entry));
268+
ASSERT_OK(blob_bunch->Add(blob_tail));
269+
ASSERT_EQ(blob_bunch->Files().size(), 2);
270+
}
271+
250272
TEST_F(DataEvolutionSplitReadTest, TestRowIdSelectionWithOverlap) {
251273
auto blob_entry =
252274
CreateBlobFile("blob1", /*first_row_id=*/0, /*row_count=*/10,

0 commit comments

Comments
 (0)