Skip to content

Commit c501f7f

Browse files
zhf999zhf999
committed
test: add test cases
1 parent 0c8443d commit c501f7f

2 files changed

Lines changed: 144 additions & 47 deletions

File tree

‎src/paimon/format/parquet/page_filtered_row_group_reader_test.cpp‎

Lines changed: 123 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -1346,39 +1346,6 @@ TEST_F(PageFilteredRowGroupReaderTest, BitmapAllAndPartialPagesMixed) {
13461346
}
13471347
}
13481348

1349-
/// Test: bitmap hits partial pages of a row group, with page-filtered option disabled.
1350-
///
1351-
/// 200 rows, 10 rows per page, 100 rows per row group → 2 row groups.
1352-
/// Bitmap: {120..149} hits pages 2-4 of RG1.
1353-
/// Expected: 100 rows (100-199) because page-filtered option is disabled, so page-level bitmap is
1354-
/// ignored.
1355-
TEST_F(PageFilteredRowGroupReaderTest, BitmapWithPageFilteredOptionDisabled) {
1356-
std::string file_name = dir_->Str() + "/bitmap_all_and_partial.parquet";
1357-
auto data = MakeSequentialIntData(200);
1358-
WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/100);
1359-
1360-
RoaringBitmap32 bitmap;
1361-
bitmap.AddRange(120, 150); // pages 2-4 of RG1
1362-
1363-
auto read_schema = arrow::schema({arrow::field("val", arrow::int32())});
1364-
std::map<std::string, std::string> options;
1365-
options[PARQUET_READ_ENABLE_PAGE_INDEX_FILTER] = "false";
1366-
std::shared_ptr<arrow::ChunkedArray> result;
1367-
ReadWithPredicateAndBitmapImpl(file_name, read_schema, /*predicate=*/nullptr, bitmap, &result,
1368-
options);
1369-
ASSERT_TRUE(result);
1370-
ASSERT_EQ(100, result->length());
1371-
1372-
// Verify: 100-199
1373-
auto flat = arrow::Concatenate(result->chunks()).ValueOrDie();
1374-
auto struct_arr = std::dynamic_pointer_cast<arrow::StructArray>(flat);
1375-
ASSERT_TRUE(struct_arr);
1376-
auto val_arr = std::dynamic_pointer_cast<arrow::Int32Array>(struct_arr->field(0));
1377-
for (int32_t i = 0; i < 100; ++i) {
1378-
ASSERT_EQ(100 + i, val_arr->Value(i));
1379-
}
1380-
}
1381-
13821349
/// Test: bitmap + predicate both applied, bitmap hits all pages of some row groups.
13831350
///
13841351
/// 200 rows, 10 rows per page, 100 rows per row group → 2 row groups.
@@ -1615,4 +1582,127 @@ TEST_F(PageFilteredRowGroupReaderTest, BitmapTrimStrategyTest) {
16151582
}
16161583
}
16171584

1585+
/// Test: invalid strategy value returns Status::Invalid.
1586+
///
1587+
/// 200 rows, 50 rows per page, 100 rows per row group → 2 row groups.
1588+
/// Bitmap: [0,10), [45, 50), [60, 70) — same as BitmapCoalesceTest.
1589+
/// Strategy: "invalid" (not one of "coalesce", "trim", "none").
1590+
/// Expected: SetReadSchema returns Status::Invalid.
1591+
TEST_F(PageFilteredRowGroupReaderTest, BitmapInvalidStrategyTest) {
1592+
std::string file_name = dir_->Str() + "/invalid_strategy.parquet";
1593+
auto data = MakeSequentialIntData(200);
1594+
WriteTestFile(file_name, data, /*write_batch_size=*/50, /*max_row_group_length=*/100);
1595+
1596+
RoaringBitmap32 bitmap;
1597+
bitmap.AddRange(0, 10);
1598+
bitmap.AddRange(45, 50);
1599+
bitmap.AddRange(60, 70);
1600+
1601+
std::map<std::string, std::string> options;
1602+
options[PARQUET_READ_BITMAP_ROW_RANGE_REFINING_STRATEGY] = "invalid";
1603+
1604+
ASSERT_OK_AND_ASSIGN(std::shared_ptr<InputStream> in, fs_->Open(file_name));
1605+
ASSERT_OK_AND_ASSIGN(int64_t length, in->Length());
1606+
auto in_stream = std::make_shared<ArrowInputStreamAdapter>(in, arrow_pool_, length);
1607+
1608+
ASSERT_OK_AND_ASSIGN(
1609+
auto batch_reader,
1610+
ParquetFileBatchReader::Create(std::move(in_stream), options, 1024, nullptr, arrow_pool_));
1611+
1612+
auto read_schema = arrow::schema({arrow::field("val", arrow::int32())});
1613+
auto c_schema = std::make_unique<ArrowSchema>();
1614+
ASSERT_TRUE(arrow::ExportSchema(*read_schema, c_schema.get()).ok());
1615+
1616+
auto status = batch_reader->SetReadSchema(c_schema.get(), /*predicate=*/nullptr, bitmap);
1617+
ASSERT_FALSE(status.ok());
1618+
ASSERT_TRUE(status.IsInvalid());
1619+
}
1620+
1621+
/// Test: "none" strategy performs no row-range refinement within a row group.
1622+
///
1623+
/// 200 rows, 50 rows per page, 100 rows per row group → 2 row groups.
1624+
/// Bitmap: [0,10), [45, 50), [60, 70) — all within RG0 (rows 0-99).
1625+
/// Strategy: "none" → no refinement, so the entire RG0 (100 rows) is read.
1626+
/// Expected: 100 rows (0-99).
1627+
/// Compare with BitmapCoalesceTest (35 rows) and BitmapTrimStrategyTest (60 rows) which use
1628+
/// the same bitmap but apply refinement.
1629+
TEST_F(PageFilteredRowGroupReaderTest, BitmapNoneStrategyTest) {
1630+
std::string file_name = dir_->Str() + "/none_strategy.parquet";
1631+
auto data = MakeSequentialIntData(200);
1632+
WriteTestFile(file_name, data, /*write_batch_size=*/50, /*max_row_group_length=*/100);
1633+
1634+
RoaringBitmap32 bitmap;
1635+
bitmap.AddRange(0, 10);
1636+
bitmap.AddRange(45, 50);
1637+
bitmap.AddRange(60, 70);
1638+
1639+
std::map<std::string, std::string> options;
1640+
options[PARQUET_READ_BITMAP_ROW_RANGE_REFINING_STRATEGY] = "none";
1641+
1642+
auto read_schema = arrow::schema({arrow::field("val", arrow::int32())});
1643+
std::shared_ptr<arrow::ChunkedArray> result;
1644+
ReadWithPredicateAndBitmapImpl(file_name, read_schema, /*predicate=*/nullptr, bitmap, &result,
1645+
options);
1646+
ASSERT_TRUE(result);
1647+
ASSERT_EQ(100, result->length());
1648+
1649+
auto flat = arrow::Concatenate(result->chunks()).ValueOrDie();
1650+
auto struct_arr = std::dynamic_pointer_cast<arrow::StructArray>(flat);
1651+
ASSERT_TRUE(struct_arr);
1652+
auto val_arr = std::dynamic_pointer_cast<arrow::Int32Array>(struct_arr->field(0));
1653+
for (int32_t i = 0; i < 100; ++i) {
1654+
ASSERT_EQ(i, val_arr->Value(i));
1655+
}
1656+
}
1657+
1658+
/// Test: trim strategy with multiple columns — intersection of per-column trimmed ranges.
1659+
///
1660+
/// 200 rows, 50 rows per page, 100 rows per row group → 2 row groups.
1661+
/// Two columns: a[i] = i, b[i] = i * 10.
1662+
/// Bitmap: [0,10), [45, 50), [60, 70) — same as BitmapTrimStrategyTest.
1663+
/// Both columns share the same page boundaries, so their trimmed ranges are identical.
1664+
/// The intersection is the same as either column alone:
1665+
/// - RG0 page 0 ([0,49]): trimmed to [0, 49] (50 rows)
1666+
/// - RG0 page 1 ([50,99]): trimmed to [60, 69] (10 rows)
1667+
/// Expected: 60 rows. Both columns must remain aligned after trimming.
1668+
TEST_F(PageFilteredRowGroupReaderTest, BitmapTrimMultiColumnTest) {
1669+
std::string file_name = dir_->Str() + "/trim_multi_col.parquet";
1670+
auto data = MakeTwoColumnData(200);
1671+
WriteTestFile(file_name, data, /*write_batch_size=*/50, /*max_row_group_length=*/100);
1672+
1673+
RoaringBitmap32 bitmap;
1674+
bitmap.AddRange(0, 10);
1675+
bitmap.AddRange(45, 50);
1676+
bitmap.AddRange(60, 70);
1677+
1678+
std::map<std::string, std::string> options;
1679+
options[PARQUET_READ_BITMAP_ROW_RANGE_REFINING_STRATEGY] = "trim";
1680+
1681+
auto read_schema =
1682+
arrow::schema({arrow::field("a", arrow::int32()), arrow::field("b", arrow::int32())});
1683+
std::shared_ptr<arrow::ChunkedArray> result;
1684+
ReadWithPredicateAndBitmapImpl(file_name, read_schema, /*predicate=*/nullptr, bitmap, &result,
1685+
options);
1686+
ASSERT_TRUE(result);
1687+
ASSERT_EQ(60, result->length());
1688+
1689+
auto flat = arrow::Concatenate(result->chunks()).ValueOrDie();
1690+
auto struct_arr = std::dynamic_pointer_cast<arrow::StructArray>(flat);
1691+
ASSERT_TRUE(struct_arr);
1692+
auto a_arr = std::dynamic_pointer_cast<arrow::Int32Array>(struct_arr->field(0));
1693+
auto b_arr = std::dynamic_pointer_cast<arrow::Int32Array>(struct_arr->field(1));
1694+
ASSERT_TRUE(a_arr);
1695+
ASSERT_TRUE(b_arr);
1696+
// RG0 page 0 trimmed to [0, 49]
1697+
for (int32_t i = 0; i < 50; ++i) {
1698+
ASSERT_EQ(i, a_arr->Value(i));
1699+
ASSERT_EQ(i * 10, b_arr->Value(i));
1700+
}
1701+
// RG0 page 1 trimmed to [60, 69]
1702+
for (int32_t i = 0; i < 10; ++i) {
1703+
ASSERT_EQ(60 + i, a_arr->Value(50 + i));
1704+
ASSERT_EQ((60 + i) * 10, b_arr->Value(50 + i));
1705+
}
1706+
}
1707+
16181708
} // namespace paimon::parquet::test

‎src/paimon/format/parquet/parquet_file_batch_reader_test.cpp‎

Lines changed: 21 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -192,18 +192,24 @@ class ParquetFileBatchReaderTest : public ::testing::Test,
192192
ASSERT_OK(out->Close());
193193
}
194194

195+
std::map<std::string, std::string> GetDefaultOptions() const {
196+
std::map<std::string, std::string> options;
197+
options[PARQUET_READ_ENABLE_PAGE_INDEX_FILTER] = "true";
198+
options[PARQUET_READ_BITMAP_ROW_RANGE_REFINING_STRATEGY] = "none";
199+
return options;
200+
}
195201
std::unique_ptr<ParquetFileBatchReader> PrepareParquetFileBatchReader(
196202
const std::string& file_name, const std::shared_ptr<arrow::Schema>& read_schema,
197203
const std::shared_ptr<Predicate>& predicate,
198204
const std::optional<RoaringBitmap32>& selection_bitmap, int32_t batch_size,
199-
bool enable_page_level_filter = false) const {
205+
std::map<std::string, std::string> options = {}) const {
206+
if (options.empty()) {
207+
options = GetDefaultOptions();
208+
}
200209
EXPECT_OK_AND_ASSIGN(auto input_stream, fs_->Open(file_name));
201210
auto length = fs_->GetFileStatus(file_name).value()->GetLen();
202211
auto in_stream =
203212
std::make_unique<ArrowInputStreamAdapter>(std::move(input_stream), pool_, length);
204-
std::map<std::string, std::string> options;
205-
options[PARQUET_READ_ENABLE_PAGE_INDEX_FILTER] =
206-
enable_page_level_filter ? "true" : "false";
207213
return PrepareParquetFileBatchReader(std::move(in_stream), options, read_schema, predicate,
208214
selection_bitmap, batch_size);
209215
}
@@ -784,9 +790,10 @@ TEST_F(ParquetFileBatchReaderTest, TestBitmapPagePushDownWithMultiRowGroups) {
784790
/*enable_dictionary=*/true,
785791
/*max_row_group_length=*/6);
786792

793+
std::map<std::string, std::string> options = {{PARQUET_READ_ENABLE_PAGE_INDEX_FILTER, "true"}};
787794
auto parquet_batch_reader =
788795
PrepareParquetFileBatchReader(file_path_, arrow_schema, /*predicate=*/nullptr, bitmap,
789-
/*batch_size=*/12, /*enable_page_level_filter=*/true);
796+
/*batch_size=*/12, options);
790797

791798
ASSERT_OK_AND_ASSIGN(
792799
std::shared_ptr<arrow::ChunkedArray> result_array,
@@ -876,6 +883,7 @@ TEST_F(ParquetFileBatchReaderTest, TestPredicateAndBitmapPagePushDown) {
876883
WriteArray(file_path_, src_array, arrow_schema, /*write_batch_size=*/1024,
877884
/*enable_dictionary=*/true,
878885
/*max_row_group_length=*/256);
886+
std::map<std::string, std::string> options = {{PARQUET_READ_ENABLE_PAGE_INDEX_FILTER, "true"}};
879887
{
880888
// simple case
881889
std::optional<RoaringBitmap32> bitmap = RoaringBitmap32::From({100, 400, 600});
@@ -888,7 +896,7 @@ TEST_F(ParquetFileBatchReaderTest, TestPredicateAndBitmapPagePushDown) {
888896
FieldType::INT, Literal(600))}));
889897
auto parquet_batch_reader =
890898
PrepareParquetFileBatchReader(file_path_, arrow_schema, predicate, bitmap,
891-
/*batch_size=*/length, /*enable_page_level_filter=*/true);
899+
/*batch_size=*/length, options);
892900
ASSERT_OK_AND_ASSIGN(
893901
std::shared_ptr<arrow::ChunkedArray> result_array,
894902
paimon::test::ReadResultCollector::CollectResult(parquet_batch_reader.get()));
@@ -905,7 +913,7 @@ TEST_F(ParquetFileBatchReaderTest, TestPredicateAndBitmapPagePushDown) {
905913
FieldType::INT, Literal(800));
906914
auto parquet_batch_reader =
907915
PrepareParquetFileBatchReader(file_path_, arrow_schema, predicate, bitmap,
908-
/*batch_size=*/length, /*enable_page_level_filter=*/true);
916+
/*batch_size=*/length, options);
909917
ASSERT_OK_AND_ASSIGN(
910918
std::shared_ptr<arrow::ChunkedArray> result_array,
911919
paimon::test::ReadResultCollector::CollectResult(parquet_batch_reader.get()));
@@ -1141,9 +1149,9 @@ TEST_F(ParquetFileBatchReaderTest, TestRowMappingSimple) {
11411149
PredicateBuilder::Between(/*field_index=*/0, /*field_name=*/"f0",
11421150
FieldType::INT, Literal(5), Literal(6))}));
11431151

1152+
std::map<std::string, std::string> options = {{PARQUET_READ_ENABLE_PAGE_INDEX_FILTER, "true"}};
11441153
auto parquet_batch_reader = PrepareParquetFileBatchReader(
1145-
file_path_, arrow_schema, /*predicate=*/predicate, std::nullopt, /*batch_size=*/2,
1146-
/*enable_page_level_filter=*/true);
1154+
file_path_, arrow_schema, /*predicate=*/predicate, std::nullopt, /*batch_size=*/2, options);
11471155

11481156
ASSERT_NOK(parquet_batch_reader->GetPreviousBatchFileRowId(0));
11491157
ASSERT_OK_AND_ASSIGN(
@@ -1207,10 +1215,9 @@ TEST_F(ParquetFileBatchReaderTest, TestRowMappingFullyAndPartially) {
12071215
FieldType::INT, Literal(6)),
12081216
PredicateBuilder::Equal(/*field_index=*/0, /*field_name=*/"f0",
12091217
FieldType::INT, Literal(8))}));
1210-
1218+
std::map<std::string, std::string> options = {{PARQUET_READ_ENABLE_PAGE_INDEX_FILTER, "true"}};
12111219
auto parquet_batch_reader = PrepareParquetFileBatchReader(
1212-
file_path_, arrow_schema, /*predicate=*/predicate, std::nullopt, /*batch_size=*/3,
1213-
/*enable_page_level_filter=*/true);
1220+
file_path_, arrow_schema, /*predicate=*/predicate, std::nullopt, /*batch_size=*/3, options);
12141221

12151222
ASSERT_NOK(parquet_batch_reader->GetPreviousBatchFileRowId(0));
12161223
ASSERT_OK_AND_ASSIGN(
@@ -1243,9 +1250,9 @@ TEST_F(ParquetFileBatchReaderTest, TestRowMappingSetReadSchemaTwice) {
12431250
PredicateBuilder::Between(/*field_index=*/0, /*field_name=*/"f0",
12441251
FieldType::INT, Literal(6), Literal(7))}));
12451252

1253+
std::map<std::string, std::string> options = {{PARQUET_READ_ENABLE_PAGE_INDEX_FILTER, "true"}};
12461254
auto parquet_batch_reader = PrepareParquetFileBatchReader(
1247-
file_path_, arrow_schema, /*predicate=*/predicate, std::nullopt, /*batch_size=*/3,
1248-
/*enable_page_level_filter=*/true);
1255+
file_path_, arrow_schema, /*predicate=*/predicate, std::nullopt, /*batch_size=*/3, options);
12491256

12501257
ASSERT_NOK(parquet_batch_reader->GetPreviousBatchFileRowId(0));
12511258
ASSERT_OK_AND_ASSIGN(

0 commit comments

Comments
 (0)