Skip to content

Commit 6f50a39

Browse files
WZhuozhjwpku
WZhuo
and
zhjwpku
authored
feat: add uniqueValue optimization for notEq/notIn in InclusiveMetricsEvaluator (#754)
When a column has a single unique value (no nulls, no NaNs, and lower bound == upper bound), notEq(X) and notIn({...X...}) can prune files where that unique value matches the predicate, returning ROWS_CANNOT_MATCH. Previously these always returned ROWS_MIGHT_MATCH, never pruning. This mirrors the upstream Java InclusiveMetricsEvaluator.uniqueValue() logic. The StrictMetricsEvaluator is unchanged: its existing bounds-based notEq/notIn logic already covers the single-value case, matching upstream Java which has no uniqueValue helper there. Ports the corresponding testNotEqWithSingleValue/testNotInWithSingleValue cases from the Java test suite. --------- Co-authored-by: Junwang Zhao <zhjwpku@gmail.com>
1 parent 9a2e21d commit 6f50a39

2 files changed

Lines changed: 196 additions & 1 deletion

File tree

‎src/iceberg/expression/inclusive_metrics_evaluator.cc‎

Lines changed: 42 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -223,6 +223,13 @@ class InclusiveMetricsVisitor : public BoundVisitor<bool> {
223223
Result<bool> NotEq(const std::shared_ptr<Bound>& expr, const Literal& lit) override {
224224
// because the bounds are not necessarily a min or max value, this cannot be answered
225225
// using them. notEq(col, X) with (X, Y) doesn't guarantee that X is a value in col.
226+
// However, when min == max and the file has no nulls or NaN values, we can safely
227+
// prune if that value equals the literal.
228+
ICEBERG_ASSIGN_OR_RAISE(auto value, UniqueValue(expr));
229+
if (value.has_value() && value.value() == lit) {
230+
return kRowCannotMatch;
231+
}
232+
226233
return kRowsMightMatch;
227234
}
228235

@@ -271,7 +278,13 @@ class InclusiveMetricsVisitor : public BoundVisitor<bool> {
271278
const BoundSetPredicate::LiteralSet& literal_set) override {
272279
// because the bounds are not necessarily a min or max value, this cannot be answered
273280
// using them. notIn(col, {X, ...}) with (X, Y) doesn't guarantee that X is a value in
274-
// col.
281+
// col. However, when min == max and the file has no nulls or NaN values, we can
282+
// safely prune if that value is in the exclusion set.
283+
ICEBERG_ASSIGN_OR_RAISE(auto value, UniqueValue(expr));
284+
if (value.has_value() && literal_set.contains(value.value())) {
285+
return kRowCannotMatch;
286+
}
287+
275288
return kRowsMightMatch;
276289
}
277290

@@ -416,6 +429,34 @@ class InclusiveMetricsVisitor : public BoundVisitor<bool> {
416429
// TODO(xiao.dong) handle extract lower and upper bounds
417430
}
418431

432+
/// Returns the column's single value if all rows contain the same value. Defined as a
433+
/// column with no nulls, no NaNs, and lower bound equals upper bound. Returns
434+
/// std::nullopt otherwise.
435+
Result<std::optional<Literal>> UniqueValue(const std::shared_ptr<Bound>& expr) {
436+
int32_t id = expr->reference()->field().field_id();
437+
if (MayContainNull(id)) {
438+
return std::nullopt;
439+
}
440+
441+
ICEBERG_ASSIGN_OR_RAISE(auto lower, LowerBound(expr));
442+
ICEBERG_ASSIGN_OR_RAISE(auto upper, UpperBound(expr));
443+
if (!lower.has_value() || !upper.has_value() || lower->IsNull() || upper->IsNull() ||
444+
lower->IsNaN() || upper->IsNaN()) {
445+
return std::nullopt;
446+
}
447+
448+
auto nan_it = data_file_.nan_value_counts.find(id);
449+
if (nan_it != data_file_.nan_value_counts.cend() && nan_it->second != 0) {
450+
return std::nullopt;
451+
}
452+
453+
if (lower.value() != upper.value()) {
454+
return std::nullopt;
455+
}
456+
457+
return lower;
458+
}
459+
419460
Result<std::optional<Literal>> ParseLowerBound(const BoundReference& ref) {
420461
int32_t id = ref.field().field_id();
421462
auto type = ref.type();

‎src/iceberg/test/inclusive_metrics_evaluator_test.cc‎

Lines changed: 154 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -945,4 +945,158 @@ TEST_F(InclusiveMetricsEvaluatorMigratedTest, IntegerNotInTest) {
945945
RunTest(Expressions::NotIn("id", ids), kRowsMightMatch, file1_);
946946
}
947947

948+
TEST_F(InclusiveMetricsEvaluatorMigratedTest, NotEqWithSingleValue) {
949+
// file has a range of values, cannot prune based on a single literal
950+
auto range_of_values = std::make_shared<DataFile>();
951+
range_of_values->file_path = "range_of_values";
952+
range_of_values->file_format = FileFormatType::kParquet;
953+
range_of_values->record_count = 10;
954+
range_of_values->value_counts = {{3, 10L}};
955+
range_of_values->null_value_counts = {{3, 0L}};
956+
range_of_values->nan_value_counts = {{3, 0L}};
957+
range_of_values->lower_bounds = {{3, Literal::String("aaa").Serialize().value()}};
958+
range_of_values->upper_bounds = {{3, Literal::String("zzz").Serialize().value()}};
959+
RunTest(Expressions::NotEqual("required", Literal::String("aaa")), kRowsMightMatch,
960+
range_of_values);
961+
962+
// file contains a single value (lower == upper) with no nulls/NaNs
963+
auto single_value = std::make_shared<DataFile>();
964+
single_value->file_path = "single_value";
965+
single_value->file_format = FileFormatType::kParquet;
966+
single_value->record_count = 10;
967+
single_value->value_counts = {{3, 10L}};
968+
single_value->null_value_counts = {{3, 0L}};
969+
single_value->nan_value_counts = {{3, 0L}};
970+
single_value->lower_bounds = {{3, Literal::String("abc").Serialize().value()}};
971+
single_value->upper_bounds = {{3, Literal::String("abc").Serialize().value()}};
972+
// single value equals the literal -> rows cannot match
973+
RunTest(Expressions::NotEqual("required", Literal::String("abc")), kRowCannotMatch,
974+
single_value);
975+
// single value differs from the literal -> rows might match
976+
RunTest(Expressions::NotEqual("required", Literal::String("def")), kRowsMightMatch,
977+
single_value);
978+
979+
// single value but the file has nulls, which satisfy the != predicate
980+
auto single_value_with_nulls = std::make_shared<DataFile>();
981+
single_value_with_nulls->file_path = "single_value_nulls";
982+
single_value_with_nulls->file_format = FileFormatType::kParquet;
983+
single_value_with_nulls->record_count = 10;
984+
single_value_with_nulls->value_counts = {{3, 10L}};
985+
single_value_with_nulls->null_value_counts = {{3, 2L}};
986+
single_value_with_nulls->nan_value_counts = {{3, 0L}};
987+
single_value_with_nulls->lower_bounds = {
988+
{3, Literal::String("abc").Serialize().value()}};
989+
single_value_with_nulls->upper_bounds = {
990+
{3, Literal::String("abc").Serialize().value()}};
991+
RunTest(Expressions::NotEqual("required", Literal::String("abc")), kRowsMightMatch,
992+
single_value_with_nulls);
993+
994+
// single value but the file has NaNs, which satisfy the != predicate
995+
auto single_value_with_nan = std::make_shared<DataFile>();
996+
single_value_with_nan->file_path = "single_value_nan";
997+
single_value_with_nan->file_format = FileFormatType::kParquet;
998+
single_value_with_nan->record_count = 10;
999+
single_value_with_nan->value_counts = {{9, 10L}};
1000+
single_value_with_nan->null_value_counts = {{9, 0L}};
1001+
single_value_with_nan->nan_value_counts = {{9, 2L}};
1002+
single_value_with_nan->lower_bounds = {{9, Literal::Float(5.0F).Serialize().value()}};
1003+
single_value_with_nan->upper_bounds = {{9, Literal::Float(5.0F).Serialize().value()}};
1004+
RunTest(Expressions::NotEqual("no_nans", Literal::Float(5.0F)), kRowsMightMatch,
1005+
single_value_with_nan);
1006+
1007+
// bounds are NaN -> unreliable, cannot prune
1008+
auto single_value_nan_bounds = std::make_shared<DataFile>();
1009+
single_value_nan_bounds->file_path = "single_value_nan_bounds";
1010+
single_value_nan_bounds->file_format = FileFormatType::kParquet;
1011+
single_value_nan_bounds->record_count = 10;
1012+
single_value_nan_bounds->value_counts = {{9, 10L}};
1013+
single_value_nan_bounds->null_value_counts = {{9, 0L}};
1014+
single_value_nan_bounds->nan_value_counts = {{9, 0L}};
1015+
single_value_nan_bounds->lower_bounds = {
1016+
{9, Literal::Float(kFloatNan).Serialize().value()}};
1017+
single_value_nan_bounds->upper_bounds = {
1018+
{9, Literal::Float(kFloatNan).Serialize().value()}};
1019+
RunTest(Expressions::NotEqual("no_nans", Literal::Float(5.0F)), kRowsMightMatch,
1020+
single_value_nan_bounds);
1021+
}
1022+
1023+
TEST_F(InclusiveMetricsEvaluatorMigratedTest, NotInWithSingleValue) {
1024+
// file has a range of values, cannot prune based on the exclusion set
1025+
auto range_of_values = std::make_shared<DataFile>();
1026+
range_of_values->file_path = "range_of_values";
1027+
range_of_values->file_format = FileFormatType::kParquet;
1028+
range_of_values->record_count = 10;
1029+
range_of_values->value_counts = {{3, 10L}};
1030+
range_of_values->null_value_counts = {{3, 0L}};
1031+
range_of_values->nan_value_counts = {{3, 0L}};
1032+
range_of_values->lower_bounds = {{3, Literal::String("aaa").Serialize().value()}};
1033+
range_of_values->upper_bounds = {{3, Literal::String("zzz").Serialize().value()}};
1034+
RunTest(
1035+
Expressions::NotIn("required", {Literal::String("aaa"), Literal::String("bbb")}),
1036+
kRowsMightMatch, range_of_values);
1037+
1038+
// file contains a single value (lower == upper) with no nulls/NaNs
1039+
auto single_value = std::make_shared<DataFile>();
1040+
single_value->file_path = "single_value";
1041+
single_value->file_format = FileFormatType::kParquet;
1042+
single_value->record_count = 10;
1043+
single_value->value_counts = {{3, 10L}};
1044+
single_value->null_value_counts = {{3, 0L}};
1045+
single_value->nan_value_counts = {{3, 0L}};
1046+
single_value->lower_bounds = {{3, Literal::String("abc").Serialize().value()}};
1047+
single_value->upper_bounds = {{3, Literal::String("abc").Serialize().value()}};
1048+
// single value is in the exclusion set -> rows cannot match
1049+
RunTest(
1050+
Expressions::NotIn("required", {Literal::String("abc"), Literal::String("def")}),
1051+
kRowCannotMatch, single_value);
1052+
// single value is not in the exclusion set -> rows might match
1053+
RunTest(
1054+
Expressions::NotIn("required", {Literal::String("def"), Literal::String("ghi")}),
1055+
kRowsMightMatch, single_value);
1056+
1057+
// single value but the file has nulls, which satisfy the notIn predicate
1058+
auto single_value_with_nulls = std::make_shared<DataFile>();
1059+
single_value_with_nulls->file_path = "single_value_nulls";
1060+
single_value_with_nulls->file_format = FileFormatType::kParquet;
1061+
single_value_with_nulls->record_count = 10;
1062+
single_value_with_nulls->value_counts = {{3, 10L}};
1063+
single_value_with_nulls->null_value_counts = {{3, 2L}};
1064+
single_value_with_nulls->nan_value_counts = {{3, 0L}};
1065+
single_value_with_nulls->lower_bounds = {
1066+
{3, Literal::String("abc").Serialize().value()}};
1067+
single_value_with_nulls->upper_bounds = {
1068+
{3, Literal::String("abc").Serialize().value()}};
1069+
RunTest(
1070+
Expressions::NotIn("required", {Literal::String("abc"), Literal::String("def")}),
1071+
kRowsMightMatch, single_value_with_nulls);
1072+
1073+
// single value but the file has NaNs, which satisfy the notIn predicate
1074+
auto single_value_with_nan = std::make_shared<DataFile>();
1075+
single_value_with_nan->file_path = "single_value_nan";
1076+
single_value_with_nan->file_format = FileFormatType::kParquet;
1077+
single_value_with_nan->record_count = 10;
1078+
single_value_with_nan->value_counts = {{9, 10L}};
1079+
single_value_with_nan->null_value_counts = {{9, 0L}};
1080+
single_value_with_nan->nan_value_counts = {{9, 2L}};
1081+
single_value_with_nan->lower_bounds = {{9, Literal::Float(5.0F).Serialize().value()}};
1082+
single_value_with_nan->upper_bounds = {{9, Literal::Float(5.0F).Serialize().value()}};
1083+
RunTest(Expressions::NotIn("no_nans", {Literal::Float(5.0F)}), kRowsMightMatch,
1084+
single_value_with_nan);
1085+
1086+
// bounds are NaN -> unreliable, cannot prune
1087+
auto single_value_nan_bounds = std::make_shared<DataFile>();
1088+
single_value_nan_bounds->file_path = "single_value_nan_bounds";
1089+
single_value_nan_bounds->file_format = FileFormatType::kParquet;
1090+
single_value_nan_bounds->record_count = 10;
1091+
single_value_nan_bounds->value_counts = {{9, 10L}};
1092+
single_value_nan_bounds->null_value_counts = {{9, 0L}};
1093+
single_value_nan_bounds->nan_value_counts = {{9, 0L}};
1094+
single_value_nan_bounds->lower_bounds = {
1095+
{9, Literal::Float(kFloatNan).Serialize().value()}};
1096+
single_value_nan_bounds->upper_bounds = {
1097+
{9, Literal::Float(kFloatNan).Serialize().value()}};
1098+
RunTest(Expressions::NotIn("no_nans", {Literal::Float(5.0F)}), kRowsMightMatch,
1099+
single_value_nan_bounds);
1100+
}
1101+
9481102
} // namespace iceberg

0 commit comments

Comments
 (0)