Skip to content

Commit f71f485

Browse files
authored
fix: Handle unknown record_count better (#747)
Fixes #732 Only special-case ```record_count == 0;``` Let negative/unknown counts fall through to normal strict metrics evaluation.
1 parent 796efb3 commit f71f485

2 files changed

Lines changed: 43 additions & 1 deletion

File tree

src/iceberg/expression/strict_metrics_evaluator.cc

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -532,9 +532,16 @@ Result<std::unique_ptr<StrictMetricsEvaluator>> StrictMetricsEvaluator::Make(
532532
}
533533

534534
Result<bool> StrictMetricsEvaluator::Evaluate(const DataFile& data_file) const {
535-
if (data_file.record_count <= 0) {
535+
if (data_file.record_count == 0) {
536536
return kRowsMustMatch;
537537
}
538+
// Only -1 is a valid sentinel for an unknown row count (set when writer metrics omit
539+
// the count); any value below -1 is invalid metadata.
540+
if (data_file.record_count < -1) {
541+
return InvalidArgument("Invalid record count: {}", data_file.record_count);
542+
}
543+
// For -1, fall through to normal strict metrics evaluation rather than
544+
// assuming all rows must match.
538545
StrictMetricsVisitor visitor(data_file, *schema_);
539546
return Visit<bool, StrictMetricsVisitor>(expr_, visitor);
540547
}

src/iceberg/test/strict_metrics_evaluator_test.cc

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -536,6 +536,24 @@ class StrictMetricsEvaluatorMigratedTest : public StrictMetricsEvaluatorTest {
536536
return data_file;
537537
}
538538

539+
std::shared_ptr<DataFile> MakeUnknownRecordCountFile() {
540+
auto data_file = std::make_shared<DataFile>();
541+
data_file->file_path = "unknown.parquet";
542+
data_file->file_format = FileFormatType::kParquet;
543+
// -1 is the sentinel for an unknown row count.
544+
data_file->record_count = -1;
545+
return data_file;
546+
}
547+
548+
std::shared_ptr<DataFile> MakeInvalidRecordCountFile() {
549+
auto data_file = std::make_shared<DataFile>();
550+
data_file->file_path = "invalid.parquet";
551+
data_file->file_format = FileFormatType::kParquet;
552+
// Anything below the -1 sentinel is invalid metadata.
553+
data_file->record_count = -2;
554+
return data_file;
555+
}
556+
539557
void ExpectShouldRead(const std::shared_ptr<Expression>& expr, bool expected,
540558
std::shared_ptr<DataFile> file = nullptr,
541559
bool case_sensitive = true) {
@@ -650,6 +668,23 @@ TEST_F(StrictMetricsEvaluatorMigratedTest, ZeroRecordFile) {
650668
}
651669
}
652670

671+
TEST_F(StrictMetricsEvaluatorMigratedTest, UnknownRecordCountFile) {
672+
// A file with an unknown row count (record_count == -1) must not be treated as if
673+
// every row matches the predicate. AlwaysFalse can never match all rows.
674+
auto unknown_record_count_file = MakeUnknownRecordCountFile();
675+
ExpectShouldRead(Expressions::AlwaysFalse(), false, unknown_record_count_file);
676+
}
677+
678+
TEST_F(StrictMetricsEvaluatorMigratedTest, InvalidRecordCountFile) {
679+
// A record count below the -1 unknown sentinel is invalid and must error out.
680+
auto invalid_record_count_file = MakeInvalidRecordCountFile();
681+
ICEBERG_UNWRAP_OR_FAIL(auto evaluator, StrictMetricsEvaluator::Make(
682+
Expressions::AlwaysFalse(), schema_, true));
683+
auto result = evaluator->Evaluate(*invalid_record_count_file);
684+
ASSERT_FALSE(result.has_value());
685+
EXPECT_EQ(result.error().kind, ErrorKind::kInvalidArgument);
686+
}
687+
653688
TEST_F(StrictMetricsEvaluatorMigratedTest, Not) {
654689
ExpectShouldRead(
655690
Expressions::Not(Expressions::LessThan("id", Literal::Long(kIntMinValue - 25))),

0 commit comments

Comments
 (0)