Skip to content

Commit 172eab8

Browse files
nssalianjakelong95
authored andcommitted
Parquet: Fix variant metrics crash when value column has no stats (apache#16585)
* Parquet: Fix variant metrics crash when value column has no stats * Fixup after 16327
1 parent 9a47f92 commit 172eab8

2 files changed

Lines changed: 52 additions & 0 deletions

File tree

parquet/src/main/java/org/apache/iceberg/parquet/ParquetMetrics.java

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -477,6 +477,11 @@ public Iterable<ParquetVariantUtil.VariantMetrics> value(
477477
return typedResult;
478478
}
479479

480+
if (Iterables.isEmpty(valueResult)) {
481+
// missing value stats invalidate typed bounds
482+
return ImmutableList.of();
483+
}
484+
480485
ParquetVariantUtil.VariantMetrics valueMetrics = Iterables.getOnlyElement(valueResult);
481486
if (typedResult != null && valueMetrics.valueCount() == valueMetrics.nullCount()) {
482487
// all the variant-encoded values are null, so the typed stats can be used

parquet/src/test/java/org/apache/iceberg/parquet/TestVariantMetrics.java

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,14 +26,17 @@
2626
import java.util.Map;
2727
import java.util.Set;
2828
import java.util.UUID;
29+
import org.apache.hadoop.conf.Configuration;
2930
import org.apache.iceberg.Metrics;
31+
import org.apache.iceberg.MetricsConfig;
3032
import org.apache.iceberg.Schema;
3133
import org.apache.iceberg.data.GenericRecord;
3234
import org.apache.iceberg.data.Record;
3335
import org.apache.iceberg.data.parquet.InternalWriter;
3436
import org.apache.iceberg.inmemory.InMemoryOutputFile;
3537
import org.apache.iceberg.io.FileAppender;
3638
import org.apache.iceberg.io.OutputFile;
39+
import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
3740
import org.apache.iceberg.types.Conversions;
3841
import org.apache.iceberg.types.Type;
3942
import org.apache.iceberg.types.Types;
@@ -48,6 +51,10 @@
4851
import org.apache.iceberg.variants.VariantTestUtil;
4952
import org.apache.iceberg.variants.VariantValue;
5053
import org.apache.iceberg.variants.Variants;
54+
import org.apache.parquet.column.ParquetProperties;
55+
import org.apache.parquet.hadoop.ParquetFileWriter;
56+
import org.apache.parquet.hadoop.metadata.CompressionCodecName;
57+
import org.apache.parquet.schema.MessageType;
5158
import org.junit.jupiter.api.Test;
5259
import org.junit.jupiter.params.ParameterizedTest;
5360
import org.junit.jupiter.params.provider.FieldSource;
@@ -479,6 +486,46 @@ public void testShreddedObjectFieldTypeMismatch() throws IOException {
479486
.isEqualTo(Map.of(1, Types.LongType.get(), 2, Types.VariantType.get()));
480487
}
481488

489+
@Test
490+
public void testShreddedValueColumnWithEmptyStats() throws IOException {
491+
// typed bounds are dropped when value-column stats are missing on a shredded variant
492+
OutputFile out = new InMemoryOutputFile();
493+
GenericRecord record = GenericRecord.create(SCHEMA);
494+
495+
VariantShreddingFunction shredding =
496+
(id, name) -> ParquetVariantUtil.toParquetSchema(Variants.of((byte) 0));
497+
MessageType parquetSchema = ParquetSchemaUtil.convert(SCHEMA, "table", shredding);
498+
ParquetProperties props = ParquetProperties.builder().withStatisticsEnabled(false).build();
499+
500+
// Parquet.write() cannot disable stats on variant sub-columns (no field IDs)
501+
ParquetWriter<Record> writer =
502+
new ParquetWriter<>(
503+
new Configuration(),
504+
out,
505+
SCHEMA,
506+
parquetSchema,
507+
1024,
508+
ImmutableMap.of(),
509+
(s, m) -> InternalWriter.create(s.asStruct(), m),
510+
CompressionCodecName.SNAPPY,
511+
props,
512+
MetricsConfig.getDefault(),
513+
ParquetFileWriter.Mode.CREATE,
514+
null,
515+
false);
516+
517+
try (writer) {
518+
record.setField("id", 1L);
519+
record.setField("var", Variant.of(EMPTY, Variants.of((byte) 5)));
520+
writer.add(record);
521+
}
522+
523+
Metrics metrics = writer.metrics();
524+
assertThat(metrics.recordCount()).isEqualTo(1L);
525+
assertThat(metrics.lowerBounds()).doesNotContainKey(2);
526+
assertThat(metrics.upperBounds()).doesNotContainKey(2);
527+
}
528+
482529
private Metrics writeParquet(VariantShreddingFunction shredding, Variant... variants)
483530
throws IOException {
484531
OutputFile out = new InMemoryOutputFile();

0 commit comments

Comments
 (0)