|
26 | 26 | import java.util.Map; |
27 | 27 | import java.util.Set; |
28 | 28 | import java.util.UUID; |
| 29 | +import org.apache.hadoop.conf.Configuration; |
29 | 30 | import org.apache.iceberg.Metrics; |
| 31 | +import org.apache.iceberg.MetricsConfig; |
30 | 32 | import org.apache.iceberg.Schema; |
31 | 33 | import org.apache.iceberg.data.GenericRecord; |
32 | 34 | import org.apache.iceberg.data.Record; |
33 | 35 | import org.apache.iceberg.data.parquet.InternalWriter; |
34 | 36 | import org.apache.iceberg.inmemory.InMemoryOutputFile; |
35 | 37 | import org.apache.iceberg.io.FileAppender; |
36 | 38 | import org.apache.iceberg.io.OutputFile; |
| 39 | +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; |
37 | 40 | import org.apache.iceberg.types.Conversions; |
38 | 41 | import org.apache.iceberg.types.Type; |
39 | 42 | import org.apache.iceberg.types.Types; |
|
48 | 51 | import org.apache.iceberg.variants.VariantTestUtil; |
49 | 52 | import org.apache.iceberg.variants.VariantValue; |
50 | 53 | import org.apache.iceberg.variants.Variants; |
| 54 | +import org.apache.parquet.column.ParquetProperties; |
| 55 | +import org.apache.parquet.hadoop.ParquetFileWriter; |
| 56 | +import org.apache.parquet.hadoop.metadata.CompressionCodecName; |
| 57 | +import org.apache.parquet.schema.MessageType; |
51 | 58 | import org.junit.jupiter.api.Test; |
52 | 59 | import org.junit.jupiter.params.ParameterizedTest; |
53 | 60 | import org.junit.jupiter.params.provider.FieldSource; |
@@ -479,6 +486,46 @@ public void testShreddedObjectFieldTypeMismatch() throws IOException { |
479 | 486 | .isEqualTo(Map.of(1, Types.LongType.get(), 2, Types.VariantType.get())); |
480 | 487 | } |
481 | 488 |
|
| 489 | + @Test |
| 490 | + public void testShreddedValueColumnWithEmptyStats() throws IOException { |
| 491 | + // typed bounds are dropped when value-column stats are missing on a shredded variant |
| 492 | + OutputFile out = new InMemoryOutputFile(); |
| 493 | + GenericRecord record = GenericRecord.create(SCHEMA); |
| 494 | + |
| 495 | + VariantShreddingFunction shredding = |
| 496 | + (id, name) -> ParquetVariantUtil.toParquetSchema(Variants.of((byte) 0)); |
| 497 | + MessageType parquetSchema = ParquetSchemaUtil.convert(SCHEMA, "table", shredding); |
| 498 | + ParquetProperties props = ParquetProperties.builder().withStatisticsEnabled(false).build(); |
| 499 | + |
| 500 | + // Parquet.write() cannot disable stats on variant sub-columns (no field IDs) |
| 501 | + ParquetWriter<Record> writer = |
| 502 | + new ParquetWriter<>( |
| 503 | + new Configuration(), |
| 504 | + out, |
| 505 | + SCHEMA, |
| 506 | + parquetSchema, |
| 507 | + 1024, |
| 508 | + ImmutableMap.of(), |
| 509 | + (s, m) -> InternalWriter.create(s.asStruct(), m), |
| 510 | + CompressionCodecName.SNAPPY, |
| 511 | + props, |
| 512 | + MetricsConfig.getDefault(), |
| 513 | + ParquetFileWriter.Mode.CREATE, |
| 514 | + null, |
| 515 | + false); |
| 516 | + |
| 517 | + try (writer) { |
| 518 | + record.setField("id", 1L); |
| 519 | + record.setField("var", Variant.of(EMPTY, Variants.of((byte) 5))); |
| 520 | + writer.add(record); |
| 521 | + } |
| 522 | + |
| 523 | + Metrics metrics = writer.metrics(); |
| 524 | + assertThat(metrics.recordCount()).isEqualTo(1L); |
| 525 | + assertThat(metrics.lowerBounds()).doesNotContainKey(2); |
| 526 | + assertThat(metrics.upperBounds()).doesNotContainKey(2); |
| 527 | + } |
| 528 | + |
482 | 529 | private Metrics writeParquet(VariantShreddingFunction shredding, Variant... variants) |
483 | 530 | throws IOException { |
484 | 531 | OutputFile out = new InMemoryOutputFile(); |
|
0 commit comments