GH-3211: Implement Variant parquet reader (#3212)

Author: cashmand

parquet-avro/pom.xml

@@ -48,6 +48,11 @@
       <artifactId>parquet-common</artifactId>
       <version>${project.version}</version>
     </dependency>
+    <dependency>
+      <groupId>org.apache.parquet</groupId>
+      <artifactId>parquet-variant</artifactId>
+      <version>${project.version}</version>
+    </dependency>
     <dependency>
       <groupId>org.apache.avro</groupId>
       <artifactId>avro</artifactId>

parquet-avro/src/main/java/org/apache/parquet/avro/AvroIndexedRecordConverter.java

@@ -35,6 +35,7 @@
 import org.apache.parquet.io.api.GroupConverter;
 import org.apache.parquet.io.api.PrimitiveConverter;
 import org.apache.parquet.schema.GroupType;
+import org.apache.parquet.schema.LogicalTypeAnnotation;
 import org.apache.parquet.schema.MessageType;
 import org.apache.parquet.schema.Type;
 
@@ -168,7 +169,11 @@ private static Converter newConverter(Schema schema, Type type, GenericData mode
       case MAP:
         return new MapConverter(parent, type.asGroupType(), schema, model);
       case RECORD:
-        return new AvroIndexedRecordConverter<>(parent, type.asGroupType(), schema, model);
+        if (type.getLogicalTypeAnnotation() instanceof LogicalTypeAnnotation.VariantLogicalTypeAnnotation) {
+          return new AvroVariantConverter(parent, type.asGroupType(), schema, model);
+        } else {
+          return new AvroIndexedRecordConverter<>(parent, type.asGroupType(), schema, model);
+        }
       case STRING:
         return new AvroConverters.FieldStringConverter(parent);
       case UNION:

parquet-avro/src/main/java/org/apache/parquet/avro/AvroRecordConverter.java

@@ -62,6 +62,7 @@
 import org.apache.parquet.io.api.Converter;
 import org.apache.parquet.io.api.GroupConverter;
 import org.apache.parquet.schema.GroupType;
+import org.apache.parquet.schema.LogicalTypeAnnotation;
 import org.apache.parquet.schema.MessageType;
 import org.apache.parquet.schema.Type;
 import org.slf4j.Logger;
@@ -394,7 +395,11 @@ private static Converter newConverter(
         }
         return newStringConverter(schema, model, parent, validator);
       case RECORD:
-        return new AvroRecordConverter(parent, type.asGroupType(), schema, model, validator);
+        if (type.getLogicalTypeAnnotation() instanceof LogicalTypeAnnotation.VariantLogicalTypeAnnotation) {
+          return new AvroVariantConverter(parent, type.asGroupType(), schema, model);
+        } else {
+          return new AvroRecordConverter(parent, type.asGroupType(), schema, model, validator);
+        }
       case ENUM:
         return new AvroConverters.FieldEnumConverter(parent, schema, model);
       case ARRAY:

parquet-avro/src/main/java/org/apache/parquet/avro/AvroSchemaConverter.java

@@ -466,6 +466,16 @@ public Optional<Schema> visit(
                   LogicalTypeAnnotation.EnumLogicalTypeAnnotation enumLogicalType) {
                 return of(Schema.create(Schema.Type.STRING));
               }
+
+              @Override
+              public Optional<Schema> visit(
+                  LogicalTypeAnnotation.VariantLogicalTypeAnnotation variantLogicalType) {
+                String name = parquetGroupType.getName();
+                List<Schema.Field> fields = new ArrayList<>();
+                fields.add(new Schema.Field("metadata", Schema.create(Schema.Type.BYTES)));
+                fields.add(new Schema.Field("value", Schema.create(Schema.Type.BYTES)));
+                return of(Schema.createRecord(name, null, namespace(name, names), false, fields));
+              }
             })
             .orElseThrow(
                 () -> new UnsupportedOperationException("Cannot convert Parquet type " + parquetType));

parquet-avro/src/main/java/org/apache/parquet/avro/AvroVariantConverter.java

@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.parquet.avro;
+
+import java.nio.ByteBuffer;
+import java.util.function.Consumer;
+import org.apache.avro.Schema;
+import org.apache.avro.generic.GenericData;
+import org.apache.parquet.Preconditions;
+import org.apache.parquet.io.api.Converter;
+import org.apache.parquet.io.api.GroupConverter;
+import org.apache.parquet.schema.GroupType;
+import org.apache.parquet.variant.ImmutableMetadata;
+import org.apache.parquet.variant.VariantBuilder;
+import org.apache.parquet.variant.VariantConverters;
+
+/**
+ * Converter for Variant values.
+ */
+class AvroVariantConverter extends GroupConverter implements VariantConverters.ParentConverter<VariantBuilder> {
+  private final ParentValueContainer parent;
+  private final Schema avroSchema;
+  private final GenericData model;
+  private final int metadataPos;
+  private final int valuePos;
+  private final GroupConverter wrappedConverter;
+
+  private VariantBuilder builder = null;
+  private ImmutableMetadata metadata = null;
+
+  AvroVariantConverter(ParentValueContainer parent, GroupType variantGroup, Schema avroSchema, GenericData model) {
+    this.parent = parent;
+    this.avroSchema = avroSchema;
+    this.metadataPos = avroSchema.getField("metadata").pos();
+    this.valuePos = avroSchema.getField("value").pos();
+    this.model = model;
+    this.wrappedConverter = VariantConverters.newVariantConverter(variantGroup, this::setMetadata, this);
+  }
+
+  @Override
+  public void build(Consumer<VariantBuilder> consumer) {
+    Preconditions.checkState(builder != null, "Cannot build variant: builder has not been initialized");
+    consumer.accept(builder);
+  }
+
+  @Override
+  public Converter getConverter(int fieldIndex) {
+    return wrappedConverter.getConverter(fieldIndex);
+  }
+
+  @Override
+  public void start() {
+    wrappedConverter.start();
+  }
+
+  @Override
+  public void end() {
+    wrappedConverter.end();
+
+    Preconditions.checkState(metadata != null, "Cannot build variant: missing metadata");
+
+    builder.appendNullIfEmpty();
+
+    Object record = model.newRecord(null, avroSchema);
+    model.setField(record, "metadata", metadataPos, metadata.getEncodedBuffer());
+    model.setField(record, "value", valuePos, builder.encodedValue());
+    parent.add(record);
+
+    this.builder = null;
+  }
+
+  void setMetadata(ByteBuffer metadataBuffer) {
+    // If the metadata hasn't changed, we don't need to rebuild the map.
+    if (metadata == null || metadata.getEncodedBuffer() != metadataBuffer) {
+      this.metadata = new ImmutableMetadata(metadataBuffer);
+    }
+
+    this.builder = new VariantBuilder(metadata);
+  }
+}