Skip to content

Commit 819b3e3

Browse files
zhangfengcdtzhangfengcdt
committed
Merge remote-tracking branch 'apache/master' into feature-apache-parquet-2417-geospatial
2 parents af23c11 + 66e0c4e commit 819b3e3

8 files changed

Lines changed: 255 additions & 5 deletions

File tree

‎parquet-column/src/main/java/org/apache/parquet/schema/LogicalTypeAnnotation.java‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,14 @@ protected LogicalTypeAnnotation fromString(List<String> params) {
6060
return listType();
6161
}
6262
},
63+
VARIANT {
64+
@Override
65+
protected LogicalTypeAnnotation fromString(List<String> params) {
66+
Preconditions.checkArgument(
67+
params.size() == 1, "Expecting only spec version for variant annotation args: %s", params);
68+
return variantType(Byte.parseByte(params.get(0)));
69+
}
70+
},
6371
STRING {
6472
@Override
6573
protected LogicalTypeAnnotation fromString(List<String> params) {
@@ -289,6 +297,10 @@ public static ListLogicalTypeAnnotation listType() {
289297
return ListLogicalTypeAnnotation.INSTANCE;
290298
}
291299

300+
public static VariantLogicalTypeAnnotation variantType(byte specVersion) {
301+
return new VariantLogicalTypeAnnotation(specVersion);
302+
}
303+
292304
public static EnumLogicalTypeAnnotation enumType() {
293305
return EnumLogicalTypeAnnotation.INSTANCE;
294306
}
@@ -1164,6 +1176,49 @@ public int hashCode() {
11641176
}
11651177
}
11661178

1179+
public static class VariantLogicalTypeAnnotation extends LogicalTypeAnnotation {
1180+
private byte specVersion;
1181+
1182+
private VariantLogicalTypeAnnotation(byte specVersion) {
1183+
this.specVersion = specVersion;
1184+
}
1185+
1186+
@Override
1187+
public OriginalType toOriginalType() {
1188+
// No OriginalType for Variant
1189+
return null;
1190+
}
1191+
1192+
@Override
1193+
public <T> Optional<T> accept(LogicalTypeAnnotationVisitor<T> logicalTypeAnnotationVisitor) {
1194+
return logicalTypeAnnotationVisitor.visit(this);
1195+
}
1196+
1197+
@Override
1198+
LogicalTypeToken getType() {
1199+
return LogicalTypeToken.VARIANT;
1200+
}
1201+
1202+
public byte getSpecVersion() {
1203+
return this.specVersion;
1204+
}
1205+
1206+
@Override
1207+
protected String typeParametersAsString() {
1208+
return "(" + specVersion + ")";
1209+
}
1210+
1211+
@Override
1212+
public boolean equals(Object obj) {
1213+
if (!(obj instanceof VariantLogicalTypeAnnotation)) {
1214+
return false;
1215+
}
1216+
1217+
VariantLogicalTypeAnnotation other = (VariantLogicalTypeAnnotation) obj;
1218+
return specVersion == other.specVersion;
1219+
}
1220+
}
1221+
11671222
public static class GeometryLogicalTypeAnnotation extends LogicalTypeAnnotation {
11681223
private final String crs;
11691224

@@ -1310,6 +1365,10 @@ default Optional<T> visit(ListLogicalTypeAnnotation listLogicalType) {
13101365
return empty();
13111366
}
13121367

1368+
default Optional<T> visit(VariantLogicalTypeAnnotation variantLogicalType) {
1369+
return empty();
1370+
}
1371+
13131372
default Optional<T> visit(EnumLogicalTypeAnnotation enumLogicalType) {
13141373
return empty();
13151374
}

‎parquet-column/src/main/java/org/apache/parquet/schema/MessageTypeParser.java‎

Lines changed: 29 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -118,12 +118,36 @@ private static void addGroupType(Tokenizer st, Repetition r, GroupBuilder<?> bui
118118
String name = st.nextToken();
119119

120120
// Read annotation, if any.
121+
String annotation = null;
121122
t = st.nextToken();
122-
OriginalType originalType = null;
123123
if (t.equalsIgnoreCase("(")) {
124-
originalType = OriginalType.valueOf(st.nextToken());
125-
childBuilder.as(originalType);
126-
check(st.nextToken(), ")", "original type ended by )", st);
124+
t = st.nextToken();
125+
if (isLogicalType(t)) {
126+
LogicalTypeAnnotation.LogicalTypeToken logicalType = LogicalTypeAnnotation.LogicalTypeToken.valueOf(t);
127+
t = st.nextToken();
128+
List<String> tokens = new ArrayList<>();
129+
if ("(".equals(t)) {
130+
while (!")".equals(t)) {
131+
if (!(",".equals(t) || "(".equals(t) || ")".equals(t))) {
132+
tokens.add(t);
133+
}
134+
t = st.nextToken();
135+
}
136+
t = st.nextToken();
137+
}
138+
139+
LogicalTypeAnnotation logicalTypeAnnotation = logicalType.fromString(tokens);
140+
childBuilder.as(logicalTypeAnnotation);
141+
annotation = logicalTypeAnnotation.toString();
142+
} else {
143+
// Try to parse as OriginalType
144+
OriginalType originalType = OriginalType.valueOf(t);
145+
childBuilder.as(originalType);
146+
annotation = originalType.toString();
147+
t = st.nextToken();
148+
}
149+
150+
check(t, ")", "logical type ended by )", st);
127151
t = st.nextToken();
128152
}
129153
if (t.equals("=")) {
@@ -134,7 +158,7 @@ private static void addGroupType(Tokenizer st, Repetition r, GroupBuilder<?> bui
134158
addGroupTypeFields(t, st, childBuilder);
135159
} catch (IllegalArgumentException e) {
136160
throw new IllegalArgumentException(
137-
"problem reading type: type = group, name = " + name + ", original type = " + originalType, e);
161+
"problem reading type: type = group, name = " + name + ", annotation = " + annotation, e);
138162
}
139163

140164
childBuilder.named(name);

‎parquet-column/src/test/java/org/apache/parquet/parser/TestParquetParser.java‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,7 @@
5555
import static org.junit.Assert.assertEquals;
5656

5757
import org.apache.parquet.schema.GroupType;
58+
import org.apache.parquet.schema.LogicalTypeAnnotation;
5859
import org.apache.parquet.schema.MessageType;
5960
import org.apache.parquet.schema.MessageTypeParser;
6061
import org.apache.parquet.schema.OriginalType;
@@ -447,4 +448,30 @@ public void testEmbeddedAnnotations() {
447448
MessageType reparsed = MessageTypeParser.parseMessageType(parsed.toString());
448449
assertEquals(expected, reparsed);
449450
}
451+
452+
@Test
453+
public void testVARIANTAnnotation() {
454+
String message = "message Message {\n"
455+
+ " required group aVariant (VARIANT(2)) {\n"
456+
+ " required binary metadata;\n"
457+
+ " required binary value;\n"
458+
+ " }\n"
459+
+ "}\n";
460+
461+
MessageType expected = buildMessage()
462+
.requiredGroup()
463+
.as(LogicalTypeAnnotation.variantType((byte) 2))
464+
.required(BINARY)
465+
.named("metadata")
466+
.required(BINARY)
467+
.named("value")
468+
.named("aVariant")
469+
.named("Message");
470+
471+
MessageType parsed = parseMessageType(message);
472+
473+
assertEquals(expected, parsed);
474+
MessageType reparsed = parseMessageType(parsed.toString());
475+
assertEquals(expected, reparsed);
476+
}
450477
}

‎parquet-column/src/test/java/org/apache/parquet/schema/TestTypeBuilders.java‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,7 @@
5050
import static org.apache.parquet.schema.Type.Repetition.OPTIONAL;
5151
import static org.apache.parquet.schema.Type.Repetition.REPEATED;
5252
import static org.apache.parquet.schema.Type.Repetition.REQUIRED;
53+
import static org.junit.Assert.assertEquals;
5354

5455
import java.util.ArrayList;
5556
import java.util.List;
@@ -1414,6 +1415,52 @@ public void testTimestampLogicalTypeWithUTCParameter() {
14141415
Assert.assertEquals(nonUtcMicrosExpected, nonUtcMicrosActual);
14151416
}
14161417

1418+
@Test
1419+
public void testVariantLogicalType() {
1420+
byte specVersion = 1;
1421+
String name = "variant_field";
1422+
GroupType variantExpected = new GroupType(
1423+
REQUIRED,
1424+
name,
1425+
LogicalTypeAnnotation.variantType(specVersion),
1426+
new PrimitiveType(REQUIRED, BINARY, "metadata"),
1427+
new PrimitiveType(REQUIRED, BINARY, "value"));
1428+
1429+
GroupType variantActual = Types.buildGroup(REQUIRED)
1430+
.addFields(
1431+
Types.required(BINARY).named("metadata"),
1432+
Types.required(BINARY).named("value"))
1433+
.as(LogicalTypeAnnotation.variantType(specVersion))
1434+
.named(name);
1435+
1436+
assertEquals(variantExpected, variantActual);
1437+
}
1438+
1439+
@Test
1440+
public void testVariantLogicalTypeWithShredded() {
1441+
byte specVersion = 1;
1442+
String name = "variant_field";
1443+
GroupType variantExpected = new GroupType(
1444+
REQUIRED,
1445+
name,
1446+
LogicalTypeAnnotation.variantType(specVersion),
1447+
new PrimitiveType(REQUIRED, BINARY, "metadata"),
1448+
new PrimitiveType(OPTIONAL, BINARY, "value"),
1449+
new PrimitiveType(OPTIONAL, BINARY, "typed_value", LogicalTypeAnnotation.stringType()));
1450+
1451+
GroupType variantActual = Types.buildGroup(REQUIRED)
1452+
.addFields(
1453+
Types.required(BINARY).named("metadata"),
1454+
Types.optional(BINARY).named("value"),
1455+
Types.optional(BINARY)
1456+
.as(LogicalTypeAnnotation.stringType())
1457+
.named("typed_value"))
1458+
.as(LogicalTypeAnnotation.variantType(specVersion))
1459+
.named(name);
1460+
1461+
assertEquals(variantExpected, variantActual);
1462+
}
1463+
14171464
@Test(expected = IllegalArgumentException.class)
14181465
public void testDecimalLogicalTypeWithDeprecatedScaleMismatch() {
14191466
Types.required(BINARY)

‎parquet-column/src/test/java/org/apache/parquet/schema/TestTypeBuildersWithLogicalTypes.java‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,8 @@
4141
import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.INT96;
4242
import static org.apache.parquet.schema.Type.Repetition.REQUIRED;
4343
import static org.junit.Assert.assertEquals;
44+
import static org.junit.Assert.assertNull;
45+
import static org.junit.Assert.assertTrue;
4446

4547
import java.util.concurrent.Callable;
4648
import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName;
@@ -473,6 +475,59 @@ public void testFloat16LogicalType() {
473475
.toString());
474476
}
475477

478+
@Test
479+
public void testVariantLogicalType() {
480+
byte specVersion = 1;
481+
String name = "variant_field";
482+
GroupType variant = new GroupType(
483+
REQUIRED,
484+
name,
485+
LogicalTypeAnnotation.variantType(specVersion),
486+
Types.required(BINARY).named("metadata"),
487+
Types.required(BINARY).named("value"));
488+
489+
assertEquals(
490+
"required group variant_field (VARIANT(1)) {\n"
491+
+ " required binary metadata;\n"
492+
+ " required binary value;\n"
493+
+ "}",
494+
variant.toString());
495+
496+
LogicalTypeAnnotation annotation = variant.getLogicalTypeAnnotation();
497+
assertEquals(LogicalTypeAnnotation.LogicalTypeToken.VARIANT, annotation.getType());
498+
assertNull(annotation.toOriginalType());
499+
assertTrue(annotation instanceof LogicalTypeAnnotation.VariantLogicalTypeAnnotation);
500+
assertEquals(specVersion, ((LogicalTypeAnnotation.VariantLogicalTypeAnnotation) annotation).getSpecVersion());
501+
}
502+
503+
@Test
504+
public void testVariantLogicalTypeWithShredded() {
505+
byte specVersion = 1;
506+
507+
String name = "variant_field";
508+
GroupType variant = new GroupType(
509+
REQUIRED,
510+
name,
511+
LogicalTypeAnnotation.variantType(specVersion),
512+
Types.required(BINARY).named("metadata"),
513+
Types.optional(BINARY).named("value"),
514+
Types.optional(BINARY).as(LogicalTypeAnnotation.stringType()).named("typed_value"));
515+
516+
assertEquals(
517+
"required group variant_field (VARIANT(1)) {\n"
518+
+ " required binary metadata;\n"
519+
+ " optional binary value;\n"
520+
+ " optional binary typed_value (STRING);\n"
521+
+ "}",
522+
variant.toString());
523+
524+
LogicalTypeAnnotation annotation = variant.getLogicalTypeAnnotation();
525+
assertEquals(LogicalTypeAnnotation.LogicalTypeToken.VARIANT, annotation.getType());
526+
assertNull(annotation.toOriginalType());
527+
assertTrue(annotation instanceof LogicalTypeAnnotation.VariantLogicalTypeAnnotation);
528+
assertEquals(specVersion, ((LogicalTypeAnnotation.VariantLogicalTypeAnnotation) annotation).getSpecVersion());
529+
}
530+
476531
/**
477532
* A convenience method to avoid a large number of @Test(expected=...) tests
478533
*

‎parquet-format-structures/src/main/java/org/apache/parquet/format/LogicalTypes.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,12 @@ public static LogicalType DECIMAL(int scale, int precision) {
3232
return LogicalType.DECIMAL(new DecimalType(scale, precision));
3333
}
3434

35+
public static LogicalType VARIANT(byte specificationVersion) {
36+
VariantType type = new VariantType();
37+
type.setSpecification_version(specificationVersion);
38+
return LogicalType.VARIANT(type);
39+
}
40+
3541
public static final LogicalType UTF8 = LogicalType.STRING(new StringType());
3642
public static final LogicalType MAP = LogicalType.MAP(new MapType());
3743
public static final LogicalType LIST = LogicalType.LIST(new ListType());

‎parquet-hadoop/src/main/java/org/apache/parquet/format/converter/ParquetMetadataConverter.java‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -109,6 +109,7 @@
109109
import org.apache.parquet.format.Type;
110110
import org.apache.parquet.format.TypeDefinedOrder;
111111
import org.apache.parquet.format.Uncompressed;
112+
import org.apache.parquet.format.VariantType;
112113
import org.apache.parquet.format.XxHash;
113114
import org.apache.parquet.hadoop.metadata.BlockMetaData;
114115
import org.apache.parquet.hadoop.metadata.ColumnChunkMetaData;
@@ -545,6 +546,11 @@ public Optional<LogicalType> visit(LogicalTypeAnnotation.GeographyLogicalTypeAnn
545546
}
546547
return of(LogicalType.GEOGRAPHY(geographyType));
547548
}
549+
550+
@Override
551+
public Optional<LogicalType> visit(LogicalTypeAnnotation.VariantLogicalTypeAnnotation variantLogicalType) {
552+
return of(LogicalTypes.VARIANT(variantLogicalType.getSpecVersion()));
553+
}
548554
}
549555

550556
private void addRowGroup(
@@ -1322,6 +1328,9 @@ LogicalTypeAnnotation getLogicalTypeAnnotation(LogicalType type) {
13221328
} else {
13231329
return LogicalTypeAnnotation.geographyType(geography.getCrs(), null);
13241330
}
1331+
case VARIANT:
1332+
VariantType variant = type.getVARIANT();
1333+
return LogicalTypeAnnotation.variantType(variant.getSpecification_version());
13251334
default:
13261335
throw new RuntimeException("Unknown logical type " + type);
13271336
}

‎parquet-hadoop/src/test/java/org/apache/parquet/format/converter/TestParquetMetadataConverter.java‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@
4141
import static org.apache.parquet.schema.LogicalTypeAnnotation.timeType;
4242
import static org.apache.parquet.schema.LogicalTypeAnnotation.timestampType;
4343
import static org.apache.parquet.schema.LogicalTypeAnnotation.uuidType;
44+
import static org.apache.parquet.schema.LogicalTypeAnnotation.variantType;
4445
import static org.apache.parquet.schema.MessageTypeParser.parseMessageType;
4546
import static org.junit.Assert.assertEquals;
4647
import static org.junit.Assert.assertFalse;
@@ -1589,6 +1590,28 @@ public void testMapConvertedTypeReadWrite() throws Exception {
15891590
verifyMapMessageType(messageType, "map");
15901591
}
15911592

1593+
@Test
1594+
public void testVariantLogicalType() {
1595+
byte specVersion = 1;
1596+
MessageType expected = Types.buildMessage()
1597+
.requiredGroup()
1598+
.as(variantType(specVersion))
1599+
.required(PrimitiveTypeName.BINARY)
1600+
.named("metadata")
1601+
.required(PrimitiveTypeName.BINARY)
1602+
.named("value")
1603+
.named("v")
1604+
.named("example");
1605+
1606+
ParquetMetadataConverter parquetMetadataConverter = new ParquetMetadataConverter();
1607+
List<SchemaElement> parquetSchema = parquetMetadataConverter.toParquetSchema(expected);
1608+
MessageType schema = parquetMetadataConverter.fromParquetSchema(parquetSchema, null);
1609+
assertEquals(expected, schema);
1610+
LogicalTypeAnnotation logicalType = schema.getType("v").getLogicalTypeAnnotation();
1611+
assertEquals(LogicalTypeAnnotation.variantType(specVersion), logicalType);
1612+
assertEquals(specVersion, ((LogicalTypeAnnotation.VariantLogicalTypeAnnotation) logicalType).getSpecVersion());
1613+
}
1614+
15921615
private void verifyMapMessageType(final MessageType messageType, final String keyValueName) throws IOException {
15931616
Path file = new Path(temporaryFolder.newFolder("verifyMapMessageType").getPath(), keyValueName + ".parquet");
15941617

0 commit comments

Comments
 (0)