Skip to content

Commit 146f096

Browse files
wgtmacwgtmac
committed
PARQUET-2249: Add column order for IEEE 754 total order
1 parent 22a23d0 commit 146f096

8 files changed

Lines changed: 290 additions & 36 deletions

File tree

‎parquet-column/src/main/java/org/apache/parquet/schema/ColumnOrder.java‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,11 +36,16 @@ public enum ColumnOrderName {
3636
/**
3737
* Type defined order meaning that the comparison order of the elements are based on its type.
3838
*/
39-
TYPE_DEFINED_ORDER
39+
TYPE_DEFINED_ORDER,
40+
/**
41+
* The column order is defined by the IEEE 754 standard.
42+
*/
43+
IEEE_754_TOTAL_ORDER,
4044
}
4145

4246
private static final ColumnOrder UNDEFINED_COLUMN_ORDER = new ColumnOrder(ColumnOrderName.UNDEFINED);
4347
private static final ColumnOrder TYPE_DEFINED_COLUMN_ORDER = new ColumnOrder(ColumnOrderName.TYPE_DEFINED_ORDER);
48+
private static final ColumnOrder IEEE_754_TOTAL_ORDER = new ColumnOrder(ColumnOrderName.IEEE_754_TOTAL_ORDER);
4449

4550
/**
4651
* @return a {@link ColumnOrder} instance representing an undefined order
@@ -58,6 +63,14 @@ public static ColumnOrder typeDefined() {
5863
return TYPE_DEFINED_COLUMN_ORDER;
5964
}
6065

66+
/**
67+
* @return a {@link ColumnOrder} instance representing an IEEE 754 total order
68+
* @see ColumnOrderName#IEEE_754_TOTAL_ORDER
69+
*/
70+
public static ColumnOrder ieee754TotalOrder() {
71+
return IEEE_754_TOTAL_ORDER;
72+
}
73+
6174
private final ColumnOrderName columnOrderName;
6275

6376
private ColumnOrder(ColumnOrderName columnOrderName) {

‎parquet-column/src/main/java/org/apache/parquet/schema/LogicalTypeAnnotation.java‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020

2121
import static java.util.Arrays.asList;
2222
import static java.util.Optional.empty;
23+
import static org.apache.parquet.schema.ColumnOrder.ColumnOrderName.IEEE_754_TOTAL_ORDER;
2324
import static org.apache.parquet.schema.ColumnOrder.ColumnOrderName.TYPE_DEFINED_ORDER;
2425
import static org.apache.parquet.schema.ColumnOrder.ColumnOrderName.UNDEFINED;
2526
import static org.apache.parquet.schema.PrimitiveStringifier.TIMESTAMP_MICROS_STRINGIFIER;
@@ -1051,6 +1052,13 @@ LogicalTypeToken getType() {
10511052
PrimitiveStringifier valueStringifier(PrimitiveType primitiveType) {
10521053
return PrimitiveStringifier.FLOAT16_STRINGIFIER;
10531054
}
1055+
1056+
@Override
1057+
boolean isValidColumnOrder(ColumnOrder columnOrder) {
1058+
return columnOrder.getColumnOrderName() == UNDEFINED
1059+
|| columnOrder.getColumnOrderName() == TYPE_DEFINED_ORDER
1060+
|| columnOrder.getColumnOrderName() == IEEE_754_TOTAL_ORDER;
1061+
}
10541062
}
10551063

10561064
public static class UnknownLogicalTypeAnnotation extends LogicalTypeAnnotation {

‎parquet-column/src/main/java/org/apache/parquet/schema/PrimitiveComparator.java‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -293,4 +293,65 @@ public String toString() {
293293
return "BINARY_AS_FLOAT16_COMPARATOR";
294294
}
295295
};
296+
297+
static final PrimitiveComparator<Float> FLOAT_IEEE_754_TOTAL_ORDER_COMPARATOR = new PrimitiveComparator<Float>() {
298+
@Override
299+
int compareNotNulls(Float o1, Float o2) {
300+
return compare(o1.floatValue(), o2.floatValue());
301+
}
302+
303+
@Override
304+
public int compare(float f1, float f2) {
305+
int f1Int = Float.floatToRawIntBits(f1);
306+
int f2Int = Float.floatToRawIntBits(f2);
307+
f1Int ^= ((f1Int >> 31) >>> 1);
308+
f2Int ^= ((f2Int >> 31) >>> 1);
309+
return Integer.compare(f1Int, f2Int);
310+
}
311+
312+
@Override
313+
public String toString() {
314+
return "FLOAT_IEEE_754_TOTAL_ORDER_COMPARATOR";
315+
}
316+
};
317+
318+
static final PrimitiveComparator<Double> DOUBLE_IEEE_754_TOTAL_ORDER_COMPARATOR =
319+
new PrimitiveComparator<Double>() {
320+
@Override
321+
int compareNotNulls(Double o1, Double o2) {
322+
return compare(o1.doubleValue(), o2.doubleValue());
323+
}
324+
325+
@Override
326+
public int compare(double d1, double d2) {
327+
long d1Long = Double.doubleToRawLongBits(d1);
328+
long d2Long = Double.doubleToRawLongBits(d2);
329+
d1Long ^= ((d1Long >> 63) >>> 1);
330+
d2Long ^= ((d2Long >> 63) >>> 1);
331+
return Long.compare(d1Long, d2Long);
332+
}
333+
334+
@Override
335+
public String toString() {
336+
return "DOUBLE_IEEE_754_TOTAL_ORDER_COMPARATOR";
337+
}
338+
};
339+
340+
static final PrimitiveComparator<Binary> BINARY_AS_FLOAT16_IEEE_754_TOTAL_ORDER_COMPARATOR =
341+
new BinaryComparator() {
342+
343+
@Override
344+
int compareBinary(Binary b1, Binary b2) {
345+
int b1Short = b1.get2BytesLittleEndian();
346+
int b2Short = b2.get2BytesLittleEndian();
347+
b1Short ^= ((b1Short >> 15) >>> 1);
348+
b2Short ^= ((b2Short >> 15) >>> 1);
349+
return Integer.compare(b1Short, b2Short);
350+
}
351+
352+
@Override
353+
public String toString() {
354+
return "BINARY_AS_FLOAT16_IEEE_754_TOTAL_ORDER_COMPARATOR";
355+
}
356+
};
296357
}

‎parquet-column/src/main/java/org/apache/parquet/schema/PrimitiveType.java‎

Lines changed: 47 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -88,7 +88,7 @@ public <T, E extends Exception> T convert(PrimitiveTypeNameConverter<T, E> conve
8888
}
8989

9090
@Override
91-
PrimitiveComparator<?> comparator(LogicalTypeAnnotation logicalType) {
91+
PrimitiveComparator<?> comparator(LogicalTypeAnnotation logicalType, ColumnOrder columnOrder) {
9292
if (logicalType == null) {
9393
return PrimitiveComparator.SIGNED_INT64_COMPARATOR;
9494
}
@@ -146,7 +146,7 @@ public <T, E extends Exception> T convert(PrimitiveTypeNameConverter<T, E> conve
146146
}
147147

148148
@Override
149-
PrimitiveComparator<?> comparator(LogicalTypeAnnotation logicalType) {
149+
PrimitiveComparator<?> comparator(LogicalTypeAnnotation logicalType, ColumnOrder columnOrder) {
150150
if (logicalType == null) {
151151
return PrimitiveComparator.SIGNED_INT32_COMPARATOR;
152152
}
@@ -210,7 +210,7 @@ public <T, E extends Exception> T convert(PrimitiveTypeNameConverter<T, E> conve
210210
}
211211

212212
@Override
213-
PrimitiveComparator<?> comparator(LogicalTypeAnnotation logicalType) {
213+
PrimitiveComparator<?> comparator(LogicalTypeAnnotation logicalType, ColumnOrder columnOrder) {
214214
return PrimitiveComparator.BOOLEAN_COMPARATOR;
215215
}
216216
},
@@ -236,7 +236,7 @@ public <T, E extends Exception> T convert(PrimitiveTypeNameConverter<T, E> conve
236236
}
237237

238238
@Override
239-
PrimitiveComparator<?> comparator(LogicalTypeAnnotation logicalType) {
239+
PrimitiveComparator<?> comparator(LogicalTypeAnnotation logicalType, ColumnOrder columnOrder) {
240240
if (logicalType == null) {
241241
return PrimitiveComparator.UNSIGNED_LEXICOGRAPHICAL_BINARY_COMPARATOR;
242242
}
@@ -310,8 +310,10 @@ public <T, E extends Exception> T convert(PrimitiveTypeNameConverter<T, E> conve
310310
}
311311

312312
@Override
313-
PrimitiveComparator<?> comparator(LogicalTypeAnnotation logicalType) {
314-
return PrimitiveComparator.FLOAT_COMPARATOR;
313+
PrimitiveComparator<?> comparator(LogicalTypeAnnotation logicalType, ColumnOrder columnOrder) {
314+
return columnOrder.getColumnOrderName() == ColumnOrderName.IEEE_754_TOTAL_ORDER
315+
? PrimitiveComparator.FLOAT_IEEE_754_TOTAL_ORDER_COMPARATOR
316+
: PrimitiveComparator.FLOAT_COMPARATOR;
315317
}
316318
},
317319
DOUBLE("getDouble", Double.TYPE) {
@@ -336,8 +338,10 @@ public <T, E extends Exception> T convert(PrimitiveTypeNameConverter<T, E> conve
336338
}
337339

338340
@Override
339-
PrimitiveComparator<?> comparator(LogicalTypeAnnotation logicalType) {
340-
return PrimitiveComparator.DOUBLE_COMPARATOR;
341+
PrimitiveComparator<?> comparator(LogicalTypeAnnotation logicalType, ColumnOrder columnOrder) {
342+
return columnOrder.getColumnOrderName() == ColumnOrderName.IEEE_754_TOTAL_ORDER
343+
? PrimitiveComparator.DOUBLE_IEEE_754_TOTAL_ORDER_COMPARATOR
344+
: PrimitiveComparator.DOUBLE_COMPARATOR;
341345
}
342346
},
343347
INT96("getBinary", Binary.class) {
@@ -362,7 +366,7 @@ public <T, E extends Exception> T convert(PrimitiveTypeNameConverter<T, E> conve
362366
}
363367

364368
@Override
365-
PrimitiveComparator<?> comparator(LogicalTypeAnnotation logicalType) {
369+
PrimitiveComparator<?> comparator(LogicalTypeAnnotation logicalType, ColumnOrder columnOrder) {
366370
return PrimitiveComparator.BINARY_AS_SIGNED_INTEGER_COMPARATOR;
367371
}
368372
},
@@ -388,11 +392,16 @@ public <T, E extends Exception> T convert(PrimitiveTypeNameConverter<T, E> conve
388392
}
389393

390394
@Override
391-
PrimitiveComparator<?> comparator(LogicalTypeAnnotation logicalType) {
395+
PrimitiveComparator<?> comparator(LogicalTypeAnnotation logicalType, ColumnOrder columnOrder) {
392396
if (logicalType == null) {
393397
return PrimitiveComparator.UNSIGNED_LEXICOGRAPHICAL_BINARY_COMPARATOR;
394398
}
395399

400+
if (logicalType.getType() == LogicalTypeAnnotation.LogicalTypeToken.FLOAT16
401+
&& columnOrder.getColumnOrderName() == ColumnOrderName.IEEE_754_TOTAL_ORDER) {
402+
return PrimitiveComparator.UNSIGNED_LEXICOGRAPHICAL_BINARY_COMPARATOR;
403+
}
404+
396405
return logicalType
397406
.accept(new LogicalTypeAnnotation.LogicalTypeAnnotationVisitor<PrimitiveComparator>() {
398407
@Override
@@ -453,7 +462,7 @@ public abstract void addValueToPrimitiveConverter(
453462

454463
public abstract <T, E extends Exception> T convert(PrimitiveTypeNameConverter<T, E> converter) throws E;
455464

456-
abstract PrimitiveComparator<?> comparator(LogicalTypeAnnotation logicalType);
465+
abstract PrimitiveComparator<?> comparator(LogicalTypeAnnotation logicalType, ColumnOrder columnOrder);
457466
}
458467

459468
private final PrimitiveTypeName primitive;
@@ -545,6 +554,12 @@ public PrimitiveType(
545554
columnOrder = primitive == PrimitiveTypeName.INT96 || originalType == OriginalType.INTERVAL
546555
? ColumnOrder.undefined()
547556
: ColumnOrder.typeDefined();
557+
} else if (columnOrder.getColumnOrderName() == ColumnOrderName.IEEE_754_TOTAL_ORDER) {
558+
Preconditions.checkArgument(
559+
primitive == PrimitiveTypeName.FLOAT || primitive == PrimitiveTypeName.DOUBLE,
560+
"The column order %s is not supported by type %s",
561+
columnOrder,
562+
primitive);
548563
}
549564
this.columnOrder = requireValidColumnOrder(columnOrder);
550565
}
@@ -591,6 +606,17 @@ public PrimitiveType(
591606
|| logicalTypeAnnotation instanceof LogicalTypeAnnotation.IntervalLogicalTypeAnnotation
592607
? ColumnOrder.undefined()
593608
: ColumnOrder.typeDefined();
609+
} else if (columnOrder.getColumnOrderName() == ColumnOrderName.IEEE_754_TOTAL_ORDER) {
610+
Preconditions.checkArgument(
611+
primitive == PrimitiveTypeName.FLOAT
612+
|| primitive == PrimitiveTypeName.DOUBLE
613+
|| (logicalTypeAnnotation != null
614+
&& logicalTypeAnnotation.getType()
615+
== LogicalTypeAnnotation.LogicalTypeToken.FLOAT16),
616+
"The column order %s is not supported by type %s logical type %s",
617+
columnOrder,
618+
primitive,
619+
logicalTypeAnnotation);
594620
}
595621
this.columnOrder = requireValidColumnOrder(columnOrder);
596622
}
@@ -631,6 +657,15 @@ public PrimitiveType withLogicalTypeAnnotation(LogicalTypeAnnotation logicalType
631657
return new PrimitiveType(getRepetition(), primitive, length, getName(), logicalType, getId());
632658
}
633659

660+
/**
661+
* @param columnOrder the column order
662+
* @return the same type with the column order set
663+
*/
664+
public Type withColumnOrder(ColumnOrder columnOrder) {
665+
return new PrimitiveType(
666+
getRepetition(), primitive, length, getName(), getLogicalTypeAnnotation(), getId(), columnOrder);
667+
}
668+
634669
/**
635670
* @return the primitive type
636671
*/
@@ -845,7 +880,7 @@ protected Type union(Type toMerge, boolean strict) {
845880
*/
846881
@SuppressWarnings("unchecked")
847882
public <T> PrimitiveComparator<T> comparator() {
848-
return (PrimitiveComparator<T>) getPrimitiveTypeName().comparator(getLogicalTypeAnnotation());
883+
return (PrimitiveComparator<T>) getPrimitiveTypeName().comparator(getLogicalTypeAnnotation(), columnOrder());
849884
}
850885

851886
/**

0 commit comments

Comments
 (0)