Skip to content

Commit 92a0d98

Browse files
xiangfu0xiangfu0
authored
[UUID 1/8] Add logical UUID type foundation (pinot-spi) (#18869)
Rebased onto master; resolves review feedback on #18869: markdown javadoc + UuidKey extracted to its own class + dropped per-value null checks and the defensive byte[] copy in UuidUtils; UUID placed right after its stored type BYTES in the enum/switches and dropped the fragile ordinal-append hack; removed the FieldSpec default-null defensive copy; restored fromUUIDBytes null-on-exception (back-compat); PinotDataType STRING->UUID now trims; added UUID version/timestamp/generator tests. Part 1/8 of splitting #18140 (logical UUID type).
1 parent f9647da commit 92a0d98

17 files changed

Lines changed: 1020 additions & 79 deletions

File tree

‎pinot-common/src/main/java/org/apache/pinot/common/function/scalar/StringFunctions.java‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -455,13 +455,12 @@ public static byte[] toUUIDBytes(String input) {
455455
}
456456

457457
/**
458-
* @param input UUID serialized to bytes
459-
* @return String representation of UUID
460-
* returns bytes and null on exception
458+
* @param input UUID serialized to its 16-byte form
459+
* @return canonical RFC 4122 String representation of the UUID
461460
*/
462461
@ScalarFunction
463462
public static String fromUUIDBytes(byte[] input) {
464-
return UuidUtils.fromBytes(input).toString();
463+
return UuidUtils.toString(input);
465464
}
466465

467466
/**

‎pinot-controller/src/main/resources/app/components/Homepage/Operations/SchemaComponent.tsx‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -106,8 +106,8 @@ export default function SchemaComponent({
106106
dateTimeFieldSpecs: []
107107
};
108108
const defaultDataTypeOptions = {
109-
dimension: ["INT", "LONG", "STRING", "FLOAT", "DOUBLE", "BYTES", "BOOLEAN", "JSON"],
110-
metric: ["INT", "LONG", "DOUBLE", "FLOAT", "BYTES"],
109+
dimension: ["INT", "LONG", "STRING", "FLOAT", "DOUBLE", "BIG_DECIMAL", "BYTES", "BOOLEAN", "JSON", "UUID"],
110+
metric: ["INT", "LONG", "DOUBLE", "FLOAT", "BIG_DECIMAL", "BYTES"],
111111
datetime: ["STRING", "INT", "LONG", "TIMESTAMP"]
112112
};
113113
const preFilledData = {

‎pinot-core/src/test/java/org/apache/pinot/core/data/manager/TableIndexingTest.java‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -159,7 +159,10 @@ private void readExpectedResults()
159159
protected void createSchemas() {
160160
for (DataType type : DataType.values()) {
161161
if (type == DataType.UNKNOWN || type == DataType.LIST || type == DataType.MAP || type == DataType.STRUCT
162-
|| type == DataType.OPEN_STRUCT) {
162+
|| type == DataType.OPEN_STRUCT || type == DataType.UUID) {
163+
// UUID is excluded because this static expectation matrix (TableIndexingTest.csv) has no UUID rows.
164+
// UUID-specific index behavior (inverted, bloom, range, dictionary/no-dictionary, SV and MV) is covered by
165+
// dedicated UUID unit and integration tests introduced in later PRs of this stack.
163166
continue;
164167
}
165168

‎pinot-core/src/test/java/org/apache/pinot/core/util/SchemaUtilsTest.java‎

Lines changed: 24 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -291,12 +291,31 @@ public void testValidateDateTimeFieldSpec() {
291291

292292
@Test
293293
public void testValidateMultiValueFieldSpec() {
294-
Schema pinotSchema = new Schema.SchemaBuilder()
295-
.setSchemaName(TABLE_NAME)
296-
.addSingleValueDimension("myCol", DataType.STRING)
297-
.addMultiValueDimension("myJsonCol", DataType.JSON)
298-
.build();
294+
Schema pinotSchema;
295+
296+
// JSON MV is rejected by SchemaUtils.validate() — the multi-value compatibility check lives there
297+
// (controller-side ingest validation), not in Schema.validate() (pure schema DTO validation).
298+
pinotSchema = new Schema.SchemaBuilder().setSchemaName(TABLE_NAME)
299+
.addSingleValueDimension("myCol", FieldSpec.DataType.STRING)
300+
.addMultiValueDimension("myJsonCol", FieldSpec.DataType.JSON).build();
299301
checkValidationFails(pinotSchema);
302+
303+
pinotSchema = new Schema.SchemaBuilder().setSchemaName(TABLE_NAME)
304+
.addSingleValueDimension("myCol", FieldSpec.DataType.STRING)
305+
.addSingleValueDimension("myJsonCol", FieldSpec.DataType.JSON).build();
306+
SchemaUtils.validate(pinotSchema);
307+
308+
pinotSchema = new Schema.SchemaBuilder().setSchemaName(TABLE_NAME)
309+
.addSingleValueDimension("myCol", FieldSpec.DataType.STRING)
310+
.addSingleValueDimension("myBigDecimalCol", FieldSpec.DataType.BIG_DECIMAL)
311+
.addMultiValueDimension("myBigDecimalMvCol", FieldSpec.DataType.BIG_DECIMAL).build();
312+
SchemaUtils.validate(pinotSchema);
313+
314+
pinotSchema = new Schema.SchemaBuilder().setSchemaName(TABLE_NAME)
315+
.addSingleValueDimension("myCol", FieldSpec.DataType.STRING)
316+
.addSingleValueDimension("myUuidCol", FieldSpec.DataType.UUID)
317+
.addMultiValueDimension("myUuidMvCol", FieldSpec.DataType.UUID).build();
318+
SchemaUtils.validate(pinotSchema);
300319
}
301320

302321
@Test

‎pinot-plugins/pinot-input-format/pinot-parquet/src/main/java/org/apache/pinot/plugin/inputformat/parquet/ParquetNativeRecordExtractor.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -196,7 +196,7 @@ private Object extractValue(Group from, int fieldIndex, Type fieldType, int inde
196196
// `FIXED_LEN_BYTE_ARRAY(16) + UUID` → [UUID] (always converted; the downstream type
197197
// transformer adapts to the Pinot column's storage type). UUID wire bytes are big-endian
198198
// per RFC 4122.
199-
return UuidUtils.fromBytes(binaryBytes);
199+
return UuidUtils.toUUID(binaryBytes);
200200
}
201201
return binaryBytes;
202202
default:

‎pinot-query-planner/src/main/java/org/apache/pinot/query/type/TypeFactory.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -102,6 +102,8 @@ private static SqlTypeName getSqlTypeName(FieldSpec fieldSpec) {
102102
case STRING:
103103
case JSON:
104104
return SqlTypeName.VARCHAR;
105+
case UUID:
106+
return SqlTypeName.UUID;
105107
case BYTES:
106108
return SqlTypeName.VARBINARY;
107109
case BIG_DECIMAL:

‎pinot-query-planner/src/test/java/org/apache/pinot/query/type/TypeFactoryTest.java‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -104,6 +104,10 @@ public Iterator<Object[]> relDataTypeConversion() {
104104
basicType = TYPE_FACTORY.createSqlType(SqlTypeName.VARCHAR);
105105
break;
106106
}
107+
case UUID: {
108+
basicType = TYPE_FACTORY.createSqlType(SqlTypeName.UUID);
109+
break;
110+
}
107111
case BYTES: {
108112
basicType = TYPE_FACTORY.createSqlType(SqlTypeName.VARBINARY);
109113
break;
@@ -180,6 +184,9 @@ private boolean isColNullable(Schema schema) {
180184

181185
@Test(dataProvider = "relDataTypeConversion")
182186
public void testArrayTypes(FieldSpec.DataType dataType, RelDataType arrayType, boolean columnNullMode) {
187+
if (dataType == FieldSpec.DataType.BIG_DECIMAL || dataType == FieldSpec.DataType.JSON) {
188+
return;
189+
}
183190
TypeFactory typeFactory = new TypeFactory();
184191
Schema testSchema = new Schema.SchemaBuilder()
185192
.addMultiValueDimension("col", dataType)
@@ -198,6 +205,9 @@ public void testArrayTypes(FieldSpec.DataType dataType, RelDataType arrayType, b
198205

199206
@Test(dataProvider = "relDataTypeConversion")
200207
public void testNullableArrayTypes(FieldSpec.DataType dataType, RelDataType arrayType, boolean columnNullMode) {
208+
if (dataType == FieldSpec.DataType.BIG_DECIMAL || dataType == FieldSpec.DataType.JSON) {
209+
return;
210+
}
201211
TypeFactory typeFactory = new TypeFactory();
202212
Schema testSchema = new Schema.SchemaBuilder()
203213
.addDimensionField("col", dataType, field -> {
@@ -219,6 +229,9 @@ public void testNullableArrayTypes(FieldSpec.DataType dataType, RelDataType arra
219229

220230
@Test(dataProvider = "relDataTypeConversion")
221231
public void testNotNullableArrayTypes(FieldSpec.DataType dataType, RelDataType arrayType, boolean columnNullMode) {
232+
if (dataType == FieldSpec.DataType.BIG_DECIMAL || dataType == FieldSpec.DataType.JSON) {
233+
return;
234+
}
222235
TypeFactory typeFactory = new TypeFactory();
223236
Schema testSchema = new Schema.SchemaBuilder()
224237
.addDimensionField("col", dataType, field -> {
@@ -244,13 +257,15 @@ public void testRelDataTypeConversion() {
244257
.addSingleValueDimension("FLOAT_COL", FieldSpec.DataType.FLOAT)
245258
.addSingleValueDimension("DOUBLE_COL", FieldSpec.DataType.DOUBLE)
246259
.addSingleValueDimension("STRING_COL", FieldSpec.DataType.STRING)
260+
.addSingleValueDimension("UUID_COL", FieldSpec.DataType.UUID)
247261
.addSingleValueDimension("BYTES_COL", FieldSpec.DataType.BYTES)
248262
.addSingleValueDimension("JSON_COL", FieldSpec.DataType.JSON)
249263
.addMultiValueDimension("INT_ARRAY_COL", FieldSpec.DataType.INT)
250264
.addMultiValueDimension("LONG_ARRAY_COL", FieldSpec.DataType.LONG)
251265
.addMultiValueDimension("FLOAT_ARRAY_COL", FieldSpec.DataType.FLOAT)
252266
.addMultiValueDimension("DOUBLE_ARRAY_COL", FieldSpec.DataType.DOUBLE)
253267
.addMultiValueDimension("STRING_ARRAY_COL", FieldSpec.DataType.STRING)
268+
.addMultiValueDimension("UUID_ARRAY_COL", FieldSpec.DataType.UUID)
254269
.addMultiValueDimension("BYTES_ARRAY_COL", FieldSpec.DataType.BYTES)
255270
.build();
256271
RelDataType relDataTypeFromSchema = typeFactory.createRelDataTypeFromSchema(testSchema);
@@ -283,6 +298,9 @@ public void testRelDataTypeConversion() {
283298
TYPE_FACTORY.createTypeWithCharsetAndCollation(new BasicSqlType(TypeSystem.INSTANCE, SqlTypeName.VARCHAR),
284299
StandardCharsets.UTF_8, SqlCollation.IMPLICIT));
285300
break;
301+
case "UUID_COL":
302+
Assert.assertEquals(field.getType(), new BasicSqlType(TypeSystem.INSTANCE, SqlTypeName.UUID));
303+
break;
286304
case "BYTES_COL":
287305
Assert.assertEquals(field.getType(), new BasicSqlType(TypeSystem.INSTANCE, SqlTypeName.VARBINARY));
288306
break;
@@ -307,6 +325,10 @@ public void testRelDataTypeConversion() {
307325
TYPE_FACTORY.createTypeWithCharsetAndCollation(new BasicSqlType(TypeSystem.INSTANCE, SqlTypeName.VARCHAR),
308326
StandardCharsets.UTF_8, SqlCollation.IMPLICIT), false));
309327
break;
328+
case "UUID_ARRAY_COL":
329+
Assert.assertEquals(field.getType(),
330+
new ArraySqlType(new BasicSqlType(TypeSystem.INSTANCE, SqlTypeName.UUID), false));
331+
break;
310332
case "BYTES_ARRAY_COL":
311333
Assert.assertEquals(field.getType(),
312334
new ArraySqlType(new BasicSqlType(TypeSystem.INSTANCE, SqlTypeName.VARBINARY), false));

‎pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/SchemaUtils.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -182,9 +182,9 @@ private static void validateDefaultIsNotNaN(FieldSpec fieldSpec) {
182182
fieldSpec.getName());
183183
}
184184

185-
/**
186-
* Validations for MV type columns
187-
*/
185+
/// Validations for MV type columns. Kept here (rather than in [Schema#validate()]) so that schema construction
186+
/// via `SchemaBuilder.build()` stays a pure DTO operation and only the controller-side ingest validation rejects
187+
/// MV JSON columns.
188188
private static void validateMultiValueCompatibility(FieldSpec fieldSpec) {
189189
Preconditions.checkState(!fieldSpec.getDataType().equals(FieldSpec.DataType.JSON),
190190
"JSON columns cannot be of multi-value type");

0 commit comments

Comments
 (0)