Skip to content

Commit b2bd219

Browse files
pratickchokhanipratickchokhani
committed
feat: add support for UUID and PG_UUID types across Spanner migration utilities
1 parent 466301e commit b2bd219

21 files changed

Lines changed: 201 additions & 16 deletions

File tree

‎v2/datastream-to-spanner/src/test/resources/PostgreSQLDataTypesIT/pg-dialect-spanner-schema.sql‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -130,7 +130,7 @@ CREATE TABLE IF NOT EXISTS t_tstzmultirange (id INT8, col VARCHAR, PRIMARY KEY (
130130
CREATE TABLE IF NOT EXISTS t_tstzrange (id INT8, col VARCHAR, PRIMARY KEY (id));
131131
CREATE TABLE IF NOT EXISTS t_tsvector (id INT8, col VARCHAR, PRIMARY KEY (id));
132132
CREATE TABLE IF NOT EXISTS t_txid_snapshot (id INT8, col VARCHAR, PRIMARY KEY (id));
133-
CREATE TABLE IF NOT EXISTS t_uuid (id INT8, col VARCHAR, PRIMARY KEY (id));
133+
CREATE TABLE IF NOT EXISTS t_uuid (id INT8, col uuid, PRIMARY KEY (id));
134134
CREATE TABLE IF NOT EXISTS t_uuid_to_bytes (id INT8, col BYTEA, PRIMARY KEY (id));
135135
CREATE TABLE IF NOT EXISTS t_varbit (id INT8, col BYTEA, PRIMARY KEY (id));
136136
CREATE TABLE IF NOT EXISTS t_varbit_to_bool_array (id INT8, col BOOL[], PRIMARY KEY (id));

‎v2/datastream-to-spanner/src/test/resources/PostgreSQLDataTypesIT/spanner-schema.sql‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -130,7 +130,7 @@ CREATE TABLE IF NOT EXISTS t_tstzmultirange (id INT64, col STRING(MAX)) PRIMARY
130130
CREATE TABLE IF NOT EXISTS t_tstzrange (id INT64, col STRING(MAX)) PRIMARY KEY (id);
131131
CREATE TABLE IF NOT EXISTS t_tsvector (id INT64, col STRING(MAX)) PRIMARY KEY (id);
132132
CREATE TABLE IF NOT EXISTS t_txid_snapshot (id INT64, col STRING(MAX)) PRIMARY KEY (id);
133-
CREATE TABLE IF NOT EXISTS t_uuid (id INT64, col STRING(MAX)) PRIMARY KEY (id);
133+
CREATE TABLE IF NOT EXISTS t_uuid (id INT64, col UUID) PRIMARY KEY (id);
134134
CREATE TABLE IF NOT EXISTS t_uuid_to_bytes (id INT64, col BYTES(MAX)) PRIMARY KEY (id);
135135
CREATE TABLE IF NOT EXISTS t_varbit (id INT64, col BYTES(MAX)) PRIMARY KEY (id);
136136
CREATE TABLE IF NOT EXISTS t_varbit_to_bool_array (id INT64, col ARRAY<BOOL>) PRIMARY KEY (id);

‎v2/sourcedb-to-spanner/src/test/resources/DataTypesIT/postgresql-pg-dialect-spanner-schema.sql‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -130,7 +130,7 @@ CREATE TABLE IF NOT EXISTS t_tstzmultirange (id INT8, col VARCHAR, PRIMARY KEY (
130130
CREATE TABLE IF NOT EXISTS t_tstzrange (id INT8, col VARCHAR, PRIMARY KEY (id));
131131
CREATE TABLE IF NOT EXISTS t_tsvector (id INT8, col VARCHAR, PRIMARY KEY (id));
132132
CREATE TABLE IF NOT EXISTS t_txid_snapshot (id INT8, col VARCHAR, PRIMARY KEY (id));
133-
CREATE TABLE IF NOT EXISTS t_uuid (id INT8, col VARCHAR, PRIMARY KEY (id));
133+
CREATE TABLE IF NOT EXISTS t_uuid (id INT8, col uuid, PRIMARY KEY (id));
134134
CREATE TABLE IF NOT EXISTS t_uuid_to_bytes (id INT8, col BYTEA, PRIMARY KEY (id));
135135
CREATE TABLE IF NOT EXISTS t_varbit (id INT8, col BYTEA, PRIMARY KEY (id));
136136
CREATE TABLE IF NOT EXISTS t_varbit_to_bool_array (id INT8, col BOOL[], PRIMARY KEY (id));

‎v2/sourcedb-to-spanner/src/test/resources/DataTypesIT/postgresql-spanner-schema.sql‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -130,7 +130,7 @@ CREATE TABLE IF NOT EXISTS t_tstzmultirange (id INT64, col STRING(MAX)) PRIMARY
130130
CREATE TABLE IF NOT EXISTS t_tstzrange (id INT64, col STRING(MAX)) PRIMARY KEY (id);
131131
CREATE TABLE IF NOT EXISTS t_tsvector (id INT64, col STRING(MAX)) PRIMARY KEY (id);
132132
CREATE TABLE IF NOT EXISTS t_txid_snapshot (id INT64, col STRING(MAX)) PRIMARY KEY (id);
133-
CREATE TABLE IF NOT EXISTS t_uuid (id INT64, col STRING(MAX)) PRIMARY KEY (id);
133+
CREATE TABLE IF NOT EXISTS t_uuid (id INT64, col UUID) PRIMARY KEY (id);
134134
CREATE TABLE IF NOT EXISTS t_uuid_to_bytes (id INT64, col BYTES(MAX)) PRIMARY KEY (id);
135135
CREATE TABLE IF NOT EXISTS t_varbit (id INT64, col BYTES(MAX)) PRIMARY KEY (id);
136136
CREATE TABLE IF NOT EXISTS t_varbit_to_bool_array (id INT64, col ARRAY<BOOL>) PRIMARY KEY (id);

‎v2/spanner-common/src/main/java/com/google/cloud/teleport/v2/spanner/ddl/Column.java‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -164,6 +164,10 @@ private static String typeString(Type type, Integer size) {
164164
return Type.Code.JSON.getName();
165165
case PG_JSONB:
166166
return Type.Code.PG_JSONB.getName();
167+
case UUID:
168+
return Type.Code.UUID.getName();
169+
case PG_UUID:
170+
return Type.Code.PG_UUID.getName();
167171
case TOKENLIST:
168172
return Type.Code.TOKENLIST.getName();
169173
case ARRAY:
@@ -250,6 +254,10 @@ public Builder bytes() {
250254
return type(Type.bytes());
251255
}
252256

257+
public Builder uuid() {
258+
return type(Type.uuid());
259+
}
260+
253261
public Builder timestamp() {
254262
return type(Type.timestamp());
255263
}
@@ -306,6 +314,10 @@ public Builder pgJsonb() {
306314
return type(Type.pgJsonb());
307315
}
308316

317+
public Builder pgUuid() {
318+
return type(Type.pgUuid());
319+
}
320+
309321
public Builder max() {
310322
return size(-1);
311323
}
@@ -377,6 +389,9 @@ private static SizedType parseSpannerType(String spannerType, Dialect dialect) {
377389
if (spannerType.equals(Type.Code.NUMERIC.getName())) {
378390
return t(Type.numeric(), null);
379391
}
392+
if (spannerType.equals(Type.Code.UUID.getName())) {
393+
return t(Type.uuid(), null);
394+
}
380395
if (spannerType.equals(Type.Code.JSON.getName())) {
381396
return t(Type.json(), null);
382397
}
@@ -432,6 +447,9 @@ private static SizedType parseSpannerType(String spannerType, Dialect dialect) {
432447
if (spannerType.equals(Type.Code.PG_NUMERIC.getName())) {
433448
return t(Type.pgNumeric(), null);
434449
}
450+
if (spannerType.equals(Type.Code.PG_UUID.getName())) {
451+
return t(Type.pgUuid(), null);
452+
}
435453
if (spannerType.equals(Type.Code.PG_JSONB.getName())) {
436454
return t(Type.pgJsonb(), null);
437455
}

‎v2/spanner-common/src/main/java/com/google/cloud/teleport/v2/spanner/migrations/avro/GenericRecordTypeConvertor.java‎

Lines changed: 16 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -215,7 +215,7 @@ public Map<String, Value> transformChangeEvent(GenericRecord record, String srcT
215215
throw e;
216216
} catch (Exception e) {
217217
throw new RuntimeException(
218-
String.format("Unable to convert spanner value for spanner col: %s", spannerColName),
218+
String.format("Unable to convert spanner value for spanner col: %s. table: %s", spannerColName, srcTableName),
219219
e);
220220
}
221221
}
@@ -570,10 +570,21 @@ private Value getSpannerValueFromObject(
570570
throw new NullPointerException("schemaMapper returned null spanner dialect.");
571571
}
572572
if (AvroToValueMapper.convertorMap().get(dialect).containsKey(spannerType)) {
573-
return AvroToValueMapper.convertorMap()
574-
.get(dialect)
575-
.get(spannerType)
576-
.apply(value, fieldSchema);
573+
try {
574+
return AvroToValueMapper.convertorMap()
575+
.get(dialect)
576+
.get(spannerType)
577+
.apply(value, fieldSchema);
578+
} catch (Exception e) {
579+
LOG.error(
580+
"Exception while converting record value {} to Field Schema: {}, colName: {}, spannerType: {}",
581+
value,
582+
fieldSchema,
583+
recordColName,
584+
spannerType,
585+
e);
586+
throw e;
587+
}
577588
} else {
578589
throw new IllegalArgumentException(
579590
"Found unsupported Spanner column type("

‎v2/spanner-common/src/main/java/com/google/cloud/teleport/v2/spanner/migrations/convertors/ChangeEventSpannerConvertor.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -111,6 +111,8 @@ public static com.google.cloud.spanner.Key changeEventToPrimaryKey(
111111
case STRING:
112112
case PG_VARCHAR:
113113
case PG_TEXT:
114+
case UUID:
115+
case PG_UUID:
114116
pk.append(
115117
ChangeEventTypeConvertor.toString(
116118
changeEvent, keyColName, /* requiredField= */ true));

‎v2/spanner-common/src/main/java/com/google/cloud/teleport/v2/spanner/migrations/convertors/ChangeEventTypeConvertor.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,8 @@ public static Value toValue(
7171
case STRING:
7272
case PG_VARCHAR:
7373
case PG_TEXT:
74+
case UUID:
75+
case PG_UUID:
7476
return Value.string(toString(changeEvent, key, requiredField));
7577
case NUMERIC:
7678
case PG_NUMERIC:

‎v2/spanner-common/src/main/java/com/google/cloud/teleport/v2/spanner/migrations/spanner/SpannerReadUtils.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -111,6 +111,7 @@ private static void bindGoogleSqlValue(
111111
break;
112112
case STRING:
113113
case JSON:
114+
case UUID:
114115
stmtBuilder.bind(bindName).to((String) value);
115116
break;
116117
case NUMERIC:
@@ -153,6 +154,7 @@ private static void bindPgValue(
153154
case PG_VARCHAR:
154155
case PG_TEXT:
155156
case PG_JSONB:
157+
case PG_UUID:
156158
stmtBuilder.bind(bindName).to((String) value);
157159
break;
158160
case PG_NUMERIC:

‎v2/spanner-common/src/main/java/com/google/cloud/teleport/v2/spanner/type/Type.java‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@ public final class Type implements Serializable {
4444
private static final Type TYPE_BYTES = new Type(Type.Code.BYTES, null, null);
4545
private static final Type TYPE_TIMESTAMP = new Type(Type.Code.TIMESTAMP, null, null);
4646
private static final Type TYPE_DATE = new Type(Type.Code.DATE, null, null);
47+
private static final Type TYPE_UUID = new Type(Type.Code.UUID, null, null);
4748
private static final Type TYPE_ARRAY_BOOL = new Type(Type.Code.ARRAY, TYPE_BOOL, null);
4849
private static final Type TYPE_ARRAY_INT64 = new Type(Type.Code.ARRAY, TYPE_INT64, null);
4950
private static final Type TYPE_ARRAY_FLOAT32 = new Type(Type.Code.ARRAY, TYPE_FLOAT32, null);
@@ -66,6 +67,7 @@ public final class Type implements Serializable {
6667
private static final Type TYPE_PG_BYTEA = new Type(Type.Code.PG_BYTEA, null, null);
6768
private static final Type TYPE_PG_TIMESTAMPTZ = new Type(Type.Code.PG_TIMESTAMPTZ, null, null);
6869
private static final Type TYPE_PG_DATE = new Type(Type.Code.PG_DATE, null, null);
70+
private static final Type TYPE_PG_UUID = new Type(Type.Code.PG_UUID, null, null);
6971
private static final Type TYPE_PG_ARRAY_BOOL = new Type(Type.Code.PG_ARRAY, TYPE_PG_BOOL, null);
7072
private static final Type TYPE_PG_ARRAY_INT8 = new Type(Type.Code.PG_ARRAY, TYPE_PG_INT8, null);
7173
private static final Type TYPE_PG_ARRAY_FLOAT4 =
@@ -160,6 +162,11 @@ public static Type date() {
160162
return TYPE_DATE;
161163
}
162164

165+
/** Returns the descriptor for the {@code UUID} type. */
166+
public static Type uuid() {
167+
return TYPE_UUID;
168+
}
169+
163170
public static Type pgBool() {
164171
return TYPE_PG_BOOL;
165172
}
@@ -204,6 +211,10 @@ public static Type pgDate() {
204211
return TYPE_PG_DATE;
205212
}
206213

214+
public static Type pgUuid() {
215+
return TYPE_PG_UUID;
216+
}
217+
207218
public static Type pgCommitTimestamp() {
208219
return TYPE_PG_COMMIT_TIMESTAMP;
209220
}
@@ -317,6 +328,7 @@ public enum Code {
317328
BYTES("BYTES", Dialect.GOOGLE_STANDARD_SQL),
318329
TIMESTAMP("TIMESTAMP", Dialect.GOOGLE_STANDARD_SQL),
319330
DATE("DATE", Dialect.GOOGLE_STANDARD_SQL),
331+
UUID("UUID", Dialect.GOOGLE_STANDARD_SQL),
320332
ARRAY("ARRAY", Dialect.GOOGLE_STANDARD_SQL),
321333
STRUCT("STRUCT", Dialect.GOOGLE_STANDARD_SQL),
322334
PG_BOOL("boolean", Dialect.POSTGRESQL),
@@ -330,6 +342,7 @@ public enum Code {
330342
PG_BYTEA("bytea", Dialect.POSTGRESQL),
331343
PG_TIMESTAMPTZ("timestamp with time zone", Dialect.POSTGRESQL),
332344
PG_DATE("date", Dialect.POSTGRESQL),
345+
PG_UUID("uuid", Dialect.POSTGRESQL),
333346
PG_ARRAY("array", Dialect.POSTGRESQL),
334347
PG_COMMIT_TIMESTAMP("spanner.commit_timestamp", Dialect.POSTGRESQL);
335348

0 commit comments

Comments
 (0)