Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -611,6 +611,7 @@ private Map<String, List<Map<String, Object>>> getExpectedData() {
expectedData.put("set", createRows("set", "v1,v2", "NULL"));
expectedData.put(
"integer_unsigned", createRows("integer_unsigned", "0", "42", "4294967295", "NULL"));
expectedData.put("uuid", createRows("uuid", "a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11", "NULL"));
return expectedData;
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -739,4 +739,9 @@ CREATE TABLE IF NOT EXISTS spatial_geometrycollection (

INSERT INTO spatial_geometrycollection (geoms) VALUES (MultiPoint(Point(77.5946, 12.9716), Point(77.6100, 12.9600)));

CREATE TABLE IF NOT EXISTS `uuid_table` (
`id` INT PRIMARY KEY,
`uuid_col` VARCHAR(36) DEFAULT NULL
);

INSERT INTO `uuid_table` (`id`, `uuid_col`) VALUES (1, 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11'), (2, NULL);
Original file line number Diff line number Diff line change
Expand Up @@ -477,3 +477,9 @@ CREATE TABLE IF NOT EXISTS spatial_geometrycollection (
geom_coll VARCHAR,
PRIMARY KEY (id)
);

CREATE TABLE IF NOT EXISTS uuid_table (

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

we should also check for scenarios where UUID is the PK

id INT8 NOT NULL,
uuid_col UUID,
PRIMARY KEY (id)
);
Original file line number Diff line number Diff line change
Expand Up @@ -424,3 +424,8 @@ CREATE TABLE IF NOT EXISTS `generated_to_non_generated_column_table` (
`generated_column_col` STRING(100) DEFAULT(NULL),
`generated_column_pk_col` STRING(100) DEFAULT(NULL),
) PRIMARY KEY (`generated_column_pk_col`);

CREATE TABLE IF NOT EXISTS uuid_table (
id INT64 NOT NULL,
uuid_col UUID,
) PRIMARY KEY(id);
Original file line number Diff line number Diff line change
Expand Up @@ -130,7 +130,7 @@ CREATE TABLE IF NOT EXISTS t_tstzmultirange (id INT8, col VARCHAR, PRIMARY KEY (
CREATE TABLE IF NOT EXISTS t_tstzrange (id INT8, col VARCHAR, PRIMARY KEY (id));
CREATE TABLE IF NOT EXISTS t_tsvector (id INT8, col VARCHAR, PRIMARY KEY (id));
CREATE TABLE IF NOT EXISTS t_txid_snapshot (id INT8, col VARCHAR, PRIMARY KEY (id));
CREATE TABLE IF NOT EXISTS t_uuid (id INT8, col VARCHAR, PRIMARY KEY (id));
CREATE TABLE IF NOT EXISTS t_uuid (id INT8, col uuid, PRIMARY KEY (id));
CREATE TABLE IF NOT EXISTS t_uuid_to_bytes (id INT8, col BYTEA, PRIMARY KEY (id));
CREATE TABLE IF NOT EXISTS t_varbit (id INT8, col BYTEA, PRIMARY KEY (id));
CREATE TABLE IF NOT EXISTS t_varbit_to_bool_array (id INT8, col BOOL[], PRIMARY KEY (id));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -130,7 +130,7 @@ CREATE TABLE IF NOT EXISTS t_tstzmultirange (id INT64, col STRING(MAX)) PRIMARY
CREATE TABLE IF NOT EXISTS t_tstzrange (id INT64, col STRING(MAX)) PRIMARY KEY (id);
CREATE TABLE IF NOT EXISTS t_tsvector (id INT64, col STRING(MAX)) PRIMARY KEY (id);
CREATE TABLE IF NOT EXISTS t_txid_snapshot (id INT64, col STRING(MAX)) PRIMARY KEY (id);
CREATE TABLE IF NOT EXISTS t_uuid (id INT64, col STRING(MAX)) PRIMARY KEY (id);
CREATE TABLE IF NOT EXISTS t_uuid (id INT64, col UUID) PRIMARY KEY (id);
CREATE TABLE IF NOT EXISTS t_uuid_to_bytes (id INT64, col BYTES(MAX)) PRIMARY KEY (id);
CREATE TABLE IF NOT EXISTS t_varbit (id INT64, col BYTES(MAX)) PRIMARY KEY (id);
CREATE TABLE IF NOT EXISTS t_varbit_to_bool_array (id INT64, col ARRAY<BOOL>) PRIMARY KEY (id);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -474,6 +474,7 @@ private Map<String, List<Map<String, Object>>> getExpectedData() {
"-2.2250738585072014E-308"));
expectedData.put(
"time_pk", createRows("time_pk", "15:50:00.200000", "838:59:58.123000", "-838:59:59"));
expectedData.put("uuid", createRows("uuid", "a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11", "NULL"));
return expectedData;
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -899,4 +899,9 @@ CREATE TABLE IF NOT EXISTS spatial_geometrycollection (

INSERT INTO spatial_geometrycollection (geoms) VALUES (MultiPoint(Point(77.5946, 12.9716), Point(77.6100, 12.9600)));

CREATE TABLE IF NOT EXISTS `uuid_table` (
`id` INT PRIMARY KEY,
`uuid_col` VARCHAR(36) DEFAULT NULL
);

INSERT INTO `uuid_table` (`id`, `uuid_col`) VALUES (1, 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11'), (2, NULL);
Original file line number Diff line number Diff line change
Expand Up @@ -546,3 +546,8 @@ CREATE TABLE time_pk_table (
id STRING(MAX) NOT NULL,
time_pk_col STRING(MAX) NOT NULL
) PRIMARY KEY(id);

CREATE TABLE IF NOT EXISTS uuid_table (
id INT64 NOT NULL,
uuid_col UUID,
) PRIMARY KEY(id);
Original file line number Diff line number Diff line change
Expand Up @@ -130,7 +130,7 @@ CREATE TABLE IF NOT EXISTS t_tstzmultirange (id INT8, col VARCHAR, PRIMARY KEY (
CREATE TABLE IF NOT EXISTS t_tstzrange (id INT8, col VARCHAR, PRIMARY KEY (id));
CREATE TABLE IF NOT EXISTS t_tsvector (id INT8, col VARCHAR, PRIMARY KEY (id));
CREATE TABLE IF NOT EXISTS t_txid_snapshot (id INT8, col VARCHAR, PRIMARY KEY (id));
CREATE TABLE IF NOT EXISTS t_uuid (id INT8, col VARCHAR, PRIMARY KEY (id));
CREATE TABLE IF NOT EXISTS t_uuid (id INT8, col uuid, PRIMARY KEY (id));

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: all the other types are in capitals. For uniformity, please do the same.

CREATE TABLE IF NOT EXISTS t_uuid_to_bytes (id INT8, col BYTEA, PRIMARY KEY (id));
CREATE TABLE IF NOT EXISTS t_varbit (id INT8, col BYTEA, PRIMARY KEY (id));
CREATE TABLE IF NOT EXISTS t_varbit_to_bool_array (id INT8, col BOOL[], PRIMARY KEY (id));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -130,7 +130,7 @@ CREATE TABLE IF NOT EXISTS t_tstzmultirange (id INT64, col STRING(MAX)) PRIMARY
CREATE TABLE IF NOT EXISTS t_tstzrange (id INT64, col STRING(MAX)) PRIMARY KEY (id);
CREATE TABLE IF NOT EXISTS t_tsvector (id INT64, col STRING(MAX)) PRIMARY KEY (id);
CREATE TABLE IF NOT EXISTS t_txid_snapshot (id INT64, col STRING(MAX)) PRIMARY KEY (id);
CREATE TABLE IF NOT EXISTS t_uuid (id INT64, col STRING(MAX)) PRIMARY KEY (id);
CREATE TABLE IF NOT EXISTS t_uuid (id INT64, col UUID) PRIMARY KEY (id);
CREATE TABLE IF NOT EXISTS t_uuid_to_bytes (id INT64, col BYTES(MAX)) PRIMARY KEY (id);
CREATE TABLE IF NOT EXISTS t_varbit (id INT64, col BYTES(MAX)) PRIMARY KEY (id);
CREATE TABLE IF NOT EXISTS t_varbit_to_bool_array (id INT64, col ARRAY<BOOL>) PRIMARY KEY (id);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -164,6 +164,10 @@ private static String typeString(Type type, Integer size) {
return Type.Code.JSON.getName();
case PG_JSONB:
return Type.Code.PG_JSONB.getName();
case UUID:
return Type.Code.UUID.getName();
case PG_UUID:
return Type.Code.PG_UUID.getName();
case TOKENLIST:
return Type.Code.TOKENLIST.getName();
case ARRAY:
Expand Down Expand Up @@ -250,6 +254,10 @@ public Builder bytes() {
return type(Type.bytes());
}

public Builder uuid() {
return type(Type.uuid());
}

public Builder timestamp() {
return type(Type.timestamp());
}
Expand Down Expand Up @@ -306,6 +314,10 @@ public Builder pgJsonb() {
return type(Type.pgJsonb());
}

public Builder pgUuid() {
return type(Type.pgUuid());
}

public Builder max() {
return size(-1);
}
Expand Down Expand Up @@ -377,6 +389,9 @@ private static SizedType parseSpannerType(String spannerType, Dialect dialect) {
if (spannerType.equals(Type.Code.NUMERIC.getName())) {
return t(Type.numeric(), null);
}
if (spannerType.equals(Type.Code.UUID.getName())) {
return t(Type.uuid(), null);
}
if (spannerType.equals(Type.Code.JSON.getName())) {
return t(Type.json(), null);
}
Expand Down Expand Up @@ -432,6 +447,9 @@ private static SizedType parseSpannerType(String spannerType, Dialect dialect) {
if (spannerType.equals(Type.Code.PG_NUMERIC.getName())) {
return t(Type.pgNumeric(), null);
}
if (spannerType.equals(Type.Code.PG_UUID.getName())) {
return t(Type.pgUuid(), null);
}
if (spannerType.equals(Type.Code.PG_JSONB.getName())) {
return t(Type.pgJsonb(), null);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -148,6 +148,10 @@ static Map<Type, AvroToValueFunction> getGsqlMap() {
avroArrayFieldToSpannerArray(
recordValue, fieldSchema, AvroToValueMapper::avroFieldToString)));

gsqlFunctions.put(
Type.uuid(),
(recordValue, fieldSchema) -> Value.string(avroFieldToString(recordValue, fieldSchema)));

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

we should add support for UUID array type too

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Array support is out of scope of this PR.

gsqlFunctions.put(
Type.json(),
(recordValue, fieldSchema) -> Value.string(avroFieldToString(recordValue, fieldSchema)));
Expand Down Expand Up @@ -245,6 +249,9 @@ static Map<Type, AvroToValueFunction> getPgMap() {
pgFunctions.put(
Type.pgDate(),
(recordValue, fieldSchema) -> Value.date(avroFieldToDate(recordValue, fieldSchema)));
pgFunctions.put(
Type.pgUuid(),
(recordValue, fieldSchema) -> Value.string(avroFieldToString(recordValue, fieldSchema)));
return pgFunctions;
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -215,7 +215,9 @@ public Map<String, Value> transformChangeEvent(GenericRecord record, String srcT
throw e;
} catch (Exception e) {
throw new RuntimeException(
String.format("Unable to convert spanner value for spanner col: %s", spannerColName),
String.format(
"Unable to convert spanner value for spanner col: %s. table: %s",
spannerColName, srcTableName),
e);
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -111,6 +111,8 @@ public static com.google.cloud.spanner.Key changeEventToPrimaryKey(
case STRING:
case PG_VARCHAR:
case PG_TEXT:
case UUID:
case PG_UUID:
pk.append(
ChangeEventTypeConvertor.toString(
changeEvent, keyColName, /* requiredField= */ true));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,8 @@ public static Value toValue(
case STRING:
case PG_VARCHAR:
case PG_TEXT:
case UUID:
case PG_UUID:
return Value.string(toString(changeEvent, key, requiredField));
case NUMERIC:
case PG_NUMERIC:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -111,6 +111,7 @@ private static void bindGoogleSqlValue(
break;
case STRING:
case JSON:
case UUID:
stmtBuilder.bind(bindName).to((String) value);
break;
case NUMERIC:
Expand Down Expand Up @@ -153,6 +154,7 @@ private static void bindPgValue(
case PG_VARCHAR:
case PG_TEXT:
case PG_JSONB:
case PG_UUID:
stmtBuilder.bind(bindName).to((String) value);
break;
case PG_NUMERIC:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ public final class Type implements Serializable {
private static final Type TYPE_BYTES = new Type(Type.Code.BYTES, null, null);
private static final Type TYPE_TIMESTAMP = new Type(Type.Code.TIMESTAMP, null, null);
private static final Type TYPE_DATE = new Type(Type.Code.DATE, null, null);
private static final Type TYPE_UUID = new Type(Type.Code.UUID, null, null);
private static final Type TYPE_ARRAY_BOOL = new Type(Type.Code.ARRAY, TYPE_BOOL, null);
private static final Type TYPE_ARRAY_INT64 = new Type(Type.Code.ARRAY, TYPE_INT64, null);
private static final Type TYPE_ARRAY_FLOAT32 = new Type(Type.Code.ARRAY, TYPE_FLOAT32, null);
Expand All @@ -66,6 +67,7 @@ public final class Type implements Serializable {
private static final Type TYPE_PG_BYTEA = new Type(Type.Code.PG_BYTEA, null, null);
private static final Type TYPE_PG_TIMESTAMPTZ = new Type(Type.Code.PG_TIMESTAMPTZ, null, null);
private static final Type TYPE_PG_DATE = new Type(Type.Code.PG_DATE, null, null);
private static final Type TYPE_PG_UUID = new Type(Type.Code.PG_UUID, null, null);
private static final Type TYPE_PG_ARRAY_BOOL = new Type(Type.Code.PG_ARRAY, TYPE_PG_BOOL, null);
private static final Type TYPE_PG_ARRAY_INT8 = new Type(Type.Code.PG_ARRAY, TYPE_PG_INT8, null);
private static final Type TYPE_PG_ARRAY_FLOAT4 =
Expand Down Expand Up @@ -160,6 +162,11 @@ public static Type date() {
return TYPE_DATE;
}

/** Returns the descriptor for the {@code UUID} type. */
public static Type uuid() {
return TYPE_UUID;
}

public static Type pgBool() {
return TYPE_PG_BOOL;
}
Expand Down Expand Up @@ -204,6 +211,10 @@ public static Type pgDate() {
return TYPE_PG_DATE;
}

public static Type pgUuid() {
return TYPE_PG_UUID;
}

public static Type pgCommitTimestamp() {
return TYPE_PG_COMMIT_TIMESTAMP;
}
Expand Down Expand Up @@ -317,6 +328,7 @@ public enum Code {
BYTES("BYTES", Dialect.GOOGLE_STANDARD_SQL),
TIMESTAMP("TIMESTAMP", Dialect.GOOGLE_STANDARD_SQL),
DATE("DATE", Dialect.GOOGLE_STANDARD_SQL),
UUID("UUID", Dialect.GOOGLE_STANDARD_SQL),
ARRAY("ARRAY", Dialect.GOOGLE_STANDARD_SQL),
STRUCT("STRUCT", Dialect.GOOGLE_STANDARD_SQL),
PG_BOOL("boolean", Dialect.POSTGRESQL),
Expand All @@ -330,6 +342,7 @@ public enum Code {
PG_BYTEA("bytea", Dialect.POSTGRESQL),
PG_TIMESTAMPTZ("timestamp with time zone", Dialect.POSTGRESQL),
PG_DATE("date", Dialect.POSTGRESQL),
PG_UUID("uuid", Dialect.POSTGRESQL),
PG_ARRAY("array", Dialect.POSTGRESQL),
PG_COMMIT_TIMESTAMP("spanner.commit_timestamp", Dialect.POSTGRESQL);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@ public void testTypeString_GoogleStandardSQL() {
assertEquals("JSON", Column.builder().name("col").type(Type.json()).autoBuild().typeString());
assertEquals(
"TOKENLIST", Column.builder().name("col").type(Type.tokenlist()).autoBuild().typeString());
assertEquals("UUID", Column.builder().name("col").type(Type.uuid()).autoBuild().typeString());
assertEquals(
"ARRAY<INT64>",
Column.builder().name("col").type(Type.array(Type.int64())).autoBuild().typeString());
Expand Down Expand Up @@ -151,6 +152,13 @@ public void testTypeString_PostgreSQL() {
.type(Type.pgCommitTimestamp())
.autoBuild()
.typeString());
assertEquals(
"uuid",
Column.builder(Dialect.POSTGRESQL)
.name("col")
.type(Type.pgUuid())
.autoBuild()
.typeString());
assertEquals(
"bigint[]",
Column.builder(Dialect.POSTGRESQL)
Expand Down Expand Up @@ -180,6 +188,7 @@ public void testParseSpannerType_GoogleStandardSQL() {
assertEquals(Type.json(), Column.builder().name("col").parseType("JSON").autoBuild().type());
assertEquals(
Type.tokenlist(), Column.builder().name("col").parseType("TOKENLIST").autoBuild().type());
assertEquals(Type.uuid(), Column.builder().name("col").parseType("UUID").autoBuild().type());
assertEquals(
Type.array(Type.int64()),
Column.builder().name("col").parseType("ARRAY<INT64>").autoBuild().type());
Expand Down Expand Up @@ -239,6 +248,9 @@ public void testParseSpannerType_PostgreSQL() {
.parseType("spanner.commit_timestamp")
.autoBuild()
.type());
assertEquals(
Type.pgUuid(),
Column.builder(Dialect.POSTGRESQL).name("col").parseType("uuid").autoBuild().type());
assertEquals(
Type.pgArray(Type.pgInt8()),
Column.builder(Dialect.POSTGRESQL).name("col").parseType("bigint[]").autoBuild().type());
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -561,6 +561,7 @@ public void testAvroValueToArrayBasic() {
assertThat(
AvroToValueMapper.getGsqlMap().keySet().stream()
.filter(t -> !t.getCode().equals(Code.ARRAY))
.filter(t -> !t.getCode().equals(Code.UUID))

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

why have we filtered out UUID type here? Shouldn't this test be passing for UUID arrays too?

.map(t -> t.toString())
.sorted()
.collect(Collectors.toList()))
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -432,4 +432,77 @@ public void mutationFromEventWithGeneratedColumn() throws ChangeEventConvertorEx
Truth.assertThat(actual.get("last_name").getAsString()).isEqualTo("B");
Truth.assertThat(actual.containsKey("full_name")).isFalse();
}

@Test
public void canConvertChangeEventWithUuidToPrimaryKey() throws Exception {
Ddl ddl =
Ddl.builder()
.createTable("UsersUuid")
.column("uuid_field")
.type(Type.uuid())
.endColumn()
.column("pg_uuid_field")
.type(Type.pgUuid())
.endColumn()
.primaryKey()
.asc("uuid_field")
.asc("pg_uuid_field")
.end()
.endTable()
.build();

JSONObject changeEvent = new JSONObject();
changeEvent.put("uuid_field", "550e8400-e29b-41d4-a716-446655440000");
changeEvent.put("pg_uuid_field", "123e4567-e89b-12d3-a456-426614174000");
changeEvent.put(Constants.EVENT_TABLE_NAME_KEY, "UsersUuid");
JsonNode ce = parseChangeEvent(changeEvent.toString());

Key key =
ChangeEventSpannerConvertor.changeEventToPrimaryKey(
"UsersUuid", ddl, ce, /* convertNameToLowerCase= */ true);

Iterable<Object> keyParts = key.getParts();
ArrayList<Object> expectedKeyParts = new ArrayList<>();
expectedKeyParts.add("550e8400-e29b-41d4-a716-446655440000");
expectedKeyParts.add("123e4567-e89b-12d3-a456-426614174000");

assertThat(keyParts, is(expectedKeyParts));
}

@Test
public void mutationFromEventWithUuid() throws Exception {
Ddl ddl =
Ddl.builder()
.createTable("UsersUuid")
.column("uuid_field")
.type(Type.uuid())
.endColumn()
.column("pg_uuid_field")
.type(Type.pgUuid())
.endColumn()
.primaryKey()
.asc("uuid_field")
.end()
.endTable()
.build();

JSONObject changeEvent = new JSONObject();
changeEvent.put("uuid_field", "550e8400-e29b-41d4-a716-446655440000");
changeEvent.put("pg_uuid_field", "123e4567-e89b-12d3-a456-426614174000");
changeEvent.put(Constants.EVENT_TABLE_NAME_KEY, "UsersUuid");
JsonNode ce = parseChangeEvent(changeEvent.toString());

Mutation mutation =
ChangeEventSpannerConvertor.mutationFromEvent(
ddl.table("UsersUuid"),
ce,
List.of("uuid_field", "pg_uuid_field"),
Set.of("uuid_field"));

Map<String, Value> actual = mutation.asMap();
Truth.assertThat(actual.get("uuid_field").getAsString())
.isEqualTo("550e8400-e29b-41d4-a716-446655440000");
Truth.assertThat(actual.get("pg_uuid_field").getAsString())
.isEqualTo("123e4567-e89b-12d3-a456-426614174000");
}
}
Loading
Loading