Skip to content

Commit d97c555

Browse files
cliff-openainielm
cliff-openai
authored and
nielm
committed
feat: add LOCALITY GROUP and table OPTIONS support
implmentation taken from #217
1 parent be2c24e commit d97c555

10 files changed

Lines changed: 229 additions & 42 deletions

File tree

‎src/main/java/com/google/cloud/solutions/spannerddl/diff/DatabaseDefinition.java‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
import com.google.cloud.solutions.spannerddl.parser.ASTcheck_constraint;
2424
import com.google.cloud.solutions.spannerddl.parser.ASTcreate_change_stream_statement;
2525
import com.google.cloud.solutions.spannerddl.parser.ASTcreate_index_statement;
26+
import com.google.cloud.solutions.spannerddl.parser.ASTcreate_locality_group_statement;
2627
import com.google.cloud.solutions.spannerddl.parser.ASTcreate_or_replace_statement;
2728
import com.google.cloud.solutions.spannerddl.parser.ASTcreate_schema_statement;
2829
import com.google.cloud.solutions.spannerddl.parser.ASTcreate_search_index_statement;
@@ -66,6 +67,8 @@ public static DatabaseDefinition create(
6667
LinkedHashMap<String, ASTcreate_change_stream_statement> changeStreams = new LinkedHashMap<>();
6768
LinkedHashMap<String, String> alterDatabaseOptions = new LinkedHashMap<>();
6869
LinkedHashMap<String, ASTcreate_schema_statement> schemas = new LinkedHashMap<>();
70+
LinkedHashMap<String, ASTcreate_locality_group_statement> localityGroups =
71+
new LinkedHashMap<>();
6972

7073
for (ASTddl_statement ddlStatement : statements) {
7174
final SimpleNode statement = (SimpleNode) ddlStatement.jjtGetChild(0);
@@ -94,6 +97,10 @@ public static DatabaseDefinition create(
9497
((ASTcreate_search_index_statement) statement).getName(),
9598
(ASTcreate_search_index_statement) statement);
9699
break;
100+
case DdlParserTreeConstants.JJTCREATE_LOCALITY_GROUP_STATEMENT:
101+
ASTcreate_locality_group_statement lg = (ASTcreate_locality_group_statement) statement;
102+
localityGroups.put(lg.getNameOrDefault(), lg);
103+
break;
97104
case DdlParserTreeConstants.JJTCREATE_INDEX_STATEMENT:
98105
indexes.put(
99106
((ASTcreate_index_statement) statement).getIndexName(),
@@ -166,7 +173,8 @@ public static DatabaseDefinition create(
166173
ImmutableMap.copyOf(ttls),
167174
ImmutableMap.copyOf(changeStreams),
168175
ImmutableMap.copyOf(alterDatabaseOptions),
169-
ImmutableMap.copyOf(schemas));
176+
ImmutableMap.copyOf(schemas),
177+
ImmutableMap.copyOf(localityGroups));
170178
}
171179

172180
public abstract ImmutableMap<String, ASTcreate_table_statement> tablesInCreationOrder();
@@ -184,4 +192,6 @@ public static DatabaseDefinition create(
184192
abstract ImmutableMap<String, String> alterDatabaseOptions();
185193

186194
abstract ImmutableMap<String, ASTcreate_schema_statement> schemas();
195+
196+
abstract ImmutableMap<String, ASTcreate_locality_group_statement> localityGroups();
187197
}

‎src/main/java/com/google/cloud/solutions/spannerddl/diff/DdlDiff.java‎

Lines changed: 65 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
import com.google.cloud.solutions.spannerddl.parser.ASTcolumns;
2929
import com.google.cloud.solutions.spannerddl.parser.ASTcreate_change_stream_statement;
3030
import com.google.cloud.solutions.spannerddl.parser.ASTcreate_index_statement;
31+
import com.google.cloud.solutions.spannerddl.parser.ASTcreate_locality_group_statement;
3132
import com.google.cloud.solutions.spannerddl.parser.ASTcreate_or_replace_statement;
3233
import com.google.cloud.solutions.spannerddl.parser.ASTcreate_schema_statement;
3334
import com.google.cloud.solutions.spannerddl.parser.ASTcreate_search_index_statement;
@@ -107,6 +108,7 @@ public class DdlDiff {
107108
private final MapDifference<String, ASTcreate_search_index_statement> searchIndexDifferences;
108109
private final String databaseName; // for alter Database
109110
private final MapDifference<String, ASTcreate_schema_statement> schemaDifferences;
111+
private final MapDifference<String, ASTcreate_locality_group_statement> localityGroupDifferences;
110112

111113
private DdlDiff(DatabaseDefinition originalDb, DatabaseDefinition newDb, String databaseName)
112114
throws DdlDiffException {
@@ -126,6 +128,8 @@ private DdlDiff(DatabaseDefinition originalDb, DatabaseDefinition newDb, String
126128
this.searchIndexDifferences =
127129
Maps.difference(originalDb.searchIndexes(), newDb.searchIndexes());
128130
this.schemaDifferences = Maps.difference(originalDb.schemas(), newDb.schemas());
131+
this.localityGroupDifferences =
132+
Maps.difference(originalDb.localityGroups(), newDb.localityGroups());
129133

130134
if (!alterDatabaseOptionsDifferences.areEqual() && Strings.isNullOrEmpty(databaseName)) {
131135
// should never happen, but...
@@ -269,6 +273,13 @@ public List<String> generateDifferenceStatements(Map<String, Boolean> options)
269273
}
270274
}
271275

276+
// Create new locality groups
277+
for (ASTcreate_locality_group_statement lg :
278+
localityGroupDifferences.entriesOnlyOnRight().values()) {
279+
LOG.info("Creating new locality group: {}", lg.getNameOrDefault());
280+
output.add(lg.toString());
281+
}
282+
272283
// Alter existing tables, or error if not possible.
273284
for (ValueDifference<ASTcreate_table_statement> difference :
274285
tableDifferences.entriesDiffering().values()) {
@@ -277,6 +288,34 @@ public List<String> generateDifferenceStatements(Map<String, Boolean> options)
277288
generateAlterTableStatements(difference.leftValue(), difference.rightValue(), options));
278289
}
279290

291+
// Drop deleted locality groups.
292+
if (options.get(ALLOW_DROP_STATEMENTS_OPT)) {
293+
for (ASTcreate_locality_group_statement lg :
294+
localityGroupDifferences.entriesOnlyOnLeft().values()) {
295+
LOG.info("Dropping deleted locality group: {}", lg.getNameOrDefault());
296+
output.add("DROP LOCALITY GROUP " + lg.getNameOrDefault());
297+
}
298+
}
299+
300+
// update existing locality groups (options only)
301+
for (ValueDifference<ASTcreate_locality_group_statement> lgDiff :
302+
localityGroupDifferences.entriesDiffering().values()) {
303+
ASTcreate_locality_group_statement left = lgDiff.leftValue();
304+
ASTcreate_locality_group_statement right = lgDiff.rightValue();
305+
306+
// Only OPTIONS diffs are supported
307+
String updateText =
308+
generateOptionsClauseUpdates(left.getOptionsClause(), right.getOptionsClause());
309+
if (!Strings.isNullOrEmpty(updateText)) {
310+
output.add(
311+
"ALTER LOCALITY GROUP "
312+
+ right.getNameOrDefault()
313+
+ " SET OPTIONS ("
314+
+ updateText
315+
+ ")");
316+
}
317+
}
318+
280319
// create schemas
281320
for (ASTcreate_schema_statement schema : schemaDifferences.entriesOnlyOnRight().values()) {
282321
LOG.info("creating schema: {}", schema.getName());
@@ -395,19 +434,10 @@ public List<String> generateDifferenceStatements(Map<String, Boolean> options)
395434
if (!oldOptions.equals(newOptions)) {
396435

397436
// need to look at old and new options values individually
398-
399-
Map<String, String> oldOptionsKv =
400-
changedChangeStream.leftValue().getOptionsClause() == null
401-
? Map.of()
402-
: changedChangeStream.leftValue().getOptionsClause().getKeyValueMap();
403-
404-
Map<String, String> newOptionsKv =
405-
changedChangeStream.rightValue().getOptionsClause() == null
406-
? Map.of()
407-
: changedChangeStream.rightValue().getOptionsClause().getKeyValueMap();
408-
409-
String optionsDiff = generateOptionsUpdates(Maps.difference(oldOptionsKv, newOptionsKv));
410-
437+
String optionsDiff =
438+
generateOptionsClauseUpdates(
439+
changedChangeStream.leftValue().getOptionsClause(),
440+
changedChangeStream.rightValue().getOptionsClause());
411441
if (optionsDiff != null) {
412442
output.add(
413443
"ALTER CHANGE STREAM "
@@ -455,6 +485,7 @@ static List<String> generateAlterTableStatements(
455485
// - Add cols
456486
// - drop cols (if enabled)
457487
// - change on-delete action for interleaved
488+
// - update options
458489
// ALTER TABLE ALTER COLUMN can:
459490
// - change options on column
460491
// - change not null on column.
@@ -512,6 +543,13 @@ static List<String> generateAlterTableStatements(
512543
addColumnDiffs(left.getTableName(), alterStatements, columnDiff);
513544
}
514545

546+
// Table Options clause changes
547+
String updateText = generateOptionsClauseUpdates(left.getOptions(), right.getOptions());
548+
if (!Strings.isNullOrEmpty(updateText)) {
549+
alterStatements.add(
550+
"ALTER TABLE " + left.getTableName() + " SET OPTIONS (" + updateText + ")");
551+
}
552+
515553
return alterStatements;
516554
}
517555

@@ -591,16 +629,9 @@ private static void addColumnDiffs(
591629
}
592630

593631
// Update options.
594-
ASToptions_clause leftOptionsClause = columnDiff.leftValue().getOptionsClause();
595-
ASToptions_clause rightOptionsClause = columnDiff.rightValue().getOptionsClause();
596-
Map<String, String> leftOptions =
597-
leftOptionsClause == null ? Collections.emptyMap() : leftOptionsClause.getKeyValueMap();
598-
Map<String, String> rightOptions =
599-
rightOptionsClause == null ? Collections.emptyMap() : rightOptionsClause.getKeyValueMap();
600-
MapDifference<String, String> optionsDiff = Maps.difference(leftOptions, rightOptions);
601-
602-
String updateText = generateOptionsUpdates(optionsDiff);
603-
632+
String updateText =
633+
generateOptionsClauseUpdates(
634+
columnDiff.leftValue().getOptionsClause(), columnDiff.rightValue().getOptionsClause());
604635
if (!Strings.isNullOrEmpty(updateText)) {
605636
alterStatements.add(
606637
"ALTER TABLE "
@@ -639,6 +670,16 @@ private static void addColumnDiffs(
639670
}
640671
}
641672

673+
private static String generateOptionsClauseUpdates(
674+
ASToptions_clause leftOptionsClause, ASToptions_clause rightOptionsClause) {
675+
Map<String, String> leftOptions =
676+
leftOptionsClause == null ? Collections.emptyMap() : leftOptionsClause.getKeyValueMap();
677+
Map<String, String> rightOptions =
678+
rightOptionsClause == null ? Collections.emptyMap() : rightOptionsClause.getKeyValueMap();
679+
MapDifference<String, String> optionsDiff = Maps.difference(leftOptions, rightOptions);
680+
return generateOptionsUpdates(optionsDiff);
681+
}
682+
642683
private static String generateOptionsUpdates(MapDifference<String, String> optionsDiff) {
643684

644685
if (optionsDiff.areEqual()) {
@@ -820,6 +861,7 @@ public static List<ASTddl_statement> parseDdl(String original, boolean parseAnno
820861
case DdlParserTreeConstants.JJTALTER_DATABASE_STATEMENT:
821862
case DdlParserTreeConstants.JJTCREATE_CHANGE_STREAM_STATEMENT:
822863
case DdlParserTreeConstants.JJTCREATE_SEARCH_INDEX_STATEMENT:
864+
case DdlParserTreeConstants.JJTCREATE_LOCALITY_GROUP_STATEMENT:
823865
case DdlParserTreeConstants.JJTCREATE_PROTO_BUNDLE_STATEMENT:
824866
case DdlParserTreeConstants.JJTALTER_PROTO_BUNDLE_STATEMENT:
825867
// no-op - allowed

‎src/main/java/com/google/cloud/solutions/spannerddl/parser/ASTcreate_locality_group_statement.java‎

Lines changed: 39 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -15,15 +15,51 @@
1515
*/
1616
package com.google.cloud.solutions.spannerddl.parser;
1717

18+
import com.google.cloud.solutions.spannerddl.diff.AstTreeUtils;
19+
import com.google.common.base.Joiner;
20+
import com.google.common.collect.ImmutableSet;
21+
import java.util.Objects;
22+
1823
public class ASTcreate_locality_group_statement extends SimpleNode {
1924
public ASTcreate_locality_group_statement(int id) {
2025
super(id);
21-
throw new UnsupportedOperationException("Not Implemented");
2226
}
2327

2428
public ASTcreate_locality_group_statement(DdlParser p, int id) {
2529
super(p, id);
26-
throw new UnsupportedOperationException("Not Implemented");
30+
}
31+
32+
public String getNameOrDefault() {
33+
ASTname name = AstTreeUtils.getOptionalChildByType(children, ASTname.class);
34+
if (name == null) {
35+
throw new IllegalArgumentException("Cannot create DEFAULT locality group");
36+
}
37+
return name.toString();
38+
}
39+
40+
@Override
41+
public int hashCode() {
42+
return toString().hashCode();
43+
}
44+
45+
@Override
46+
public String toString() {
47+
AstTreeUtils.validateChildrenClasses(
48+
children,
49+
ImmutableSet.of(
50+
ASTname.class, ASTdefaultt.class, ASTif_not_exists.class, ASToptions_clause.class));
51+
52+
return Joiner.on(" ")
53+
.skipNulls()
54+
.join(
55+
"CREATE LOCALITY GROUP",
56+
Objects.toString(
57+
AstTreeUtils.getOptionalChildByType(children, ASTif_not_exists.class), null),
58+
getNameOrDefault(),
59+
getOptionsClause());
60+
}
61+
62+
public ASToptions_clause getOptionsClause() {
63+
return AstTreeUtils.getOptionalChildByType(children, ASToptions_clause.class);
2764
}
2865
}
29-
/* ParserGeneratorCC - OriginalChecksum=71ef8285a44bd93d58a5f657d4fd3b78 (do not edit this line) */

‎src/main/java/com/google/cloud/solutions/spannerddl/parser/ASTcreate_table_statement.java‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -135,7 +135,12 @@ public String toStringOptionalExistClause(boolean includeExists) {
135135
getOptionalChildByType(children, ASTtable_interleave_clause.class),
136136
(withConstraints
137137
? getOptionalChildByType(children, ASTrow_deletion_policy_clause.class)
138-
: null)));
138+
: null),
139+
getOptions()));
140+
}
141+
142+
public ASToptions_clause getOptions() {
143+
return getOptionalChildByType(children, ASToptions_clause.class);
139144
}
140145

141146
private void validateChildren() {
@@ -151,9 +156,8 @@ private void validateChildren() {
151156
// ASTsynonym_clause.class,
152157
ASTannotation.class,
153158
ASTtable_interleave_clause.class,
154-
ASTrow_deletion_policy_clause.class
155-
// ASToptions_clause.class
156-
));
159+
ASTrow_deletion_policy_clause.class,
160+
ASToptions_clause.class));
157161
}
158162

159163
@Override

‎src/test/java/com/google/cloud/solutions/spannerddl/diff/DdlDiffFromFilesTest.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -126,7 +126,8 @@ public void compareDddNoDrops() throws DdlDiffException {
126126
expectedDiff.stream()
127127
.filter(
128128
statement ->
129-
!statement.matches(".*DROP (SCHEMA|TABLE|COLUMN|CHANGE STREAM|SEARCH INDEX).*"))
129+
!statement.matches(
130+
".*DROP (SCHEMA|TABLE|COLUMN|CHANGE STREAM|SEARCH INDEX|LOCALITY GROUP).*"))
130131
.collect(Collectors.toList());
131132

132133
// remove any drop indexes from the expectedResults if they do not have an equivalent

‎src/test/resources/ddlParserUnsupported.txt‎

Lines changed: 3 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -183,11 +183,11 @@ ALTER PLACEMENT p1 SET OPTIONS (instance_partition = 'us-central1')
183183

184184
CREATE FUNCTION udf_simple_add(a INT64, b INT64) RETURNS INT64 AS (a + b)
185185

186-
== Test 22a create locality_group
186+
== Test 22a create default locality_group
187187

188-
create LOCALITY GROUP lg OPTIONS (storage = 'hdd', ssd_to_hdd_spill_timespan = '1h')
188+
create LOCALITY GROUP DEFAULT OPTIONS (storage = 'hdd', ssd_to_hdd_spill_timespan = '1h')
189189

190-
== Test 22b alter locality_group
190+
== Test 22c alter locality_group
191191

192192
ALTER LOCALITY GROUP lg SET OPTIONS (storage = 'hdd', ssd_to_hdd_spill_timespan = '1h')
193193

@@ -236,13 +236,6 @@ CREATE TABLE test1 (
236236
value INT64 DEFAULT (keycol * 100 + PI ( ))
237237
)
238238

239-
== Test 29 create table options clause
240-
241-
CREATE TABLE test1 (
242-
keycol INT64,
243-
value INT64 DEFAULT (keycol * 100 + PI ( ))
244-
) PRIMARY KEY (keycol ASC), OPTIONS ( key = 'value')
245-
246239
== Test 30 create table column on update clause
247240

248241
CREATE TABLE test1 (

‎src/test/resources/ddlParserValidation.txt‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -185,4 +185,15 @@ CREATE SCHEMA IF NOT EXISTS schema_name
185185

186186
CREATE SCHEMA IF NOT EXISTS schema_name OPTIONS (opt1=TRUE)
187187

188+
== Test 19 create locality_group
189+
190+
CREATE LOCALITY GROUP lg OPTIONS (storage='hdd',ssd_to_hdd_spill_timespan='1h')
191+
192+
== Test 20 create table with options clause
193+
194+
CREATE TABLE test1 (
195+
keycol INT64,
196+
value INT64
197+
) PRIMARY KEY (keycol ASC), OPTIONS (key='value')
198+
188199
==

‎src/test/resources/expectedDdlDiff.txt‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -350,4 +350,34 @@ DROP SCHEMA schema2
350350
CREATE SCHEMA schema3
351351
CREATE TABLE schema3.table1 ( col1 INT64 ) PRIMARY KEY (col1)
352352

353+
== test 68 Add locality_group
354+
355+
CREATE LOCALITY GROUP lg2 OPTIONS (storage='hdd')
356+
357+
== test 69 drop locality group
358+
359+
DROP LOCALITY GROUP lg2
360+
361+
== test 70 change locality_group
362+
363+
ALTER LOCALITY GROUP lg1 SET OPTIONS (ssd_to_hdd_spill_timespan='10d',storage='ssd')
364+
365+
== test 71 add locality_group to table
366+
367+
ALTER TABLE mytable SET OPTIONS (locality_group='lg1')
368+
369+
== test 72 remove locality_group from table and drop
370+
371+
ALTER TABLE mytable SET OPTIONS (locality_group=NULL)
372+
DROP LOCALITY GROUP lg1
373+
374+
== test 73 change locality_group in table
375+
376+
ALTER TABLE mytable SET OPTIONS (locality_group='lg2')
377+
378+
== test 74 table with options
379+
380+
ALTER TABLE mytable SET OPTIONS (droppedKey=NULL,newKey='value2')
381+
353382
==
383+

0 commit comments

Comments
 (0)