Skip to content

Commit e19264c

Browse files
nielmnielm
authored
feat: Support for CREATE SCHEMA statements (#197)
This only allows for diffing DDLs with CREATE SCHEMA statements. support for ROLEs and GRANTS are not included in this change
1 parent 6b9032d commit e19264c

11 files changed

Lines changed: 229 additions & 15 deletions

File tree

‎src/main/java/com/google/cloud/solutions/spannerddl/diff/DatabaseDefinition.java‎

Lines changed: 24 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,8 @@
2323
import com.google.cloud.solutions.spannerddl.parser.ASTcheck_constraint;
2424
import com.google.cloud.solutions.spannerddl.parser.ASTcreate_change_stream_statement;
2525
import com.google.cloud.solutions.spannerddl.parser.ASTcreate_index_statement;
26+
import com.google.cloud.solutions.spannerddl.parser.ASTcreate_or_replace_statement;
27+
import com.google.cloud.solutions.spannerddl.parser.ASTcreate_schema_statement;
2628
import com.google.cloud.solutions.spannerddl.parser.ASTcreate_search_index_statement;
2729
import com.google.cloud.solutions.spannerddl.parser.ASTcreate_table_statement;
2830
import com.google.cloud.solutions.spannerddl.parser.ASTddl_statement;
@@ -61,6 +63,7 @@ public static DatabaseDefinition create(List<ASTddl_statement> statements) {
6163
LinkedHashMap<String, ASTrow_deletion_policy_clause> ttls = new LinkedHashMap<>();
6264
LinkedHashMap<String, ASTcreate_change_stream_statement> changeStreams = new LinkedHashMap<>();
6365
LinkedHashMap<String, String> alterDatabaseOptions = new LinkedHashMap<>();
66+
LinkedHashMap<String, ASTcreate_schema_statement> schemas = new LinkedHashMap<>();
6467

6568
for (ASTddl_statement ddlStatement : statements) {
6669
final SimpleNode statement = (SimpleNode) ddlStatement.jjtGetChild(0);
@@ -124,6 +127,23 @@ public static DatabaseDefinition create(List<ASTddl_statement> statements) {
124127
((ASTcreate_change_stream_statement) statement).getName(),
125128
(ASTcreate_change_stream_statement) statement);
126129
break;
130+
131+
case DdlParserTreeConstants.JJTCREATE_OR_REPLACE_STATEMENT:
132+
// can be one of several types.
133+
switch (((ASTcreate_or_replace_statement) statement).getSchemaObject().getId()) {
134+
case DdlParserTreeConstants.JJTCREATE_SCHEMA_STATEMENT:
135+
schemas.put(
136+
((ASTcreate_schema_statement)
137+
((ASTcreate_or_replace_statement) statement).getSchemaObject())
138+
.getName(),
139+
(ASTcreate_schema_statement)
140+
((ASTcreate_or_replace_statement) statement).getSchemaObject());
141+
break;
142+
default:
143+
throw new IllegalArgumentException(
144+
"Unsupported statement: " + AstTreeUtils.tokensToString(ddlStatement));
145+
}
146+
break;
127147
default:
128148
throw new IllegalArgumentException(
129149
"Unsupported statement: " + AstTreeUtils.tokensToString(ddlStatement));
@@ -136,7 +156,8 @@ public static DatabaseDefinition create(List<ASTddl_statement> statements) {
136156
ImmutableMap.copyOf(constraints),
137157
ImmutableMap.copyOf(ttls),
138158
ImmutableMap.copyOf(changeStreams),
139-
ImmutableMap.copyOf(alterDatabaseOptions));
159+
ImmutableMap.copyOf(alterDatabaseOptions),
160+
ImmutableMap.copyOf(schemas));
140161
}
141162

142163
public abstract ImmutableMap<String, ASTcreate_table_statement> tablesInCreationOrder();
@@ -152,4 +173,6 @@ public static DatabaseDefinition create(List<ASTddl_statement> statements) {
152173
abstract ImmutableMap<String, ASTcreate_change_stream_statement> changeStreams();
153174

154175
abstract ImmutableMap<String, String> alterDatabaseOptions();
176+
177+
abstract ImmutableMap<String, ASTcreate_schema_statement> schemas();
155178
}

‎src/main/java/com/google/cloud/solutions/spannerddl/diff/DdlDiff.java‎

Lines changed: 41 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,8 @@
2727
import com.google.cloud.solutions.spannerddl.parser.ASTcolumn_type;
2828
import com.google.cloud.solutions.spannerddl.parser.ASTcreate_change_stream_statement;
2929
import com.google.cloud.solutions.spannerddl.parser.ASTcreate_index_statement;
30+
import com.google.cloud.solutions.spannerddl.parser.ASTcreate_or_replace_statement;
31+
import com.google.cloud.solutions.spannerddl.parser.ASTcreate_schema_statement;
3032
import com.google.cloud.solutions.spannerddl.parser.ASTcreate_search_index_statement;
3133
import com.google.cloud.solutions.spannerddl.parser.ASTcreate_table_statement;
3234
import com.google.cloud.solutions.spannerddl.parser.ASTddl_statement;
@@ -100,6 +102,7 @@ public class DdlDiff {
100102
private final MapDifference<String, ASTcreate_change_stream_statement> changeStreamDifferences;
101103
private final MapDifference<String, ASTcreate_search_index_statement> searchIndexDifferences;
102104
private final String databaseName; // for alter Database
105+
private final MapDifference<String, ASTcreate_schema_statement> schemaDifferences;
103106

104107
private DdlDiff(DatabaseDefinition originalDb, DatabaseDefinition newDb, String databaseName)
105108
throws DdlDiffException {
@@ -118,6 +121,7 @@ private DdlDiff(DatabaseDefinition originalDb, DatabaseDefinition newDb, String
118121
Maps.difference(originalDb.changeStreams(), newDb.changeStreams());
119122
this.searchIndexDifferences =
120123
Maps.difference(originalDb.searchIndexes(), newDb.searchIndexes());
124+
this.schemaDifferences = Maps.difference(originalDb.schemas(), newDb.schemas());
121125

122126
if (!alterDatabaseOptionsDifferences.areEqual() && Strings.isNullOrEmpty(databaseName)) {
123127
// should never happen, but...
@@ -160,6 +164,12 @@ public List<String> generateDifferenceStatements(Map<String, Boolean> options)
160164
+ Joiner.on(", ").join(constraintDifferences.entriesDiffering().keySet()));
161165
}
162166

167+
if (!schemaDifferences.entriesDiffering().isEmpty()) {
168+
throw new DdlDiffException(
169+
"At least one schema differs but ALTER SCHEMA is not supported"
170+
+ Joiner.on(", ").join(schemaDifferences.entriesDiffering().keySet()));
171+
}
172+
163173
// check for modified Alter Database statements
164174
if (!alterDatabaseOptionsDifferences.areEqual()) {
165175
String optionsUpdates = generateOptionsUpdates(alterDatabaseOptionsDifferences);
@@ -247,6 +257,14 @@ public List<String> generateDifferenceStatements(Map<String, Boolean> options)
247257
}
248258
}
249259

260+
// Drop schemas
261+
if (options.get(ALLOW_DROP_STATEMENTS_OPT)) {
262+
for (ASTcreate_schema_statement schema : schemaDifferences.entriesOnlyOnLeft().values()) {
263+
LOG.info("Dropping schema: {}", schema.getName());
264+
output.add("DROP SCHEMA " + schema.getName());
265+
}
266+
}
267+
250268
// Alter existing tables, or error if not possible.
251269
for (ValueDifference<ASTcreate_table_statement> difference :
252270
tableDifferences.entriesDiffering().values()) {
@@ -255,6 +273,12 @@ public List<String> generateDifferenceStatements(Map<String, Boolean> options)
255273
generateAlterTableStatements(difference.leftValue(), difference.rightValue(), options));
256274
}
257275

276+
// create schemas
277+
for (ASTcreate_schema_statement schema : schemaDifferences.entriesOnlyOnRight().values()) {
278+
LOG.info("creating schema: {}", schema.getName());
279+
output.add(schema.toString());
280+
}
281+
258282
// Create new tables. Must be done in the order of creation in the new DDL.
259283
for (Map.Entry<String, ASTcreate_table_statement> newTableEntry :
260284
newDb.tablesInCreationOrder().entrySet()) {
@@ -755,8 +779,8 @@ public static List<ASTddl_statement> parseDdl(String original, boolean parseAnno
755779
"Unsupported statement:\n"
756780
+ statement
757781
+ "\n"
758-
+ "Can only create diffs from 'CREATE TABLE, CREATE INDEX and 'ALTER TABLE"
759-
+ " table_name ADD [constraint|row deletion policy]' DDL statements");
782+
+ "ALTER TABLE statements only support 'ADD [constraint|row deletion"
783+
+ " policy]'");
760784
}
761785
if (alterTableStatement.jjtGetChild(1) instanceof ASTforeign_key
762786
&& ((ASTforeign_key) alterTableStatement.jjtGetChild(1))
@@ -791,14 +815,24 @@ public static List<ASTddl_statement> parseDdl(String original, boolean parseAnno
791815
case DdlParserTreeConstants.JJTALTER_DATABASE_STATEMENT:
792816
case DdlParserTreeConstants.JJTCREATE_CHANGE_STREAM_STATEMENT:
793817
case DdlParserTreeConstants.JJTCREATE_SEARCH_INDEX_STATEMENT:
794-
// no-op
818+
// no-op - allowed
819+
break;
820+
case DdlParserTreeConstants.JJTCREATE_OR_REPLACE_STATEMENT:
821+
// can be one of several types.
822+
switch (((ASTcreate_or_replace_statement) ddlStatement.jjtGetChild(0))
823+
.getSchemaObject()
824+
.getId()) {
825+
case DdlParserTreeConstants.JJTCREATE_SCHEMA_STATEMENT:
826+
// no-op - allowed
827+
break;
828+
default:
829+
throw new IllegalArgumentException(
830+
"Unsupported statement for creating diffs:\n" + statement);
831+
}
795832
break;
796833
default:
797834
throw new IllegalArgumentException(
798-
"Unsupported statement:\n"
799-
+ statement
800-
+ "\nCan only create diffs from 'CREATE TABLE, CREATE INDEX, and "
801-
+ "'ALTER TABLE table_name ADD CONSTRAINT' DDL statements");
835+
"Unsupported statement for creating diffs:\n" + statement);
802836
}
803837
ddlStatements.add(ddlStatement);
804838
} catch (ParseException e) {

‎src/main/java/com/google/cloud/solutions/spannerddl/parser/ASTcreate_or_replace_statement.java‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,11 @@
1515
*/
1616
package com.google.cloud.solutions.spannerddl.parser;
1717

18+
import static com.google.cloud.solutions.spannerddl.diff.AstTreeUtils.getOptionalChildByType;
19+
20+
import java.util.Objects;
21+
import java.util.stream.Stream;
22+
1823
public class ASTcreate_or_replace_statement extends SimpleNode {
1924
public ASTcreate_or_replace_statement(int id) {
2025
super(id);
@@ -23,4 +28,30 @@ public ASTcreate_or_replace_statement(int id) {
2328
public ASTcreate_or_replace_statement(DdlParser p, int id) {
2429
super(p, id);
2530
}
31+
32+
public SimpleNode getSchemaObject() {
33+
return Stream.of(
34+
getOptionalChildByType(children, ASTcreate_view_statement.class),
35+
getOptionalChildByType(children, ASTcreate_model_statement.class),
36+
getOptionalChildByType(children, ASTcreate_schema_statement.class))
37+
.filter(Objects::nonNull)
38+
.findFirst()
39+
.get();
40+
}
41+
42+
@Override
43+
public String toString() {
44+
return getSchemaObject().toString();
45+
}
46+
47+
@Override
48+
public int hashCode() {
49+
return getSchemaObject().hashCode();
50+
}
51+
52+
@Override
53+
public boolean equals(Object obj) {
54+
return obj instanceof ASTcreate_or_replace_statement
55+
&& getSchemaObject().equals(((ASTcreate_or_replace_statement) obj).getSchemaObject());
56+
}
2657
}

‎src/main/java/com/google/cloud/solutions/spannerddl/parser/ASTcreate_schema_statement.java‎

Lines changed: 44 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -15,15 +15,56 @@
1515
*/
1616
package com.google.cloud.solutions.spannerddl.parser;
1717

18-
// TODO
18+
import static com.google.cloud.solutions.spannerddl.diff.AstTreeUtils.getChildByType;
19+
import static com.google.cloud.solutions.spannerddl.diff.AstTreeUtils.getOptionalChildByType;
20+
21+
import com.google.cloud.solutions.spannerddl.diff.AstTreeUtils;
22+
import com.google.common.base.Joiner;
23+
import com.google.common.collect.ImmutableSet;
24+
1925
public class ASTcreate_schema_statement extends SimpleNode {
2026
public ASTcreate_schema_statement(int id) {
2127
super(id);
22-
throw new UnsupportedOperationException("Not Implemented");
2328
}
2429

2530
public ASTcreate_schema_statement(DdlParser p, int id) {
2631
super(p, id);
27-
throw new UnsupportedOperationException("Not Implemented");
32+
}
33+
34+
private void validateChildren() {
35+
AstTreeUtils.validateChildrenClasses(
36+
children, ImmutableSet.of(ASTif_not_exists.class, ASTname.class, ASToptions_clause.class));
37+
}
38+
39+
@Override
40+
public String toString() {
41+
return toStringOptionalExistClause(true);
42+
}
43+
44+
public String getName() {
45+
return getChildByType(children, ASTname.class).toString();
46+
}
47+
48+
/** Create string version, optionally including the IF NOT EXISTS clause */
49+
public String toStringOptionalExistClause(boolean includeExists) {
50+
validateChildren();
51+
return Joiner.on(" ")
52+
.skipNulls()
53+
.join(
54+
"CREATE",
55+
"SCHEMA",
56+
(includeExists ? getOptionalChildByType(children, ASTif_not_exists.class) : null),
57+
getName(),
58+
AstTreeUtils.getOptionalChildByType(children, ASToptions_clause.class));
59+
}
60+
61+
@Override
62+
public boolean equals(Object obj) {
63+
return (obj instanceof ASTcreate_schema_statement) && toString().equals(obj.toString());
64+
}
65+
66+
@Override
67+
public int hashCode() {
68+
return toString().hashCode();
2869
}
2970
}

‎src/test/java/com/google/cloud/solutions/spannerddl/diff/DdlDiffFromFilesTest.java‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -78,12 +78,13 @@ public void compareDddTextFiles() throws IOException {
7878

7979
// TEST PART 2: with allowDropStatements=false
8080

81-
// build an expectedResults without any column or table drops.
81+
// build an expectedResults without any drops.
8282
List<String> expectedDiffNoDrops =
8383
expectedDiff.stream()
8484
.filter(
8585
statement ->
86-
!statement.matches(".*DROP (TABLE|COLUMN|CHANGE STREAM|SEARCH INDEX).*"))
86+
!statement.matches(
87+
".*DROP (SCHEMA|TABLE|COLUMN|CHANGE STREAM|SEARCH INDEX).*"))
8788
.collect(Collectors.toList());
8889

8990
// remove any drop indexes from the expectedResults if they do not have an equivalent

‎src/test/java/com/google/cloud/solutions/spannerddl/parser/DDLParserFromFileTest.java‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -35,9 +35,10 @@ public void validateDDLfromFile() throws IOException {
3535
assertWithMessage("Mismatch for section " + segmentName + ":")
3636
.that(parsedStatement.toString())
3737
.isEqualTo(ddlStatement);
38-
} catch (ParseException e) {
38+
} catch (Throwable e) {
39+
e.printStackTrace(System.err);
3940
fail(
40-
"Failed to parse section: '"
41+
"Exception when parsing section: '"
4142
+ segmentName
4243
+ "': "
4344
+ e

‎src/test/java/com/google/cloud/solutions/spannerddl/parser/DDLParserTest.java‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -178,6 +178,14 @@ public void doNotNormalizeSQLExpressions() throws ParseException {
178178
assertThat(statement).isEqualTo(statement2);
179179
}
180180

181+
@Test
182+
public void ignoreCreateOrReplace() throws ParseException {
183+
ASTcreate_or_replace_statement statement =
184+
(ASTcreate_or_replace_statement)
185+
parse("CREATE OR REPLACE SCHEMA schema_name").jjtGetChild(0);
186+
assertThat(statement.toString()).isEqualTo("CREATE SCHEMA schema_name");
187+
}
188+
181189
private static void parseCheckingParseException(String ddlStatement, String exceptionContains) {
182190
ParseException e =
183191
assertThrows(ParseException.class, () -> parseAndVerifyToString(ddlStatement));

‎src/test/resources/ddlParserValidation.txt‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -173,4 +173,16 @@ OPTIONS (sort_order_sharding=TRUE)
173173

174174
CREATE CHANGE STREAM usage FOR table1
175175

176+
== Test 17 create schema
177+
178+
CREATE SCHEMA schema_name
179+
180+
== Test 18 create schema if not exists
181+
182+
CREATE SCHEMA IF NOT EXISTS schema_name
183+
184+
== Test 18 create schema with options
185+
186+
CREATE SCHEMA IF NOT EXISTS schema_name OPTIONS (opt1=TRUE)
187+
176188
==

‎src/test/resources/expectedDdlDiff.txt‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -335,4 +335,19 @@ ALTER SEARCH INDEX AlbumsIndex DROP STORED COLUMN scol1
335335

336336
ALTER TABLE AlbumsIndex ADD COLUMN new_col STRING(255)
337337

338+
== TEST 65 Adding new schema
339+
340+
CREATE SCHEMA schema2
341+
342+
== TEST 66 Dropping schema
343+
344+
DROP SCHEMA schema2
345+
346+
== TEST 67 Creating and dropping schema with tables in correct order
347+
348+
DROP TABLE schema2.table1
349+
DROP SCHEMA schema2
350+
CREATE SCHEMA schema3
351+
CREATE TABLE schema3.table1 ( col1 INT64 ) PRIMARY KEY (col1 ASC)
352+
338353
==

‎src/test/resources/newDdl.txt‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -540,4 +540,27 @@ CREATE TABLE AlbumsIndex (
540540
new_col STRING(255),
541541
) PRIMARY KEY (id)
542542

543+
== TEST 65 Adding new schema
544+
545+
CREATE SCHEMA schema1;
546+
CREATE SCHEMA schema2;
547+
548+
== TEST 66 Dropping schema
549+
550+
CREATE SCHEMA schema1;
551+
552+
== TEST 67 Creating and dropping schema with tables in correct order
553+
554+
CREATE SCHEMA schema1;
555+
556+
CREATE TABLE schema1.table1 (
557+
col1 INT64,
558+
)PRIMARY KEY (col1);
559+
560+
CREATE SCHEMA schema3;
561+
562+
CREATE TABLE schema3.table1 (
563+
col1 INT64,
564+
)PRIMARY KEY (col1);
565+
543566
==

0 commit comments

Comments
 (0)