Skip to content

Commit 3daf9e3

Browse files
JasonHuang9527JasonHuang9527
authored
Merge pull request #17 from JasonHuang9527/hsuanhs/non_create_table_parsing
Only process create table statement and skip non create table statement in the cql file
2 parents 7251da0 + 5d386de commit 3daf9e3

14 files changed

Lines changed: 3741 additions & 944 deletions

‎README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ This CLI tool streamlines Cassandra to Spanner schema migration by automating th
1111

1212
## Note on the DDL Translation
1313

14-
* Only `CREATE TABLE` statements are supported. Other statements will result in a syntax error.
14+
* Only `CREATE TABLE` statements are processed. Other statements are ignored.
1515
* Table options in the `CREATE TABLE` statements are silently ignored.
1616
* Static columns are not supported (syntax error).
1717
* The following data types are not supported (syntax error): `Duration`, `Tuple`, and `Frozen`.

‎schema_converter.go‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -92,6 +92,8 @@ func checkGCPCredentials() error {
9292
//
9393
// TODO: Considering refactoring this function to a struct that has a getNextStmt function.
9494
// The streaming approach can reduce memory usage and it is crucial for handling large files.
95+
//
96+
// TODO: Address statements with semicolons (CREATE FUNCTION and BATCH DML).
9597
func parseCqlFile(filePath string) ([]string, error) {
9698
file, err := os.Open(filePath)
9799
if err != nil {
@@ -247,11 +249,15 @@ func main() {
247249
log.Printf("----------------------------------------------")
248250
for _, stmt := range stmts {
249251
log.Printf("[Cassandra statement]\n%s\n", stmt)
250-
spannerCreateTableStmt, err := translator.ToSpannerCreateTableStmt(stmt, flags.databaseID)
252+
spannerCreateTableStmt, ignored, err := translator.ToSpannerStmt(stmt, flags.databaseID)
251253
if err != nil {
252254
log.Fatalf("%v\n", err)
253255
}
254-
log.Printf("[Converted Spanner statement]\n%s\n", spannerCreateTableStmt)
256+
if ignored {
257+
log.Printf("[Skipping non-CREATE TABLE statement]\n")
258+
} else {
259+
log.Printf("[Converted Spanner statement]\n%s\n", spannerCreateTableStmt)
260+
}
255261
log.Printf("----------------------------------------------")
256262
spannerCreateTableStmts = append(spannerCreateTableStmts, spannerCreateTableStmt)
257263
}

‎translator/CqlLexer.g4‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -150,10 +150,13 @@ K_VARINT : 'VARINT'; // non-reserved
150150
K_WHERE : 'WHERE'; // reserved
151151
K_WITH : 'WITH'; // reserved
152152
K_WRITETIME : 'WRITETIME'; // non-reserved
153+
K_MATERIALIZED : 'MATERIALIZED'; // reserved
154+
K_VIEW : 'VIEW'; // reserved
153155

154156
// Punctuation
155157
// When adding a new punctuation, add entry to nonSemicolonToken in the CqlParser.g4.
156158
SEMICOLON : ';';
159+
SQUOTE : '\'';
157160
DQUOTE : '"';
158161
DOT : '.';
159162
COMMA : ',';

‎translator/CqlParser.g4‎

Lines changed: 57 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -29,12 +29,60 @@ root
2929

3030
cqlStatement
3131
: createTable
32+
| unsupportedStatement
3233
;
3334

3435
createTable
35-
: K_CREATE K_TABLE ifNotExist? tableName L_PAREN columnDefinitionList R_PAREN wihtTableOptions?
36+
: K_CREATE K_TABLE ifNotExists? tableName L_PAREN columnDefinitionList R_PAREN wihtTableOptions?
3637
;
3738

39+
// Reference: https://cassandra.apache.org/doc/4.0/cassandra/cql/definitions.html#statements.
40+
// The syntax is not precisely defined; it is a simplification.
41+
unsupportedStatement
42+
: K_USE keyspaceName
43+
| K_CREATE K_KEYSPACE ifNotExists? keyspaceName K_WITH nonSemicolonToken*
44+
| K_ALTER K_KEYSPACE keyspaceName K_WITH nonSemicolonToken*
45+
| K_DROP K_KEYSPACE ifExists? keyspaceName
46+
| K_ALTER K_TABLE tableName nonSemicolonToken*
47+
| K_DROP K_TABLE ifExists? tableName
48+
| K_TRUNCATE K_TABLE? tableName
49+
50+
| K_SELECT nonSemicolonToken*
51+
| K_INSERT K_INTO nonSemicolonToken*
52+
| K_UPDATE tableName nonSemicolonToken*
53+
| K_DELETE nonSemicolonToken*
54+
| K_BEGIN (K_UNLOGGED | K_COUNTER)? K_BATCH (nonSemicolonToken* SEMICOLON)+ K_APPLY K_BATCH SEMICOLON
55+
56+
| K_CREATE K_CUSTOM? K_INDEX nonSemicolonToken*
57+
| K_DROP K_INDEX ifExists? nonSemicolonToken*
58+
59+
| K_CREATE K_MATERIALIZED K_VIEW ifNotExists? nonSemicolonToken*
60+
| K_ALTER K_MATERIALIZED K_VIEW nonSemicolonToken*
61+
| K_DROP K_MATERIALIZED K_VIEW ifExists? nonSemicolonToken*
62+
63+
| K_CREATE K_ROLE ifNotExists? nonSemicolonToken*
64+
| K_ALTER K_ROLE nonSemicolonToken*
65+
| K_DROP K_ROLE nonSemicolonToken*
66+
| K_GRANT nonSemicolonToken*
67+
| K_REVOKE nonSemicolonToken*
68+
| K_LIST nonSemicolonToken*
69+
| K_CREATE K_USER nonSemicolonToken*
70+
| K_ALTER K_USER nonSemicolonToken*
71+
| K_DROP K_USER nonSemicolonToken*
72+
73+
| K_CREATE (K_OR K_REPLACE)? K_FUNCTION ifNotExists? nonSemicolonToken* K_AS SQUOTE nonSemicolonToken* SEMICOLON SQUOTE
74+
| K_DROP K_FUNCTION ifExists? nonSemicolonToken*
75+
| K_CREATE (K_OR K_REPLACE)? K_AGGREGATE ifNotExists? nonSemicolonToken*
76+
| K_DROP K_AGGREGATE ifExists? nonSemicolonToken*
77+
78+
| K_CREATE K_TYPE ifNotExists? nonSemicolonToken*
79+
| K_ALTER K_TYPE nonSemicolonToken*
80+
| K_DROP K_TYPE ifExists? nonSemicolonToken*
81+
82+
| K_CREATE K_TRIGGER ifNotExists? nonSemicolonToken*
83+
| K_DROP K_TRIGGER ifExists? nonSemicolonToken*
84+
;
85+
3886
columnDefinitionList
3987
: columnDefinition (COMMA columnDefinition)* (COMMA primaryKeyClause)?
4088
;
@@ -97,7 +145,11 @@ primaryKeyKeywords
97145
: K_PRIMARY K_KEY
98146
;
99147

100-
ifNotExist
148+
ifExists
149+
: K_IF K_EXISTS
150+
;
151+
152+
ifNotExists
101153
: K_IF K_NOT K_EXISTS
102154
;
103155

@@ -136,6 +188,7 @@ wihtTableOptions
136188

137189
nonSemicolonToken
138190
: keyword
191+
| SQUOTE
139192
| DQUOTE
140193
| DOT
141194
| COMMA
@@ -209,6 +262,8 @@ reservedKeyword
209262
| K_USING
210263
| K_WHERE
211264
| K_WITH
265+
| K_MATERIALIZED
266+
| K_VIEW
212267
;
213268

214269
nonReservedKeyword

‎translator/cqlparser/CqlLexer.interp‎

Lines changed: 10 additions & 1 deletion
Large diffs are not rendered by default.

‎translator/cqlparser/CqlLexer.tokens‎

Lines changed: 28 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -121,20 +121,23 @@ K_VARINT=120
121121
K_WHERE=121
122122
K_WITH=122
123123
K_WRITETIME=123
124-
SEMICOLON=124
125-
DQUOTE=125
126-
DOT=126
127-
COMMA=127
128-
L_PAREN=128
129-
R_PAREN=129
130-
L_ANGLE_BRACKET=130
131-
R_ANGLE_BRACKET=131
132-
IDENTIFIER=132
133-
IDENTIFIER_WITH_HYPHEN=133
134-
WS=134
135-
COMMENT=135
136-
MULTILINE_COMMENT=136
137-
UNKNOWN=137
124+
K_MATERIALIZED=124
125+
K_VIEW=125
126+
SEMICOLON=126
127+
SQUOTE=127
128+
DQUOTE=128
129+
DOT=129
130+
COMMA=130
131+
L_PAREN=131
132+
R_PAREN=132
133+
L_ANGLE_BRACKET=133
134+
R_ANGLE_BRACKET=134
135+
IDENTIFIER=135
136+
IDENTIFIER_WITH_HYPHEN=136
137+
WS=137
138+
COMMENT=138
139+
MULTILINE_COMMENT=139
140+
UNKNOWN=140
138141
'ADD'=1
139142
'AGGREGATE'=2
140143
'ALL'=3
@@ -258,11 +261,14 @@ UNKNOWN=137
258261
'WHERE'=121
259262
'WITH'=122
260263
'WRITETIME'=123
261-
';'=124
262-
'"'=125
263-
'.'=126
264-
','=127
265-
'('=128
266-
')'=129
267-
'<'=130
268-
'>'=131
264+
'MATERIALIZED'=124
265+
'VIEW'=125
266+
';'=126
267+
'\''=127
268+
'"'=128
269+
'.'=129
270+
','=130
271+
'('=131
272+
')'=132
273+
'<'=133
274+
'>'=134

‎translator/cqlparser/CqlParser.interp‎

Lines changed: 10 additions & 2 deletions
Large diffs are not rendered by default.

‎translator/cqlparser/CqlParser.tokens‎

Lines changed: 28 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -121,20 +121,23 @@ K_VARINT=120
121121
K_WHERE=121
122122
K_WITH=122
123123
K_WRITETIME=123
124-
SEMICOLON=124
125-
DQUOTE=125
126-
DOT=126
127-
COMMA=127
128-
L_PAREN=128
129-
R_PAREN=129
130-
L_ANGLE_BRACKET=130
131-
R_ANGLE_BRACKET=131
132-
IDENTIFIER=132
133-
IDENTIFIER_WITH_HYPHEN=133
134-
WS=134
135-
COMMENT=135
136-
MULTILINE_COMMENT=136
137-
UNKNOWN=137
124+
K_MATERIALIZED=124
125+
K_VIEW=125
126+
SEMICOLON=126
127+
SQUOTE=127
128+
DQUOTE=128
129+
DOT=129
130+
COMMA=130
131+
L_PAREN=131
132+
R_PAREN=132
133+
L_ANGLE_BRACKET=133
134+
R_ANGLE_BRACKET=134
135+
IDENTIFIER=135
136+
IDENTIFIER_WITH_HYPHEN=136
137+
WS=137
138+
COMMENT=138
139+
MULTILINE_COMMENT=139
140+
UNKNOWN=140
138141
'ADD'=1
139142
'AGGREGATE'=2
140143
'ALL'=3
@@ -258,11 +261,14 @@ UNKNOWN=137
258261
'WHERE'=121
259262
'WITH'=122
260263
'WRITETIME'=123
261-
';'=124
262-
'"'=125
263-
'.'=126
264-
','=127
265-
'('=128
266-
')'=129
267-
'<'=130
268-
'>'=131
264+
'MATERIALIZED'=124
265+
'VIEW'=125
266+
';'=126
267+
'\''=127
268+
'"'=128
269+
'.'=129
270+
','=130
271+
'('=131
272+
')'=132
273+
'<'=133
274+
'>'=134

0 commit comments

Comments
 (0)