Skip to content

Commit 898e780

Browse files
kyleconroykyleconroy
authored
Implement all TODO test cases for ClickHouse SQL parser (#5)
1 parent 1c72418 commit 898e780

File tree

5 files changed

+1125
-174
lines changed

5 files changed

+1125
-174
lines changed

‎ast/ast.go‎

Lines changed: 174 additions & 60 deletions
Original file line numberDiff line numberDiff line change
@@ -28,9 +28,10 @@ type Expression interface {
2828

2929
// SelectWithUnionQuery represents a SELECT query possibly with UNION.
3030
type SelectWithUnionQuery struct {
31-
Position token.Position `json:"-"`
32-
Selects []Statement `json:"selects"`
33-
UnionAll bool `json:"union_all,omitempty"`
31+
Position token.Position `json:"-"`
32+
Selects []Statement `json:"selects"`
33+
UnionAll bool `json:"union_all,omitempty"`
34+
UnionModes []string `json:"union_modes,omitempty"` // "ALL", "DISTINCT", or "" for each union
3435
}
3536

3637
func (s *SelectWithUnionQuery) Pos() token.Position { return s.Position }
@@ -39,25 +40,57 @@ func (s *SelectWithUnionQuery) statementNode() {}
3940

4041
// SelectQuery represents a SELECT statement.
4142
type SelectQuery struct {
42-
Position token.Position `json:"-"`
43-
With []Expression `json:"with,omitempty"`
44-
Distinct bool `json:"distinct,omitempty"`
45-
Top Expression `json:"top,omitempty"`
46-
Columns []Expression `json:"columns"`
47-
From *TablesInSelectQuery `json:"from,omitempty"`
48-
PreWhere Expression `json:"prewhere,omitempty"`
49-
Where Expression `json:"where,omitempty"`
50-
GroupBy []Expression `json:"group_by,omitempty"`
51-
WithRollup bool `json:"with_rollup,omitempty"`
52-
WithTotals bool `json:"with_totals,omitempty"`
53-
Having Expression `json:"having,omitempty"`
54-
OrderBy []*OrderByElement `json:"order_by,omitempty"`
55-
Limit Expression `json:"limit,omitempty"`
56-
Offset Expression `json:"offset,omitempty"`
57-
Settings []*SettingExpr `json:"settings,omitempty"`
58-
Format *Identifier `json:"format,omitempty"`
43+
Position token.Position `json:"-"`
44+
With []Expression `json:"with,omitempty"`
45+
Distinct bool `json:"distinct,omitempty"`
46+
Top Expression `json:"top,omitempty"`
47+
Columns []Expression `json:"columns"`
48+
From *TablesInSelectQuery `json:"from,omitempty"`
49+
ArrayJoin *ArrayJoinClause `json:"array_join,omitempty"`
50+
PreWhere Expression `json:"prewhere,omitempty"`
51+
Where Expression `json:"where,omitempty"`
52+
GroupBy []Expression `json:"group_by,omitempty"`
53+
WithRollup bool `json:"with_rollup,omitempty"`
54+
WithTotals bool `json:"with_totals,omitempty"`
55+
Having Expression `json:"having,omitempty"`
56+
Window []*WindowDefinition `json:"window,omitempty"`
57+
OrderBy []*OrderByElement `json:"order_by,omitempty"`
58+
Limit Expression `json:"limit,omitempty"`
59+
Offset Expression `json:"offset,omitempty"`
60+
Settings []*SettingExpr `json:"settings,omitempty"`
61+
IntoOutfile *IntoOutfileClause `json:"into_outfile,omitempty"`
62+
Format *Identifier `json:"format,omitempty"`
63+
}
64+
65+
// ArrayJoinClause represents an ARRAY JOIN clause.
66+
type ArrayJoinClause struct {
67+
Position token.Position `json:"-"`
68+
Left bool `json:"left,omitempty"`
69+
Columns []Expression `json:"columns"`
70+
}
71+
72+
func (a *ArrayJoinClause) Pos() token.Position { return a.Position }
73+
func (a *ArrayJoinClause) End() token.Position { return a.Position }
74+
75+
// WindowDefinition represents a named window definition in the WINDOW clause.
76+
type WindowDefinition struct {
77+
Position token.Position `json:"-"`
78+
Name string `json:"name"`
79+
Spec *WindowSpec `json:"spec"`
80+
}
81+
82+
func (w *WindowDefinition) Pos() token.Position { return w.Position }
83+
func (w *WindowDefinition) End() token.Position { return w.Position }
84+
85+
// IntoOutfileClause represents INTO OUTFILE clause.
86+
type IntoOutfileClause struct {
87+
Position token.Position `json:"-"`
88+
Filename string `json:"filename"`
5989
}
6090

91+
func (i *IntoOutfileClause) Pos() token.Position { return i.Position }
92+
func (i *IntoOutfileClause) End() token.Position { return i.Position }
93+
6194
func (s *SelectQuery) Pos() token.Position { return s.Position }
6295
func (s *SelectQuery) End() token.Position { return s.Position }
6396
func (s *SelectQuery) statementNode() {}
@@ -146,6 +179,9 @@ type OrderByElement struct {
146179
NullsFirst *bool `json:"nulls_first,omitempty"`
147180
Collate string `json:"collate,omitempty"`
148181
WithFill bool `json:"with_fill,omitempty"`
182+
FillFrom Expression `json:"fill_from,omitempty"`
183+
FillTo Expression `json:"fill_to,omitempty"`
184+
FillStep Expression `json:"fill_step,omitempty"`
149185
}
150186

151187
func (o *OrderByElement) Pos() token.Position { return o.Position }
@@ -185,6 +221,8 @@ type CreateQuery struct {
185221
Table string `json:"table,omitempty"`
186222
View string `json:"view,omitempty"`
187223
Materialized bool `json:"materialized,omitempty"`
224+
To string `json:"to,omitempty"` // Target table for materialized views
225+
Populate bool `json:"populate,omitempty"` // POPULATE for materialized views
188226
Columns []*ColumnDeclaration `json:"columns,omitempty"`
189227
Constraints []*Constraint `json:"constraints,omitempty"`
190228
Engine *EngineClause `json:"engine,omitempty"`
@@ -271,14 +309,15 @@ func (t *TTLClause) End() token.Position { return t.Position }
271309

272310
// DropQuery represents a DROP statement.
273311
type DropQuery struct {
274-
Position token.Position `json:"-"`
275-
IfExists bool `json:"if_exists,omitempty"`
276-
Database string `json:"database,omitempty"`
277-
Table string `json:"table,omitempty"`
278-
View string `json:"view,omitempty"`
279-
Temporary bool `json:"temporary,omitempty"`
280-
OnCluster string `json:"on_cluster,omitempty"`
281-
DropDatabase bool `json:"drop_database,omitempty"`
312+
Position token.Position `json:"-"`
313+
IfExists bool `json:"if_exists,omitempty"`
314+
Database string `json:"database,omitempty"`
315+
Table string `json:"table,omitempty"`
316+
View string `json:"view,omitempty"`
317+
Temporary bool `json:"temporary,omitempty"`
318+
OnCluster string `json:"on_cluster,omitempty"`
319+
DropDatabase bool `json:"drop_database,omitempty"`
320+
Sync bool `json:"sync,omitempty"`
282321
}
283322

284323
func (d *DropQuery) Pos() token.Position { return d.Position }
@@ -300,17 +339,24 @@ func (a *AlterQuery) statementNode() {}
300339

301340
// AlterCommand represents an ALTER command.
302341
type AlterCommand struct {
303-
Position token.Position `json:"-"`
304-
Type AlterCommandType `json:"type"`
305-
Column *ColumnDeclaration `json:"column,omitempty"`
306-
ColumnName string `json:"column_name,omitempty"`
307-
AfterColumn string `json:"after_column,omitempty"`
308-
NewName string `json:"new_name,omitempty"`
309-
Index string `json:"index,omitempty"`
310-
Constraint *Constraint `json:"constraint,omitempty"`
311-
Partition Expression `json:"partition,omitempty"`
312-
TTL *TTLClause `json:"ttl,omitempty"`
313-
Settings []*SettingExpr `json:"settings,omitempty"`
342+
Position token.Position `json:"-"`
343+
Type AlterCommandType `json:"type"`
344+
Column *ColumnDeclaration `json:"column,omitempty"`
345+
ColumnName string `json:"column_name,omitempty"`
346+
AfterColumn string `json:"after_column,omitempty"`
347+
NewName string `json:"new_name,omitempty"`
348+
IfNotExists bool `json:"if_not_exists,omitempty"`
349+
IfExists bool `json:"if_exists,omitempty"`
350+
Index string `json:"index,omitempty"`
351+
IndexExpr Expression `json:"index_expr,omitempty"`
352+
IndexType string `json:"index_type,omitempty"`
353+
Granularity int `json:"granularity,omitempty"`
354+
Constraint *Constraint `json:"constraint,omitempty"`
355+
ConstraintName string `json:"constraint_name,omitempty"`
356+
Partition Expression `json:"partition,omitempty"`
357+
FromTable string `json:"from_table,omitempty"`
358+
TTL *TTLClause `json:"ttl,omitempty"`
359+
Settings []*SettingExpr `json:"settings,omitempty"`
314360
}
315361

316362
func (a *AlterCommand) Pos() token.Position { return a.Position }
@@ -320,21 +366,26 @@ func (a *AlterCommand) End() token.Position { return a.Position }
320366
type AlterCommandType string
321367

322368
const (
323-
AlterAddColumn AlterCommandType = "ADD_COLUMN"
324-
AlterDropColumn AlterCommandType = "DROP_COLUMN"
325-
AlterModifyColumn AlterCommandType = "MODIFY_COLUMN"
326-
AlterRenameColumn AlterCommandType = "RENAME_COLUMN"
327-
AlterClearColumn AlterCommandType = "CLEAR_COLUMN"
328-
AlterCommentColumn AlterCommandType = "COMMENT_COLUMN"
329-
AlterAddIndex AlterCommandType = "ADD_INDEX"
330-
AlterDropIndex AlterCommandType = "DROP_INDEX"
331-
AlterAddConstraint AlterCommandType = "ADD_CONSTRAINT"
332-
AlterDropConstraint AlterCommandType = "DROP_CONSTRAINT"
333-
AlterModifyTTL AlterCommandType = "MODIFY_TTL"
334-
AlterModifySetting AlterCommandType = "MODIFY_SETTING"
335-
AlterDropPartition AlterCommandType = "DROP_PARTITION"
336-
AlterDetachPartition AlterCommandType = "DETACH_PARTITION"
337-
AlterAttachPartition AlterCommandType = "ATTACH_PARTITION"
369+
AlterAddColumn AlterCommandType = "ADD_COLUMN"
370+
AlterDropColumn AlterCommandType = "DROP_COLUMN"
371+
AlterModifyColumn AlterCommandType = "MODIFY_COLUMN"
372+
AlterRenameColumn AlterCommandType = "RENAME_COLUMN"
373+
AlterClearColumn AlterCommandType = "CLEAR_COLUMN"
374+
AlterCommentColumn AlterCommandType = "COMMENT_COLUMN"
375+
AlterAddIndex AlterCommandType = "ADD_INDEX"
376+
AlterDropIndex AlterCommandType = "DROP_INDEX"
377+
AlterClearIndex AlterCommandType = "CLEAR_INDEX"
378+
AlterMaterializeIndex AlterCommandType = "MATERIALIZE_INDEX"
379+
AlterAddConstraint AlterCommandType = "ADD_CONSTRAINT"
380+
AlterDropConstraint AlterCommandType = "DROP_CONSTRAINT"
381+
AlterModifyTTL AlterCommandType = "MODIFY_TTL"
382+
AlterModifySetting AlterCommandType = "MODIFY_SETTING"
383+
AlterDropPartition AlterCommandType = "DROP_PARTITION"
384+
AlterDetachPartition AlterCommandType = "DETACH_PARTITION"
385+
AlterAttachPartition AlterCommandType = "ATTACH_PARTITION"
386+
AlterReplacePartition AlterCommandType = "REPLACE_PARTITION"
387+
AlterFreezePartition AlterCommandType = "FREEZE_PARTITION"
388+
AlterFreeze AlterCommandType = "FREEZE"
338389
)
339390

340391
// TruncateQuery represents a TRUNCATE statement.
@@ -390,10 +441,13 @@ func (s *ShowQuery) statementNode() {}
390441
type ShowType string
391442

392443
const (
393-
ShowTables ShowType = "TABLES"
394-
ShowDatabases ShowType = "DATABASES"
395-
ShowProcesses ShowType = "PROCESSLIST"
396-
ShowCreate ShowType = "CREATE"
444+
ShowTables ShowType = "TABLES"
445+
ShowDatabases ShowType = "DATABASES"
446+
ShowProcesses ShowType = "PROCESSLIST"
447+
ShowCreate ShowType = "CREATE"
448+
ShowCreateDB ShowType = "CREATE_DATABASE"
449+
ShowColumns ShowType = "COLUMNS"
450+
ShowDictionaries ShowType = "DICTIONARIES"
397451
)
398452

399453
// ExplainQuery represents an EXPLAIN statement.
@@ -455,6 +509,30 @@ func (s *SystemQuery) Pos() token.Position { return s.Position }
455509
func (s *SystemQuery) End() token.Position { return s.Position }
456510
func (s *SystemQuery) statementNode() {}
457511

512+
// RenameQuery represents a RENAME TABLE statement.
513+
type RenameQuery struct {
514+
Position token.Position `json:"-"`
515+
From string `json:"from"`
516+
To string `json:"to"`
517+
OnCluster string `json:"on_cluster,omitempty"`
518+
}
519+
520+
func (r *RenameQuery) Pos() token.Position { return r.Position }
521+
func (r *RenameQuery) End() token.Position { return r.Position }
522+
func (r *RenameQuery) statementNode() {}
523+
524+
// ExchangeQuery represents an EXCHANGE TABLES statement.
525+
type ExchangeQuery struct {
526+
Position token.Position `json:"-"`
527+
Table1 string `json:"table1"`
528+
Table2 string `json:"table2"`
529+
OnCluster string `json:"on_cluster,omitempty"`
530+
}
531+
532+
func (e *ExchangeQuery) Pos() token.Position { return e.Position }
533+
func (e *ExchangeQuery) End() token.Position { return e.Position }
534+
func (e *ExchangeQuery) statementNode() {}
535+
458536
// -----------------------------------------------------------------------------
459537
// Expressions
460538

@@ -522,18 +600,42 @@ const (
522600

523601
// Asterisk represents a *.
524602
type Asterisk struct {
525-
Position token.Position `json:"-"`
526-
Table string `json:"table,omitempty"` // for table.*
603+
Position token.Position `json:"-"`
604+
Table string `json:"table,omitempty"` // for table.*
605+
Except []string `json:"except,omitempty"` // for * EXCEPT (col1, col2)
606+
Replace []*ReplaceExpr `json:"replace,omitempty"` // for * REPLACE (expr AS col)
527607
}
528608

529609
func (a *Asterisk) Pos() token.Position { return a.Position }
530610
func (a *Asterisk) End() token.Position { return a.Position }
531611
func (a *Asterisk) expressionNode() {}
532612

613+
// ReplaceExpr represents an expression in REPLACE clause.
614+
type ReplaceExpr struct {
615+
Position token.Position `json:"-"`
616+
Expr Expression `json:"expr"`
617+
Name string `json:"name"`
618+
}
619+
620+
func (r *ReplaceExpr) Pos() token.Position { return r.Position }
621+
func (r *ReplaceExpr) End() token.Position { return r.Position }
622+
623+
// ColumnsMatcher represents COLUMNS('pattern') expression.
624+
type ColumnsMatcher struct {
625+
Position token.Position `json:"-"`
626+
Pattern string `json:"pattern"`
627+
Except []string `json:"except,omitempty"`
628+
}
629+
630+
func (c *ColumnsMatcher) Pos() token.Position { return c.Position }
631+
func (c *ColumnsMatcher) End() token.Position { return c.Position }
632+
func (c *ColumnsMatcher) expressionNode() {}
633+
533634
// FunctionCall represents a function call.
534635
type FunctionCall struct {
535636
Position token.Position `json:"-"`
536637
Name string `json:"name"`
638+
Parameters []Expression `json:"parameters,omitempty"` // For parametric functions like quantile(0.9)(x)
537639
Arguments []Expression `json:"arguments,omitempty"`
538640
Distinct bool `json:"distinct,omitempty"`
539641
Over *WindowSpec `json:"over,omitempty"`
@@ -620,6 +722,18 @@ func (u *UnaryExpr) Pos() token.Position { return u.Position }
620722
func (u *UnaryExpr) End() token.Position { return u.Position }
621723
func (u *UnaryExpr) expressionNode() {}
622724

725+
// TernaryExpr represents a ternary conditional expression (cond ? then : else).
726+
type TernaryExpr struct {
727+
Position token.Position `json:"-"`
728+
Condition Expression `json:"condition"`
729+
Then Expression `json:"then"`
730+
Else Expression `json:"else"`
731+
}
732+
733+
func (t *TernaryExpr) Pos() token.Position { return t.Position }
734+
func (t *TernaryExpr) End() token.Position { return t.Position }
735+
func (t *TernaryExpr) expressionNode() {}
736+
623737
// Subquery represents a subquery.
624738
type Subquery struct {
625739
Position token.Position `json:"-"`

‎lexer/lexer.go‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -131,6 +131,11 @@ func (l *Lexer) NextToken() Item {
131131
if l.peekChar() == '=' {
132132
l.readChar()
133133
l.readChar()
134+
// Check for <=>
135+
if l.ch == '>' {
136+
l.readChar()
137+
return Item{Token: token.NULL_SAFE_EQ, Value: "<=>", Pos: pos}
138+
}
134139
return Item{Token: token.LTE, Value: "<=", Pos: pos}
135140
}
136141
if l.peekChar() == '>' {

0 commit comments

Comments
 (0)