Skip to content

Commit efa0542

Browse files
committed
Fix identifier and materialized view parsing for 03711 test
- Fix lexer to handle identifiers starting with numbers after a dot (e.g., db.03711_table) - Add isIdentifierAfterDot() to distinguish 03711_table from decimal numbers - Fix TO clause in CREATE MATERIALIZED VIEW to support qualified names (database.table) - Add ToDatabase field to CreateQuery for materialized view targets
1 parent 3c0c798 commit efa0542

4 files changed

Lines changed: 67 additions & 26 deletions

File tree

ast/ast.go

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -254,8 +254,9 @@ type CreateQuery struct {
254254
Table string `json:"table,omitempty"`
255255
View string `json:"view,omitempty"`
256256
Materialized bool `json:"materialized,omitempty"`
257-
To string `json:"to,omitempty"` // Target table for materialized views
258-
Populate bool `json:"populate,omitempty"` // POPULATE for materialized views
257+
ToDatabase string `json:"to_database,omitempty"` // Target database for materialized views
258+
To string `json:"to,omitempty"` // Target table for materialized views
259+
Populate bool `json:"populate,omitempty"` // POPULATE for materialized views
259260
Columns []*ColumnDeclaration `json:"columns,omitempty"`
260261
Indexes []*IndexDefinition `json:"indexes,omitempty"`
261262
Projections []*Projection `json:"projections,omitempty"`

lexer/lexer.go

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,55 @@ func (l *Lexer) skipWhitespace() {
8080
}
8181
}
8282

83+
// isIdentifierAfterDot checks if what follows the DOT looks like an identifier pattern.
84+
// This handles cases like db.03711_table where 03711_table should be treated as an identifier.
85+
// We look ahead to see if we have: digits followed by underscore followed by letter/digit
86+
func (l *Lexer) isIdentifierAfterDot() bool {
87+
// We're currently at '.', peek ahead to check the pattern
88+
// Save current reader position by peeking
89+
bytes, _ := l.reader.Peek(32) // peek enough to see the pattern, ignore EOF error
90+
if len(bytes) == 0 {
91+
return false
92+
}
93+
94+
idx := 0
95+
// Skip initial digits
96+
for idx < len(bytes) && bytes[idx] >= '0' && bytes[idx] <= '9' {
97+
idx++
98+
}
99+
// If no digits found, it's not our pattern
100+
if idx == 0 {
101+
return false
102+
}
103+
// Check for underscore followed by letter/digit (identifier pattern)
104+
if idx < len(bytes) && bytes[idx] == '_' {
105+
idx++
106+
if idx < len(bytes) && isIdentContinueByte(bytes[idx]) {
107+
return true
108+
}
109+
}
110+
// Also check for letter directly after digits (like 1abc)
111+
if idx < len(bytes) {
112+
ch, _ := utf8.DecodeRune(bytes[idx:])
113+
// Check it's a letter but not exponent (e/E followed by digit/+/-)
114+
if unicode.IsLetter(ch) {
115+
if (ch == 'e' || ch == 'E') && idx+1 < len(bytes) {
116+
next := bytes[idx+1]
117+
if next >= '0' && next <= '9' || next == '+' || next == '-' {
118+
return false // This is a scientific notation number
119+
}
120+
}
121+
return true
122+
}
123+
}
124+
return false
125+
}
126+
127+
// isIdentContinueByte checks if a byte is a valid identifier continuation character
128+
func isIdentContinueByte(b byte) bool {
129+
return (b >= 'a' && b <= 'z') || (b >= 'A' && b <= 'Z') || (b >= '0' && b <= '9') || b == '_'
130+
}
131+
83132
// isClickHouseWhitespace returns true for characters ClickHouse treats as whitespace
84133
// but Go's unicode.IsSpace does not recognize.
85134
func isClickHouseWhitespace(ch rune) bool {
@@ -226,6 +275,12 @@ func (l *Lexer) NextToken() Item {
226275
return Item{Token: token.COMMA, Value: ",", Pos: pos}
227276
case '.':
228277
if unicode.IsDigit(l.peekChar()) {
278+
// Before treating .digits as a number, check if it's part of an identifier pattern
279+
// For example, db.03711_table should be DOT + IDENT, not DOT + NUMBER + IDENT
280+
if l.isIdentifierAfterDot() {
281+
l.readChar()
282+
return Item{Token: token.DOT, Value: ".", Pos: pos}
283+
}
229284
return l.readNumber()
230285
}
231286
l.readChar()

parser/parser.go

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1975,7 +1975,14 @@ func (p *Parser) parseCreateView(create *ast.CreateQuery) {
19751975
return
19761976
}
19771977
p.nextToken()
1978-
create.To = p.parseIdentifierName()
1978+
toName := p.parseIdentifierName()
1979+
if p.currentIs(token.DOT) {
1980+
p.nextToken()
1981+
create.ToDatabase = toName
1982+
create.To = p.parseIdentifierName()
1983+
} else {
1984+
create.To = toName
1985+
}
19791986
}
19801987

19811988
// Parse ENGINE (for materialized views)
Lines changed: 1 addition & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -1,23 +1 @@
1-
{
2-
"explain_todo": {
3-
"stmt10": true,
4-
"stmt11": true,
5-
"stmt12": true,
6-
"stmt13": true,
7-
"stmt14": true,
8-
"stmt15": true,
9-
"stmt16": true,
10-
"stmt17": true,
11-
"stmt18": true,
12-
"stmt19": true,
13-
"stmt20": true,
14-
"stmt27": true,
15-
"stmt3": true,
16-
"stmt4": true,
17-
"stmt5": true,
18-
"stmt6": true,
19-
"stmt7": true,
20-
"stmt8": true,
21-
"stmt9": true
22-
}
23-
}
1+
{}

0 commit comments

Comments
 (0)