Skip to content

Commit 632cadc

Browse files
committed
Handle Unicode minus (U+2212) as comment-like syntax
ClickHouse doesn't recognize the Unicode minus sign (−, U+2212) as a mathematical operator. Instead of parsing it as a minus, treat everything from the Unicode minus to the end of line or semicolon as a comment. This matches ClickHouse's behavior where SELECT 1 − 2 produces just Literal UInt64_1 in the EXPLAIN AST output.
1 parent 678daab commit 632cadc

2 files changed

Lines changed: 22 additions & 4 deletions

File tree

lexer/lexer.go

Lines changed: 21 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -98,6 +98,11 @@ func (l *Lexer) NextToken() Item {
9898
if l.ch == '/' && l.peekChar() == '*' {
9999
return l.readBlockComment()
100100
}
101+
// Unicode minus (U+2212) is treated as starting a line comment
102+
// ClickHouse doesn't recognize it as an operator
103+
if l.ch == '\u2212' {
104+
return l.readUnicodeMinusComment()
105+
}
101106

102107
switch l.ch {
103108
case '+':
@@ -227,9 +232,6 @@ func (l *Lexer) NextToken() Item {
227232
return l.readQuotedIdentifier()
228233
case '\u201C', '\u201D': // Unicode curly double quotes " "
229234
return l.readUnicodeQuotedIdentifier(l.ch)
230-
case '\u2212': // Unicode minus sign −
231-
l.readChar()
232-
return Item{Token: token.MINUS, Value: "−", Pos: pos}
233235
case '`':
234236
return l.readBacktickIdentifier()
235237
case '@':
@@ -297,6 +299,22 @@ func (l *Lexer) readHashComment() Item {
297299
return Item{Token: token.COMMENT, Value: sb.String(), Pos: pos}
298300
}
299301

302+
// readUnicodeMinusComment reads from a unicode minus (U+2212) to the end of line or semicolon.
303+
// ClickHouse doesn't recognize unicode minus as an operator, so we treat it as a comment.
304+
func (l *Lexer) readUnicodeMinusComment() Item {
305+
pos := l.pos
306+
var sb strings.Builder
307+
// Skip −
308+
sb.WriteRune(l.ch)
309+
l.readChar()
310+
311+
for l.ch != '\n' && l.ch != ';' && l.ch != 0 && !l.eof {
312+
sb.WriteRune(l.ch)
313+
l.readChar()
314+
}
315+
return Item{Token: token.COMMENT, Value: sb.String(), Pos: pos}
316+
}
317+
300318
func (l *Lexer) readBlockComment() Item {
301319
pos := l.pos
302320
var sb strings.Builder
Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
{"todo": true}
1+
{}

0 commit comments

Comments
 (0)