Skip to content

Commit ffb37c9

Browse files
committed
Fix string escape handling in lexer and explain layer
The lexer now properly unescapes string literals: - \' → single quote - \\ → backslash - \n → newline - \t → tab - \r → carriage return - \0 → null byte - \b → backspace - \f → form feed - \xNN → hex byte value - '' → single quote (SQL-style escape) The explain layer re-escapes these characters for display in EXPLAIN AST output, using ClickHouse's double-escaping format (e.g., null becomes \\0). Test results improved from 6,000 to 6,006 passing tests.
1 parent 83f88da commit ffb37c9

3 files changed

Lines changed: 94 additions & 21 deletions

File tree

TODO.md

Lines changed: 3 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,8 @@
22

33
## Current State
44

5-
- **Tests passing:** 6,000 (87.9%)
6-
- **Tests skipped:** 824 (12.1%)
5+
- **Tests passing:** 6,006 (88.0%)
6+
- **Tests skipped:** 819 (12.0%)
77

88
## Recently Fixed (explain layer)
99

@@ -21,6 +21,7 @@
2121
- ✅ DROP TABLE with multiple tables (e.g., `DROP TABLE t1, t2, t3`)
2222
- ✅ Negative integer/float literals (e.g., `-1``Literal Int64_-1`)
2323
- ✅ Empty tuple in ORDER BY (e.g., `ORDER BY ()``Function tuple` with empty `ExpressionList`)
24+
- ✅ String escape handling (lexer now unescapes `\'`, `\\`, `\n`, `\t`, `\0`, etc.)
2425

2526
## Parser Issues (High Priority)
2627

@@ -46,14 +47,6 @@ CREATE TABLE t (c Int TTL expr()) ENGINE=MergeTree;
4647
-- Expected: ColumnDeclaration with 2 children (type + TTL function)
4748
```
4849

49-
### String Escape Handling
50-
Parser stores escaped characters literally instead of unescaping:
51-
```sql
52-
SELECT 'x\'e2\'';
53-
-- Parser stores: x\'e2\' (with backslashes)
54-
-- Should store: x'e2' (unescaped)
55-
```
56-
5750
## Parser Issues (Medium Priority)
5851

5952
### CREATE DICTIONARY

internal/explain/format.go

Lines changed: 31 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,35 @@ func FormatFloat(val float64) string {
1414
return strconv.FormatFloat(val, 'f', -1, 64)
1515
}
1616

17+
// escapeStringLiteral escapes special characters in a string for EXPLAIN AST output
18+
// Uses double-escaping as ClickHouse EXPLAIN AST displays strings
19+
func escapeStringLiteral(s string) string {
20+
var sb strings.Builder
21+
for _, r := range s {
22+
switch r {
23+
case '\\':
24+
sb.WriteString("\\\\\\\\") // backslash becomes four backslashes (\\\\)
25+
case '\'':
26+
sb.WriteString("\\'")
27+
case '\n':
28+
sb.WriteString("\\\\n") // newline becomes \\n
29+
case '\t':
30+
sb.WriteString("\\\\t") // tab becomes \\t
31+
case '\r':
32+
sb.WriteString("\\\\r") // carriage return becomes \\r
33+
case '\x00':
34+
sb.WriteString("\\\\0") // null becomes \\0
35+
case '\b':
36+
sb.WriteString("\\\\b") // backspace becomes \\b
37+
case '\f':
38+
sb.WriteString("\\\\f") // form feed becomes \\f
39+
default:
40+
sb.WriteRune(r)
41+
}
42+
}
43+
return sb.String()
44+
}
45+
1746
// FormatLiteral formats a literal value for EXPLAIN AST output
1847
func FormatLiteral(lit *ast.Literal) string {
1948
switch lit.Type {
@@ -35,8 +64,8 @@ func FormatLiteral(lit *ast.Literal) string {
3564
return fmt.Sprintf("Float64_%s", FormatFloat(val))
3665
case ast.LiteralString:
3766
s := lit.Value.(string)
38-
// Escape backslashes in strings
39-
s = strings.ReplaceAll(s, "\\", "\\\\")
67+
// Escape special characters for display
68+
s = escapeStringLiteral(s)
4069
return fmt.Sprintf("\\'%s\\'", s)
4170
case ast.LiteralBoolean:
4271
if lit.Value.(bool) {

lexer/lexer.go

Lines changed: 60 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -270,24 +270,62 @@ func (l *Lexer) readString(quote rune) Item {
270270

271271
for !l.eof {
272272
if l.ch == quote {
273-
// Check for escaped quote
273+
// Check for escaped quote (e.g., '' becomes ')
274274
if l.peekChar() == quote {
275-
sb.WriteRune(l.ch)
276-
l.readChar()
277-
sb.WriteRune(l.ch)
278-
l.readChar()
275+
sb.WriteRune(l.ch) // Write one quote (the escaped result)
276+
l.readChar() // skip first quote
277+
l.readChar() // skip second quote
279278
continue
280279
}
281280
l.readChar() // skip closing quote
282281
break
283282
}
284283
if l.ch == '\\' {
285-
sb.WriteRune(l.ch)
286-
l.readChar()
287-
if !l.eof {
288-
sb.WriteRune(l.ch)
284+
l.readChar() // consume backslash
285+
if l.eof {
286+
break
287+
}
288+
// Interpret escape sequence
289+
switch l.ch {
290+
case '\'':
291+
sb.WriteRune('\'')
292+
case '"':
293+
sb.WriteRune('"')
294+
case '\\':
295+
sb.WriteRune('\\')
296+
case 'n':
297+
sb.WriteRune('\n')
298+
case 't':
299+
sb.WriteRune('\t')
300+
case 'r':
301+
sb.WriteRune('\r')
302+
case '0':
303+
sb.WriteRune('\x00')
304+
case 'b':
305+
sb.WriteRune('\b')
306+
case 'f':
307+
sb.WriteRune('\f')
308+
case 'x':
309+
// Hex escape: \xNN
289310
l.readChar()
311+
if l.eof {
312+
break
313+
}
314+
hex1 := l.ch
315+
l.readChar()
316+
if l.eof {
317+
sb.WriteRune(rune(hexValue(hex1)))
318+
continue
319+
}
320+
hex2 := l.ch
321+
// Convert hex digits to byte
322+
val := hexValue(hex1)*16 + hexValue(hex2)
323+
sb.WriteByte(byte(val))
324+
default:
325+
// Unknown escape, just write the character after backslash
326+
sb.WriteRune(l.ch)
290327
}
328+
l.readChar()
291329
continue
292330
}
293331
sb.WriteRune(l.ch)
@@ -428,6 +466,19 @@ func isHexDigit(ch rune) bool {
428466
return unicode.IsDigit(ch) || (ch >= 'a' && ch <= 'f') || (ch >= 'A' && ch <= 'F')
429467
}
430468

469+
func hexValue(ch rune) int {
470+
if ch >= '0' && ch <= '9' {
471+
return int(ch - '0')
472+
}
473+
if ch >= 'a' && ch <= 'f' {
474+
return int(ch-'a') + 10
475+
}
476+
if ch >= 'A' && ch <= 'F' {
477+
return int(ch-'A') + 10
478+
}
479+
return 0
480+
}
481+
431482
func (l *Lexer) readIdentifier() Item {
432483
pos := l.pos
433484
var sb strings.Builder

0 commit comments

Comments
 (0)