Skip to content

Commit 93d6e99

Browse files
committed
Add hex, binary, octal literals and number separators to lexer
- Support hex literals: 0xDEADBEEF, 0x123abc - Support binary literals: 0b101010 - Support octal literals: 0o755 - Support underscore separators in numbers: 100_000_000 (underscores are only allowed between digits, not at boundaries) Test count: 5542 -> 5547 (+5 tests)
1 parent 0bc058e commit 93d6e99

1 file changed

Lines changed: 52 additions & 1 deletion

File tree

lexer/lexer.go

Lines changed: 52 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -344,10 +344,49 @@ func (l *Lexer) readNumber() Item {
344344
l.readChar()
345345
}
346346

347-
// Read integer part
347+
// Check for hex (0x), binary (0b), or octal (0o) prefix
348+
if l.ch == '0' {
349+
sb.WriteRune(l.ch)
350+
l.readChar()
351+
if l.ch == 'x' || l.ch == 'X' {
352+
// Hex literal
353+
sb.WriteRune(l.ch)
354+
l.readChar()
355+
for isHexDigit(l.ch) {
356+
sb.WriteRune(l.ch)
357+
l.readChar()
358+
}
359+
return Item{Token: token.NUMBER, Value: sb.String(), Pos: pos}
360+
} else if l.ch == 'b' || l.ch == 'B' {
361+
// Binary literal
362+
sb.WriteRune(l.ch)
363+
l.readChar()
364+
for l.ch == '0' || l.ch == '1' {
365+
sb.WriteRune(l.ch)
366+
l.readChar()
367+
}
368+
return Item{Token: token.NUMBER, Value: sb.String(), Pos: pos}
369+
} else if l.ch == 'o' || l.ch == 'O' {
370+
// Octal literal
371+
sb.WriteRune(l.ch)
372+
l.readChar()
373+
for l.ch >= '0' && l.ch <= '7' {
374+
sb.WriteRune(l.ch)
375+
l.readChar()
376+
}
377+
return Item{Token: token.NUMBER, Value: sb.String(), Pos: pos}
378+
}
379+
// Otherwise, continue with normal number parsing (leading 0)
380+
}
381+
382+
// Read integer part (including underscores as separators, but only between digits)
348383
for unicode.IsDigit(l.ch) {
349384
sb.WriteRune(l.ch)
350385
l.readChar()
386+
// Handle underscore separators (only if followed by a digit)
387+
for l.ch == '_' && unicode.IsDigit(l.peekChar()) {
388+
l.readChar() // skip underscore
389+
}
351390
}
352391

353392
// Check for decimal point
@@ -357,6 +396,10 @@ func (l *Lexer) readNumber() Item {
357396
for unicode.IsDigit(l.ch) {
358397
sb.WriteRune(l.ch)
359398
l.readChar()
399+
// Handle underscore separators
400+
for l.ch == '_' && unicode.IsDigit(l.peekChar()) {
401+
l.readChar()
402+
}
360403
}
361404
}
362405

@@ -371,12 +414,20 @@ func (l *Lexer) readNumber() Item {
371414
for unicode.IsDigit(l.ch) {
372415
sb.WriteRune(l.ch)
373416
l.readChar()
417+
// Handle underscore separators
418+
for l.ch == '_' && unicode.IsDigit(l.peekChar()) {
419+
l.readChar()
420+
}
374421
}
375422
}
376423

377424
return Item{Token: token.NUMBER, Value: sb.String(), Pos: pos}
378425
}
379426

427+
func isHexDigit(ch rune) bool {
428+
return unicode.IsDigit(ch) || (ch >= 'a' && ch <= 'f') || (ch >= 'A' && ch <= 'F')
429+
}
430+
380431
func (l *Lexer) readIdentifier() Item {
381432
pos := l.pos
382433
var sb strings.Builder

0 commit comments

Comments
 (0)