improve parser to use string instead of []rune

Author: diegommm

parser/lexer/lexer.go

@@ -3,16 +3,18 @@ package lexer
 import (
 	"fmt"
 	"strings"
+	"unicode/utf8"
 
 	"github.com/expr-lang/expr/file"
 )
 
+const minTokens = 10
+
 func Lex(source file.Source) ([]Token, error) {
 	raw := source.String()
 	l := &lexer{
 		raw:    raw,
-		runes:  []rune(raw),
-		tokens: make([]Token, 0),
+		tokens: make([]Token, 0, minTokens),
 	}
 
 	for state := root; state != nil; {
@@ -28,7 +30,6 @@ func Lex(source file.Source) ([]Token, error) {
 
 type lexer struct {
 	raw        string
-	runes      []rune
 	tokens     []Token
 	err        *file.Error
 	start, end pos
@@ -46,25 +47,37 @@ func (l *lexer) commit() {
 }
 
 func (l *lexer) next() rune {
-	if l.end.rune >= len(l.runes) {
+	if l.end.byte >= len(l.raw) {
 		l.eof = true
 		return eof
 	}
-	r := l.runes[l.end.rune]
+	r, sz := utf8.DecodeRuneInString(l.raw[l.end.byte:])
 	l.end.rune++
+	l.end.byte += sz
 	return r
 }
 
 func (l *lexer) peek() rune {
-	r := l.next()
-	l.backup()
-	return r
+	if l.end.byte < len(l.raw) {
+		r, _ := utf8.DecodeRuneInString(l.raw[l.end.byte:])
+		return r
+	}
+	return eof
+}
+
+func (l *lexer) peekByte() (byte, bool) {
+	if l.end.byte >= 0 && l.end.byte < len(l.raw) {
+		return l.raw[l.end.byte], true
+	}
+	return 0, false
 }
 
 func (l *lexer) backup() {
 	if l.eof {
 		l.eof = false
-	} else {
+	} else if l.end.rune > 0 {
+		_, sz := utf8.DecodeLastRuneInString(l.raw[:l.end.byte])
+		l.end.byte -= sz
 		l.end.rune--
 	}
 }
@@ -103,54 +116,27 @@ func (l *lexer) skip() {
 }
 
 func (l *lexer) word() string {
-	// TODO: boundary check is NOT needed here, but for some reason CI fuzz tests are failing.
-	if l.start.rune > len(l.runes) || l.end.rune > len(l.runes) {
-		return "__invalid__"
-	}
-	return string(l.runes[l.start.rune:l.end.rune])
+	return l.raw[l.start.byte:l.end.byte]
 }
 
 func (l *lexer) accept(valid string) bool {
-	if strings.ContainsRune(valid, l.next()) {
+	if strings.ContainsRune(valid, l.peek()) {
+		l.next()
 		return true
 	}
-	l.backup()
 	return false
 }
 
 func (l *lexer) acceptRun(valid string) {
-	for strings.ContainsRune(valid, l.next()) {
+	for l.accept(valid) {
 	}
-	l.backup()
 }
 
 func (l *lexer) skipSpaces() {
-	r := l.peek()
-	for ; r == ' '; r = l.peek() {
-		l.next()
-	}
+	l.acceptRun(" ")
 	l.skip()
 }
 
-func (l *lexer) acceptWord(word string) bool {
-	pos := l.end
-
-	l.skipSpaces()
-
-	for _, ch := range word {
-		if l.next() != ch {
-			l.end = pos
-			return false
-		}
-	}
-	if r := l.peek(); r != ' ' && r != eof {
-		l.end = pos
-		return false
-	}
-
-	return true
-}
-
 func (l *lexer) error(format string, args ...any) stateFn {
 	if l.err == nil { // show first error
 		end := l.end.rune
@@ -238,6 +224,6 @@ func (l *lexer) scanRawString(quote rune) (n int) {
 		ch = l.next()
 		n++
 	}
-	l.emitValue(String, string(l.runes[l.start.rune+1:l.end.rune-1]))
+	l.emitValue(String, l.raw[l.start.byte+1:l.end.byte-1])
 	return
 }

parser/lexer/token.go

@@ -31,17 +31,13 @@ func (t Token) String() string {
 }
 
 func (t Token) Is(kind Kind, values ...string) bool {
-	if len(values) == 0 {
-		return kind == t.Kind
+	if kind != t.Kind {
+		return false
 	}
-
 	for _, v := range values {
 		if v == t.Value {
-			goto found
+			return true
 		}
 	}
-	return false
-
-found:
-	return kind == t.Kind
+	return len(values) == 0
 }

parser/lexer/utils.go

@@ -36,21 +36,22 @@ func unescape(value string) (string, error) {
 	if size >= math.MaxInt {
 		return "", fmt.Errorf("too large string")
 	}
-	buf := make([]byte, 0, size)
+	buf := new(strings.Builder)
+	buf.Grow(int(size))
 	for len(value) > 0 {
 		c, multibyte, rest, err := unescapeChar(value)
 		if err != nil {
 			return "", err
 		}
 		value = rest
 		if c < utf8.RuneSelf || !multibyte {
-			buf = append(buf, byte(c))
+			buf.WriteByte(byte(c))
 		} else {
 			n := utf8.EncodeRune(runeTmp[:], c)
-			buf = append(buf, runeTmp[:n]...)
+			buf.Write(runeTmp[:n])
 		}
 	}
-	return string(buf), nil
+	return buf.String(), nil
 }
 
 // unescapeChar takes a string input and returns the following info: