token.go

package tokenizer

import (
	"fmt"
)

///////////////////////////////////////////////////////////////////////////////
// TYPES

// TokenKind classifies the type of a token produced by the scanner.
// Each token has a kind that identifies what type of lexical element it represents,
// such as an identifier, number, string, operator, or punctuation.
type TokenKind uint

// Token represents a lexical token produced by the scanner.
// It contains the token's kind, its literal value as a string,
// and the position in the source where it was found.
type Token struct {
	// Kind identifies the type of token (e.g., Ident, String, NumberInteger)
	Kind TokenKind
	// Val contains the literal text of the token
	Val string
	// Pos indicates where in the source the token was found
	Pos Pos
}

///////////////////////////////////////////////////////////////////////////////
// GLOBALS

const (
	Any TokenKind = iota
	String
	Expr
	Space
	Ident
	NumberInteger
	NumberFloat
	NumberOctal
	NumberHex
	NumberBinary
	Punkt
	Question
	Colon
	SemiColon
	Comma
	OpenParen
	CloseParen
	OpenSquare
	CloseSquare
	OpenBrace
	CloseBrace
	Ampersand
	Equal
	Less
	Greater
	Plus
	Minus
	Multiply
	Divide
	Not
	Backtick
	Tilde
	Pipe
	Backslash
	SingleQuote
	DoubleQuote
	Underscore
	Hash
	At
	Caret
	Percent
	Dollar
	True
	False
	Null
	Comment
	Newline
	EOF
	Lowest = Equal // Lowest precedence
)

var (
	// Special end of file rune
	eof = rune(0)

	// Special characters
	tokenKindMap = map[rune]TokenKind{
		'.':  Punkt,
		'?':  Question,
		':':  Colon,
		';':  SemiColon,
		',':  Comma,
		'(':  OpenParen,
		')':  CloseParen,
		'[':  OpenSquare,
		']':  CloseSquare,
		'{':  OpenBrace,
		'}':  CloseBrace,
		'&':  Ampersand,
		'=':  Equal,
		'<':  Less,
		'>':  Greater,
		'!':  Not,
		'+':  Plus,
		'-':  Minus,
		'*':  Multiply,
		'/':  Divide,
		'`':  Backtick,
		'~':  Tilde,
		'|':  Pipe,
		'\\': Backslash,
		'\'': SingleQuote,
		'"':  DoubleQuote,
		'_':  Underscore,
		'#':  Hash,
		'@':  At,
		'^':  Caret,
		'%':  Percent,
		'$':  Dollar,
		eof:  EOF,
	}
	// Reserved words
	tokenKeywordMap = map[string]TokenKind{
		"true":  True,
		"false": False,
		"null":  Null,
	}
)

///////////////////////////////////////////////////////////////////////////////
// LIFECYCLE

// NewToken creates a new Token with the specified kind, value, and position.
func NewToken(kind TokenKind, val string, pos Pos) *Token {
	return &Token{Kind: kind, Val: val, Pos: pos}
}

///////////////////////////////////////////////////////////////////////////////
// STRINGIFY

func (k TokenKind) String() string {
	switch k {
	case Any:
		return "Any"
	case String:
		return "String"
	case Expr:
		return "Expr"
	case Space:
		return "Space"
	case Ident:
		return "Ident"
	case NumberInteger:
		return "NumberInteger"
	case NumberFloat:
		return "NumberFloat"
	case NumberOctal:
		return "NumberOctal"
	case NumberHex:
		return "NumberHex"
	case NumberBinary:
		return "NumberBinary"
	case Punkt:
		return "Punkt"
	case Question:
		return "Question"
	case Colon:
		return "Colon"
	case SemiColon:
		return "SemiColon"
	case Comma:
		return "Comma"
	case Comment:
		return "Comment"
	case OpenParen:
		return "OpenParen"
	case CloseParen:
		return "CloseParen"
	case OpenSquare:
		return "OpenSquare"
	case CloseSquare:
		return "CloseSquare"
	case OpenBrace:
		return "OpenBrace"
	case CloseBrace:
		return "CloseBrace"
	case Ampersand:
		return "Ampersand"
	case Equal:
		return "Equal"
	case Less:
		return "Less"
	case Greater:
		return "Greater"
	case Plus:
		return "Plus"
	case Minus:
		return "Minus"
	case Multiply:
		return "Star"
	case Divide:
		return "Slash"
	case Not:
		return "Not"
	case Backtick:
		return "Backtick"
	case Tilde:
		return "Tilde"
	case Pipe:
		return "Pipe"
	case Backslash:
		return "Backslash"
	case SingleQuote:
		return "SingleQuote"
	case DoubleQuote:
		return "DoubleQuote"
	case Underscore:
		return "Underscore"
	case Hash:
		return "Hash"
	case At:
		return "At"
	case Caret:
		return "Caret"
	case Percent:
		return "Percent"
	case Dollar:
		return "Dollar"
	case True:
		return "True"
	case False:
		return "False"
	case Null:
		return "Null"
	case Newline:
		return "Newline"
	case EOF:
		return "EOF"
	default:
		return "[?? Invalid TokenKind value]"
	}
}

func (t *Token) String() string {
	return fmt.Sprintf("%v<%q>", t.Kind, t.Val)
}