scanner_test.go

package tokenizer_test

import (
	"strings"
	"testing"

	// Packages
	tokenizer "github.com/mutablelogic/go-tokenizer"
	assert "github.com/stretchr/testify/assert"
)

///////////////////////////////////////////////////////////////////////////////
// Scanner Tests

func Test_Scanner_001(t *testing.T) {
	// Non-error cases
	tests := []struct {
		in string
	}{
		{""},
		{"      "},
		{"0 1 2 3 4 5 6 7 8 9"},
		{"func(test)"},
		{"   var.test   "},
	}

	for _, test := range tests {
		t.Run(test.in, func(t *testing.T) {
			assert := assert.New(t)
			scanner := tokenizer.NewScanner(strings.NewReader(test.in), tokenizer.Pos{})
			assert.NotNil(scanner)
			for tok := scanner.Next(); tok.Kind != tokenizer.EOF; tok = scanner.Next() {
				assert.NotNil(tok)
			}
		})
	}
}

func Test_Scanner_002(t *testing.T) {
	// Non-error cases - general
	tests := []struct {
		in     string
		out    string
		values []string
	}{
		{"", "", nil},
		{"    ", "Space", []string{"    "}},
		{" \n\t\t   ", "Space", []string{" \n\t\t   "}},
		{" ; ", "Space SemiColon Space", []string{" ", ";", " "}},
		{"0 1", "NumberInteger Space NumberInteger", []string{"0", " ", "1"}},
		{"func(test)", "Ident OpenParen Ident CloseParen", []string{"func", "(", "test", ")"}},
		{"   var.test   ", "Space Ident Punkt Ident Space", []string{"   ", "var", ".", "test", "   "}},
		{`'test'`, "String", []string{"test"}},
		{`'te"st'`, "String", []string{"te\"st"}},
		{`"e'st"`, "String", []string{"e'st"}},
		{`"te\"st"`, "String", []string{"te\"st"}},
		{`"e\""`, "String", []string{"e\""}},
		{`"t\"'t"`, "String", []string{"t\"'t"}},
		{`'e\'st'`, "String", []string{"e'st"}},
		{`'e"st'`, "String", []string{"e\"st"}},
		{`!!!`, "Not Not Not", []string{"!", "!", "!"}},
	}

	for _, test := range tests {
		t.Run(test.in, func(t *testing.T) {
			assert := assert.New(t)
			tokens, err := tokenizer.NewScanner(strings.NewReader(test.in), tokenizer.Pos{}).Tokens()
			assert.NoError(err)
			assert.NotNil(tokens)
			assert.Equal(test.out, toTokenString(tokens))
			assert.Equal(test.values, toTokenValues(tokens))
		})
	}
}

func Test_Scanner_003(t *testing.T) {
	// Non-error cases - numbers and prefixes (without NumberFloatToken)
	tests := []struct {
		in  string
		out string
	}{
		{"1267650600228229401496703205376", "NumberInteger"},
		{"-1267650600228229401496703205376", "NumberInteger"},
		{"+1267650600228229401496703205376", "NumberInteger"},
		{"0x12AB", "NumberHex"},
		{"0X12AB", "NumberHex"},
		{"-0x12CD", "NumberHex"},
		{"-0XED", "NumberHex"},
		{"+0xEF12", "NumberHex"},
		{"+0xFE12", "NumberHex"},
		{"012", "NumberOctal"},
		{"-0123445", "NumberOctal"},
		{"+012345", "NumberOctal"},
		{"0b1010", "NumberBinary"},
		{"-0b101011", "NumberBinary"},
		{"+0b1001001", "NumberBinary"},
		{".", "Punkt"},
		{"+", "Plus"},
		{"-", "Minus"},
		{"++", "Plus Plus"},
		{"--", "Minus Minus"},
		{"e", "Ident"},
		{"ee", "Ident"},
		{"e++", "Ident Plus Plus"},
		{"e+-", "Ident Plus Minus"},
		{".+", "Punkt Plus"},
		{".+-", "Punkt Plus Minus"},
		{"-.", "Minus Punkt"},
		{"-..", "Minus Punkt Punkt"},
		{"+.", "Plus Punkt"},
		{"+..", "Plus Punkt Punkt"},
		{"..", "Punkt Punkt"},
		{"..-+", "Punkt Punkt Minus Plus"},
		{"-.e", "Minus Punkt Ident"},
		{"+.e", "Plus Punkt Ident"},
		{"-e", "Minus Ident"},
		{"+e", "Plus Ident"},
		// Without NumberFloatToken, floats are parsed as separate tokens
		{"1.5", "NumberInteger Punkt NumberInteger"},
		{"0.5", "NumberInteger Punkt NumberInteger"},
		{".5", "Punkt NumberInteger"},
		{"1e5", "NumberInteger Ident"},
	}

	for _, test := range tests {
		t.Run(test.in, func(t *testing.T) {
			assert := assert.New(t)
			tokens, err := tokenizer.NewScanner(strings.NewReader(test.in), tokenizer.Pos{}).Tokens()
			assert.NoError(err)
			assert.NotNil(tokens)
			assert.Equal(test.out, toTokenString(tokens))
			t.Logf("%s => %s", test.in, toTokenString(tokens))
		})
	}
}

func Test_Scanner_003b(t *testing.T) {
	// Non-error cases - float numbers (with NumberFloatToken enabled)
	tests := []struct {
		in  string
		out string
	}{
		{"0.00000000000000000000000000000078886090522101180541", "NumberFloat"},
		{".00000000000000000000000000000078886090522101180541", "NumberFloat"},
		{"-.00000000000000000000000000000078886090522101180541", "NumberFloat"},
		{"+.00000000000000000000000000000078886090522101180541", "NumberFloat"},
		{"-0.00000000000000000000000000000078886090522101180541", "NumberFloat"},
		{"+0.00000000000000000000000000000078886090522101180541", "NumberFloat"},
		{"-0.0000000000000000000000000000007E8886090522101180541", "NumberFloat"},
		{"+0.0000000000000000000000000000007e8886090522101180541", "NumberFloat"},
		{"1.5", "NumberFloat"},
		{"0.5", "NumberFloat"},
		{".5", "NumberFloat"},
		{"1e5", "NumberFloat"},
		{"1.5e10", "NumberFloat"},
		{"-1.5e-10", "NumberFloat"},
	}

	for _, test := range tests {
		t.Run(test.in, func(t *testing.T) {
			assert := assert.New(t)
			tokens, err := tokenizer.NewScanner(strings.NewReader(test.in), tokenizer.Pos{}, tokenizer.NumberFloatToken).Tokens()
			assert.NoError(err)
			assert.NotNil(tokens)
			assert.Equal(test.out, toTokenString(tokens))
			t.Logf("%s => %s", test.in, toTokenString(tokens))
		})
	}
}

func Test_Scanner_004(t *testing.T) {
	// Error cases - numbers (with NumberFloatToken enabled for float errors)
	tests := []struct {
		in string
	}{
		{"12ee"},
		{"+08888"},
		{"0b2344"},
		{"0.0000000e.45"},
		{"0.0000000e-.45"},
		{"0.0000000e+.45"},
		{"-45e++6"},
		{"-45e-.6"},
	}

	for _, test := range tests {
		t.Run(test.in, func(t *testing.T) {
			assert := assert.New(t)
			tokens, err := tokenizer.NewScanner(strings.NewReader(test.in), tokenizer.Pos{}, tokenizer.NumberFloatToken).Tokens()
			if err == nil {
				t.Logf("%s => %s", test.in, toTokenString(tokens))
			}
			assert.Error(err)
		})
	}
}

func Test_Scanner_004b(t *testing.T) {
	// Cases that look like errors but are valid multi-token sequences
	tests := []struct {
		in  string
		out string
	}{
		{"-.e16", "Minus Punkt Ident"},   // -  .  e16 (identifier)
		{"0x100g100", "NumberHex Ident"}, // 0x100  g100 (identifier)
	}

	for _, test := range tests {
		t.Run(test.in, func(t *testing.T) {
			assert := assert.New(t)
			tokens, err := tokenizer.NewScanner(strings.NewReader(test.in), tokenizer.Pos{}).Tokens()
			assert.NoError(err)
			assert.NotNil(tokens)
			assert.Equal(test.out, toTokenString(tokens))
		})
	}
}

func Test_Scanner_005(t *testing.T) {
	// Non-error cases - comments (with features enabled)
	tests := []struct {
		in       string
		features tokenizer.Feature
		out      string
	}{
		{"# Hash comment #", tokenizer.HashComment, "Comment"},
		{"# Hash comment\n # Another comment", tokenizer.HashComment, "Comment Space Comment"},
		{"// Line comment", tokenizer.LineComment, "Comment"},
		{"//// Line comment", tokenizer.LineComment, "Comment"},
		{"/ / // Line comment", tokenizer.LineComment, "Slash Space Slash Space Comment"},
		{"// Line comment\n// Line comment", tokenizer.LineComment, "Comment Space Comment"},
		{"test /* Block comment */ test", tokenizer.BlockComment, "Ident Space Comment Space Ident"},
		{"test /* Block \n comment */ test", tokenizer.BlockComment, "Ident Space Comment Space Ident"},
		{"test /*** Block \n comment ***/ test", tokenizer.BlockComment, "Ident Space Comment Space Ident"},
	}

	for _, test := range tests {
		t.Run(test.in, func(t *testing.T) {
			assert := assert.New(t)
			tokens, err := tokenizer.NewScanner(strings.NewReader(test.in), tokenizer.Pos{}, test.features).Tokens()
			assert.NoError(err)
			assert.NotNil(tokens)
			assert.Equal(test.out, toTokenString(tokens))
		})
	}
}

func Test_Scanner_006(t *testing.T) {
	// Non-error cases - unary operators
	tests := []struct {
		in  string
		out string
	}{
		{"!test", "Not Ident"},
		{"!(test)", "Not OpenParen Ident CloseParen"},
		{"!55", "Not NumberInteger"},
		{"(!0b101)", "OpenParen Not NumberBinary CloseParen"},
		{"(!0x101)", "OpenParen Not NumberHex CloseParen"},
		{"(!0101)", "OpenParen Not NumberOctal CloseParen"},
	}

	for _, test := range tests {
		t.Run(test.in, func(t *testing.T) {
			assert := assert.New(t)
			tokens, err := tokenizer.NewScanner(strings.NewReader(test.in), tokenizer.Pos{}).Tokens()
			assert.NoError(err)
			assert.NotNil(tokens)
			assert.Equal(test.out, toTokenString(tokens))
		})
	}
}

func Test_Scanner_007(t *testing.T) {
	// Reserved words
	tests := []struct {
		in  string
		out string
	}{
		{"true", "True"},
		{"false", "False"},
		{"null", "Null"},
		{"TRUE", "True"},
		{"FALSE", "False"},
		{"NULL", "Null"},
		{"True", "True"},
		{"False", "False"},
		{"Null", "Null"},
		{"true false null", "True Space False Space Null"},
		{"truefalse", "Ident"}, // Not reserved - it's a single identifier
	}

	for _, test := range tests {
		t.Run(test.in, func(t *testing.T) {
			assert := assert.New(t)
			tokens, err := tokenizer.NewScanner(strings.NewReader(test.in), tokenizer.Pos{}).Tokens()
			assert.NoError(err)
			assert.NotNil(tokens)
			assert.Equal(test.out, toTokenString(tokens))
		})
	}
}

func Test_Scanner_008(t *testing.T) {
	// String escape sequences
	tests := []struct {
		in  string
		out string
		val string
	}{
		{`"hello\nworld"`, "String", "hello\nworld"},
		{`"hello\rworld"`, "String", "hello\rworld"},
		{`"hello\tworld"`, "String", "hello\tworld"},
		{`"hello\\world"`, "String", "hello\\world"},
		{`"hello\"world"`, "String", "hello\"world"},
		{`'hello\'world'`, "String", "hello'world"},
		{`"\n\r\t\\"`, "String", "\n\r\t\\"},
	}

	for _, test := range tests {
		t.Run(test.in, func(t *testing.T) {
			assert := assert.New(t)
			tokens, err := tokenizer.NewScanner(strings.NewReader(test.in), tokenizer.Pos{}).Tokens()
			assert.NoError(err)
			assert.NotNil(tokens)
			assert.Equal(test.out, toTokenString(tokens))
			if len(tokens) > 0 {
				assert.Equal(test.val, tokens[0].Val)
			}
		})
	}
}

func Test_Scanner_009(t *testing.T) {
	// Error cases - unterminated strings and comments
	tests := []struct {
		in       string
		desc     string
		features tokenizer.Feature
	}{
		{`"unterminated string`, "unterminated double-quote string", 0},
		{`'unterminated string`, "unterminated single-quote string", 0},
		{`"unterminated with escape\"`, "unterminated string ending with escape", 0},
		{"/* unterminated block comment", "unterminated block comment", tokenizer.BlockComment},
		{"/* unterminated * block comment", "unterminated block comment with star", tokenizer.BlockComment},
	}

	for _, test := range tests {
		t.Run(test.desc, func(t *testing.T) {
			assert := assert.New(t)
			tokens, err := tokenizer.NewScanner(strings.NewReader(test.in), tokenizer.Pos{}, test.features).Tokens()
			if err == nil {
				t.Logf("%s => %s", test.in, toTokenString(tokens))
			}
			assert.Error(err)
		})
	}
}

func Test_Scanner_010(t *testing.T) {
	// All operator and bracket tokens
	tests := []struct {
		in  string
		out string
	}{
		{"<", "Less"},
		{">", "Greater"},
		{"&", "Ampersand"},
		{"=", "Equal"},
		{"[", "OpenSquare"},
		{"]", "CloseSquare"},
		{"{", "OpenBrace"},
		{"}", "CloseBrace"},
		{"(", "OpenParen"},
		{")", "CloseParen"},
		{",", "Comma"},
		{":", "Colon"},
		{";", "SemiColon"},
		{"?", "Question"},
		{"*", "Star"},
		{"/", "Slash"},
		{"<>=&[]{}(),;:?*!", "Less Greater Equal Ampersand OpenSquare CloseSquare OpenBrace CloseBrace OpenParen CloseParen Comma SemiColon Colon Question Star Not"},
	}

	for _, test := range tests {
		t.Run(test.in, func(t *testing.T) {
			assert := assert.New(t)
			tokens, err := tokenizer.NewScanner(strings.NewReader(test.in), tokenizer.Pos{}).Tokens()
			assert.NoError(err)
			assert.NotNil(tokens)
			assert.Equal(test.out, toTokenString(tokens))
		})
	}
}

func Test_Scanner_011(t *testing.T) {
	// Peak() method - should not advance scanner
	assert := assert.New(t)
	scanner := tokenizer.NewScanner(strings.NewReader("hello world"), tokenizer.Pos{})

	// Peak should return the same token multiple times
	tok1 := scanner.Peak()
	tok2 := scanner.Peak()
	assert.Equal(tok1.Kind, tok2.Kind)
	assert.Equal(tok1.Val, tok2.Val)
	assert.Equal(tokenizer.Ident, tok1.Kind)
	assert.Equal("hello", tok1.Val)

	// Next should return the same token and advance
	tok3 := scanner.Next()
	assert.Equal(tok1.Kind, tok3.Kind)
	assert.Equal(tok1.Val, tok3.Val)

	// Now Peak should return the next token (space)
	tok4 := scanner.Peak()
	assert.Equal(tokenizer.Space, tok4.Kind)

	// Next should advance to space
	tok5 := scanner.Next()
	assert.Equal(tokenizer.Space, tok5.Kind)

	// Next should return "world"
	tok6 := scanner.Next()
	assert.Equal(tokenizer.Ident, tok6.Kind)
	assert.Equal("world", tok6.Val)

	// Next should return EOF
	tok7 := scanner.Next()
	assert.Equal(tokenizer.EOF, tok7.Kind)

	// Continued calls to Next should keep returning EOF
	tok8 := scanner.Next()
	assert.Equal(tokenizer.EOF, tok8.Kind)
}

func Test_Scanner_012(t *testing.T) {
	// Position tracking
	tests := []struct {
		in       string
		expected []struct {
			line uint
			col  uint
		}
	}{
		{
			"a b c",
			[]struct {
				line uint
				col  uint
			}{
				{0, 0}, // a at line 1, col 1 (0-indexed: 0, 0)
				{0, 1}, // space
				{0, 2}, // b
				{0, 3}, // space
				{0, 4}, // c
			},
		},
		{
			"a\nb\nc",
			[]struct {
				line uint
				col  uint
			}{
				{0, 0}, // a
				{0, 1}, // \n (newline is at end of line 1)
				{1, 0}, // b
				{1, 1}, // \n
				{2, 0}, // c
			},
		},
	}

	for _, test := range tests {
		t.Run(test.in, func(t *testing.T) {
			assert := assert.New(t)
			tokens, err := tokenizer.NewScanner(strings.NewReader(test.in), tokenizer.Pos{}).Tokens()
			assert.NoError(err)
			assert.Equal(len(test.expected), len(tokens))
			for i, tok := range tokens {
				if i < len(test.expected) {
					assert.Equal(test.expected[i].line, tok.Pos.Line, "token %d (%s) line mismatch", i, tok.Val)
					assert.Equal(test.expected[i].col, tok.Pos.Col, "token %d (%s) col mismatch", i, tok.Val)
				}
			}
		})
	}
}

func Test_Scanner_013(t *testing.T) {
	// NewError method
	assert := assert.New(t)
	scanner := tokenizer.NewScanner(strings.NewReader("test"), tokenizer.Pos{})

	// Read a token to advance position
	scanner.Next()

	// Create an error
	err := scanner.NewError(tokenizer.ErrBadParameter)
	assert.Error(err)
	assert.Contains(err.Error(), "ErrBadParameter")
}

func Test_Scanner_014(t *testing.T) {
	// Invalid escape sequences in strings
	tests := []struct {
		in   string
		desc string
	}{
		{`"\x"`, "invalid escape \\x"},
		{`"\a"`, "invalid escape \\a"},
		{`"\z"`, "invalid escape \\z"},
	}

	for _, test := range tests {
		t.Run(test.desc, func(t *testing.T) {
			assert := assert.New(t)
			tokens, err := tokenizer.NewScanner(strings.NewReader(test.in), tokenizer.Pos{}).Tokens()
			if err == nil {
				t.Logf("%s => %s", test.in, toTokenString(tokens))
			}
			assert.Error(err)
		})
	}
}

///////////////////////////////////////////////////////////////////////////////
// Private Methods

func toTokenString(tokens []*tokenizer.Token) string {
	var result []string
	for _, token := range tokens {
		result = append(result, token.Kind.String())
	}
	return strings.Join(result, " ")
}

func toTokenValues(tokens []*tokenizer.Token) []string {
	var result []string
	for _, token := range tokens {
		result = append(result, token.Val)
	}
	return result
}

func Test_Scanner_015(t *testing.T) {
	// Test new character tokens for Markdown support
	tests := []struct {
		in   string
		kind tokenizer.TokenKind
	}{
		{"`", tokenizer.Backtick},
		{"~", tokenizer.Tilde},
		{"|", tokenizer.Pipe},
		{"\\", tokenizer.Backslash},
		{"@", tokenizer.At},
		{"^", tokenizer.Caret},
		{"%", tokenizer.Percent},
		{"$", tokenizer.Dollar},
	}

	for _, test := range tests {
		t.Run(test.in, func(t *testing.T) {
			assert := assert.New(t)
			tokens, err := tokenizer.NewScanner(strings.NewReader(test.in), tokenizer.Pos{}).Tokens()
			assert.NoError(err)
			assert.Equal(1, len(tokens))
			assert.Equal(test.kind, tokens[0].Kind)
			assert.Equal(test.in, tokens[0].Val)
		})
	}
}

func Test_Scanner_016(t *testing.T) {
	// Test sequences of new character tokens
	tests := []struct {
		in       string
		expected string
	}{
		{"```", "Backtick Backtick Backtick"},
		{"~~~", "Tilde Tilde Tilde"},
		{"||", "Pipe Pipe"},
		{"\\\\", "Backslash Backslash"},
		{"`code`", "Backtick Ident Backtick"},
		{"~~strike~~", "Tilde Tilde Ident Tilde Tilde"},
		{"a|b|c", "Ident Pipe Ident Pipe Ident"},
		{"100%", "NumberInteger Percent"},
		{"$100", "Dollar NumberInteger"},
		{"x^2", "Ident Caret NumberInteger"},
		{"@user", "At Ident"},
	}

	for _, test := range tests {
		t.Run(test.in, func(t *testing.T) {
			assert := assert.New(t)
			tokens, err := tokenizer.NewScanner(strings.NewReader(test.in), tokenizer.Pos{}).Tokens()
			assert.NoError(err)
			assert.Equal(test.expected, toTokenString(tokens))
		})
	}
}

func Test_Scanner_017(t *testing.T) {
	// Test feature flags - comments disabled by default
	tests := []struct {
		in       string
		features tokenizer.Feature
		expected string
	}{
		// Without features, # is a Hash token
		{"# heading", 0, "Hash Space Ident"},
		// With HashComment, # starts a comment
		{"# heading", tokenizer.HashComment, "Comment"},
		// Without features, // is two Slash tokens
		{"// comment", 0, "Slash Slash Space Ident"},
		// With LineComment, // starts a comment
		{"// comment", tokenizer.LineComment, "Comment"},
		// Without features, /* is Slash Star
		{"/* block */", 0, "Slash Star Space Ident Space Star Slash"},
		// With BlockComment, /* starts a block comment
		{"/* block */", tokenizer.BlockComment, "Comment"},
		// Multiple features can be combined
		{"# hash\n// line\n/* block */", tokenizer.HashComment | tokenizer.LineComment | tokenizer.BlockComment, "Comment Space Comment Space Comment"},
		// Only some features enabled
		{"# hash\n// line", tokenizer.HashComment, "Comment Space Slash Slash Space Ident"},
		{"# hash\n// line", tokenizer.LineComment, "Hash Space Ident Space Comment"},
	}

	for _, test := range tests {
		t.Run(test.in, func(t *testing.T) {
			assert := assert.New(t)
			tokens, err := tokenizer.NewScanner(strings.NewReader(test.in), tokenizer.Pos{}, test.features).Tokens()
			assert.NoError(err)
			assert.Equal(test.expected, toTokenString(tokens))
		})
	}
}

func Test_Scanner_018(t *testing.T) {
	// Test UnderscoreToken feature flag
	tests := []struct {
		in       string
		features tokenizer.Feature
		expected string
	}{
		// With UnderscoreToken, underscores are separate tokens
		{"hello_world", tokenizer.UnderscoreToken, "Ident Underscore Ident"},
		{"_private", tokenizer.UnderscoreToken, "Underscore Ident"},
		{"value_", tokenizer.UnderscoreToken, "Ident Underscore"},
		{"a_b_c", tokenizer.UnderscoreToken, "Ident Underscore Ident Underscore Ident"},
		{"__init__", tokenizer.UnderscoreToken, "Underscore Underscore Ident Underscore Underscore"},
		// Without UnderscoreToken, underscores are part of identifiersers
		{"hello_world", 0, "Ident"},
		{"_private", 0, "Ident"},
		{"value_", 0, "Ident"},
		{"a_b_c", 0, "Ident"},
		{"__init__", 0, "Ident"},
		// Mixed with other tokens
		{"_italic_", tokenizer.UnderscoreToken, "Underscore Ident Underscore"},
		{"_italic_", 0, "Ident"},
	}

	for _, test := range tests {
		t.Run(test.in, func(t *testing.T) {
			assert := assert.New(t)
			tokens, err := tokenizer.NewScanner(strings.NewReader(test.in), tokenizer.Pos{}, test.features).Tokens()
			assert.NoError(err)
			assert.Equal(test.expected, toTokenString(tokens))
		})
	}
}

func Test_Scanner_019(t *testing.T) {
	// Test NewlineToken feature flag
	tests := []struct {
		in       string
		features tokenizer.Feature
		expected string
	}{
		// Without NewlineToken, newlines are part of Space
		{"a\nb", 0, "Ident Space Ident"},
		{"a\n\nb", 0, "Ident Space Ident"},
		{"a \n b", 0, "Ident Space Ident"},
		// With NewlineToken, newlines are separate tokens
		{"a\nb", tokenizer.NewlineToken, "Ident Newline Ident"},
		{"a\n\nb", tokenizer.NewlineToken, "Ident Newline Newline Ident"},
		{"a \n b", tokenizer.NewlineToken, "Ident Space Newline Space Ident"},
		// Multiple newlines
		{"a\n\n\nb", tokenizer.NewlineToken, "Ident Newline Newline Newline Ident"},
		// Newline at start and end
		{"\na\n", tokenizer.NewlineToken, "Newline Ident Newline"},
		// Mixed whitespace
		{"a  \n  b", tokenizer.NewlineToken, "Ident Space Newline Space Ident"},
		{"\t\n\t", tokenizer.NewlineToken, "Space Newline Space"},
		// Without feature, all whitespace combined
		{"\t\n\t", 0, "Space"},
	}

	for _, test := range tests {
		t.Run(test.in, func(t *testing.T) {
			assert := assert.New(t)
			tokens, err := tokenizer.NewScanner(strings.NewReader(test.in), tokenizer.Pos{}, test.features).Tokens()
			assert.NoError(err)
			assert.Equal(test.expected, toTokenString(tokens))
		})
	}
}

func Test_Scanner_020(t *testing.T) {
	// Test HyphenIdentToken feature flag
	tests := []struct {
		in       string
		features tokenizer.Feature
		expected string
		values   []string
	}{
		{"X-ORIGIN", tokenizer.HyphenIdentToken, "Ident", []string{"X-ORIGIN"}},
		{"NO-USER-MODIFICATION", tokenizer.HyphenIdentToken, "Ident", []string{"NO-USER-MODIFICATION"}},
		{"SINGLE-VALUE", tokenizer.HyphenIdentToken, "Ident", []string{"SINGLE-VALUE"}},
		{"hello-world", tokenizer.HyphenIdentToken, "Ident", []string{"hello-world"}},
		{"X-ORIGIN", 0, "Ident Minus Ident", []string{"X", "-", "ORIGIN"}},
		{"hello-world", 0, "Ident Minus Ident", []string{"hello", "-", "world"}},
		{"hello-world_test", tokenizer.HyphenIdentToken, "Ident", []string{"hello-world_test"}},
		{"hello-world_test", tokenizer.HyphenIdentToken | tokenizer.UnderscoreToken, "Ident Underscore Ident", []string{"hello-world", "_", "test"}},
	}

	for _, test := range tests {
		t.Run(test.in, func(t *testing.T) {
			assert := assert.New(t)
			tokens, err := tokenizer.NewScanner(strings.NewReader(test.in), tokenizer.Pos{}, test.features).Tokens()
			assert.NoError(err)
			assert.Equal(test.expected, toTokenString(tokens))
			assert.Equal(test.values, toTokenValues(tokens))
		})
	}
}

func Test_Scanner_021(t *testing.T) {
	// Test quote token feature flags
	tests := []struct {
		in       string
		features tokenizer.Feature
		expected string
		values   []string
	}{
		{"story's", tokenizer.SingleQuoteToken, "Ident SingleQuote Ident", []string{"story", "'", "s"}},
		{"\"quoted\"", tokenizer.DoubleQuoteToken, "DoubleQuote Ident DoubleQuote", []string{"\"", "quoted", "\""}},
		{"'quoted'", tokenizer.SingleQuoteToken, "SingleQuote Ident SingleQuote", []string{"'", "quoted", "'"}},
		{"say \"hello\"", tokenizer.DoubleQuoteToken, "Ident Space DoubleQuote Ident DoubleQuote", []string{"say", " ", "\"", "hello", "\""}},
		{"rock 'n' roll", tokenizer.SingleQuoteToken, "Ident Space SingleQuote Ident SingleQuote Space Ident", []string{"rock", " ", "'", "n", "'", " ", "roll"}},
		{"'mix' \"both\"", tokenizer.SingleQuoteToken | tokenizer.DoubleQuoteToken, "SingleQuote Ident SingleQuote Space DoubleQuote Ident DoubleQuote", []string{"'", "mix", "'", " ", "\"", "both", "\""}},
	}

	for _, test := range tests {
		t.Run(test.in, func(t *testing.T) {
			assert := assert.New(t)
			tokens, err := tokenizer.NewScanner(strings.NewReader(test.in), tokenizer.Pos{}, test.features).Tokens()
			assert.NoError(err)
			assert.Equal(test.expected, toTokenString(tokens))
			assert.Equal(test.values, toTokenValues(tokens))
		})
	}
}

func Test_Scanner_022(t *testing.T) {
	assert := assert.New(t)
	scanner := tokenizer.NewScanner(strings.NewReader(`"unterminated`), tokenizer.Pos{})

	tok := scanner.Peak()
	assert.NotNil(tok)
	assert.Equal(tokenizer.EOF, tok.Kind)
	assert.Error(scanner.Err())

	tok = scanner.Next()
	assert.NotNil(tok)
	assert.Equal(tokenizer.EOF, tok.Kind)
	assert.Error(scanner.Err())
	assert.Contains(scanner.Err().Error(), "ErrBadParameter")
}