feat: move %-literals to external tokenizer with full delimiter support (#11) (#19)

Replace inline PercentStringLiteral token with external tokenizer that:
- Supports ALL delimiters, not just brackets: %w|a b|, %q!hello!, etc.
- Handles paired bracket nesting: %Q(foo(bar)) tracks depth correctly
- Disambiguates % as modulo vs percent literal using stack.canShift()
- Rejects alphanumeric characters as delimiters (prevents %3 misparse)

Same pattern as regex/division and less-than/heredoc tokenizers:
try the longer match (percent literal) first, fall back to the
single-char operator (moduloOp aliased to ArithOp).

83 tests passing (3 new: pipe delimiter, exclamation delimiter, modulo).

Author: jeanpaulsio

src/syntax.grammar

@@ -238,6 +238,7 @@ Arg {
 BinaryExpression {
   expression !add ArithOp expression |
   expression !add "-" expression |
+  expression !mul moduloOp expression |
   expression !mul divideOp expression |
   expression !mul "*" expression |
   expression !exp "**" expression |
@@ -364,18 +365,13 @@ semi { ";" | newline }
   Symbol { ":" $[a-zA-Z_] $[a-zA-Z0-9_]* }
   CharacterLiteral { "?" ("\\" _ | ![$ \t\n\r]) }
 
-  // %-literals: %w[a b], %i[foo], %q(str), %Q(str), %r(regex), %()
-  // Supports [] () {} <> delimiters. Other delimiters (e.g. %w|a b|) need
-  // an external tokenizer and are deferred.
-  PercentStringLiteral {
-    "%" $[wWiIqQrsx]? "[" (![\\\]] | "\\" _)* "]" |
-    "%" $[wWiIqQrsx]? "(" (![\\)] | "\\" _)* ")" |
-    "%" $[wWiIqQrsx]? "{" (![\\}] | "\\" _)* "}" |
-    "%" $[wWiIqQrsx]? "<" (![\\>] | "\\" _)* ">"
-  }
+  // %-literals are now handled by the external tokenizer to support
+  // all delimiters (not just brackets) and proper content scanning.
   // "/" is NOT in ArithOp — it's handled by the external tokenizer which
   // emits either Regex or divideOp depending on context.
-  ArithOp { "+" | "%" }
+  // "%" is NOT in ArithOp — handled by external tokenizer which emits
+  // either PercentStringLiteral or moduloOp depending on context.
+  ArithOp { "+" }
   // "<" and "<=" are NOT in CompareOp — they're handled by the external
   // tokenizer which also emits heredocStart for <<DELIM.
   CompareOp { "==" | "!=" | ">" | ">=" | "<=>" | "===" }
@@ -396,7 +392,6 @@ semi { ";" | newline }
     Identifier,
     Symbol,
     CharacterLiteral,
-    PercentStringLiteral,
     CompareOp,
     "::",
     "=>",
@@ -416,6 +411,7 @@ semi { ";" | newline }
 // External tokenizers for context-dependent tokens
 // External tokenizers for context-dependent tokens
 @external tokens regexTokenizer from "./tokens" { Regex, divideOp[@name=ArithOp] }
+@external tokens percentLiteralTokenizer from "./tokens" { PercentStringLiteral, moduloOp[@name=ArithOp] }
 @external tokens lessThanTokenizer from "./tokens" {
   Heredoc,
   lessThanOp[@name=CompareOp],

src/syntax.grammar.terms.d.ts

@@ -4,3 +4,5 @@ export declare const Heredoc: number
 export declare const lessThanOp: number
 export declare const lessThanEqOp: number
 export declare const inheritsOp: number
+export declare const PercentStringLiteral: number
+export declare const moduloOp: number

src/tokens.ts

@@ -2,6 +2,7 @@ import {ExternalTokenizer} from "@lezer/lr"
 import {
   Regex, divideOp,
   Heredoc, lessThanOp, lessThanEqOp, inheritsOp,
+  PercentStringLiteral, moduloOp,
 } from "./syntax.grammar.terms"
 
 // ============================================================
@@ -175,6 +176,88 @@ function tryMatchHeredoc(input: {peek(offset: number): number}): number {
   }
 }
 
+// ============================================================
+// Percent literal external tokenizer (#11)
+//
+// Matches %w[], %i[], %q(), %Q(), %r(), %x(), %s(), %(), and
+// any single-character delimiter: %w|a b|, %q!hello!, etc.
+//
+// Bracket delimiters ([], (), {}, <>) are paired — the tokenizer
+// tracks nesting depth. Non-bracket delimiters use the same
+// character for open and close.
+// ============================================================
+
+const BRACKET_PAIRS: Record<number, number> = {
+  91: 93,   // [ → ]
+  40: 41,   // ( → )
+  123: 125, // { → }
+  60: 62,   // < → >
+}
+
+export const percentLiteralTokenizer = new ExternalTokenizer((input, stack) => {
+  if (input.next !== 37 /* '%' */) return
+
+  // Try to match a percent literal
+  const literalLen = tryMatchPercentLiteral(input)
+  if (literalLen > 0 && stack.canShift(PercentStringLiteral)) {
+    input.acceptToken(PercentStringLiteral, literalLen)
+    return
+  }
+
+  // Fall back to modulo operator
+  if (stack.canShift(moduloOp)) {
+    input.acceptToken(moduloOp, 1)
+  }
+})
+
+function tryMatchPercentLiteral(input: {peek(offset: number): number}): number {
+  let pos = 1
+  const modifier = input.peek(pos)
+
+  // Optional type modifier: w W i I q Q r x s
+  if (modifier >= 65 && modifier <= 90 || modifier >= 97 && modifier <= 122) {
+    const valid = [119, 87, 105, 73, 113, 81, 114, 120, 115] // w W i I q Q r x s
+    if (valid.indexOf(modifier) === -1) return 0
+    pos++
+  }
+
+  // Read the delimiter character
+  const openDelim = input.peek(pos)
+  // Reject whitespace, EOF, and alphanumeric characters as delimiters.
+  // Digits would cause %3 or %300 to be misread as percent literals.
+  if (openDelim === -1 || openDelim === 32 || openDelim === 9 ||
+      openDelim === 10 || openDelim === 13 ||
+      (openDelim >= 48 && openDelim <= 57) ||   // 0-9
+      (openDelim >= 65 && openDelim <= 90) ||    // A-Z
+      (openDelim >= 97 && openDelim <= 122)) return 0  // a-z
+
+  // Don't match %letter that isn't followed by a delimiter (e.g. % in arithmetic)
+  pos++
+  const closeDelim = BRACKET_PAIRS[openDelim] || openDelim
+
+  if (BRACKET_PAIRS[openDelim]) {
+    let depth = 1
+    while (depth > 0) {
+      const ch = input.peek(pos)
+      if (ch === -1) return pos // unterminated
+      if (ch === 92 /* \\ */) { pos += 2; continue }
+      if (ch === openDelim) depth++
+      if (ch === closeDelim) depth--
+      pos++
+    }
+  } else {
+    while (true) {
+      const ch = input.peek(pos)
+      if (ch === -1) return pos // unterminated
+      if (ch === 92 /* \\ */) { pos += 2; continue }
+      if (ch === closeDelim) { pos++; break }
+      pos++
+    }
+  }
+
+  return pos
+}
+
 function isIdentStart(ch: number): boolean {
   return (ch >= 65 && ch <= 90) || (ch >= 97 && ch <= 122) || ch === 95
 }

test/literals.txt

@@ -187,3 +187,27 @@ RUBY
 ==>
 
 Program(ExpressionStatement(Heredoc))
+
+# Percent literal with pipe delimiter
+
+%w|foo bar baz|
+
+==>
+
+Program(ExpressionStatement(PercentStringLiteral))
+
+# Percent literal with exclamation delimiter
+
+%q!hello world!
+
+==>
+
+Program(ExpressionStatement(PercentStringLiteral))
+
+# Modulo operator (not percent literal)
+
+10 % 3
+
+==>
+
+Program(ExpressionStatement(BinaryExpression(Integer, ArithOp, Integer)))