feat: add external tokenizer for regex vs division disambiguation (#7) (#16)

Implement the first external tokenizer in src/tokens.ts. The regex/division
ambiguity is resolved by checking parser state:

- At expression-start positions: /pattern/flags produces Regex token
- After a value (identifier, number, ), ]): / produces divideOp (ArithOp)
- Uses stack.canShift() to let the parser state decide

Grammar changes:
- Add Regex back to expression alternatives
- Remove / from ArithOp (handled entirely by external tokenizer)
- Add divideOp[@name=ArithOp] as external token alias

Real-world impact:
- Devise lib: 396 → 343 errors (85.5% clean node rate)
- Regex patterns like @@email_regexp = /\A[^@\s]+@[^@\s]+\z/ now parse

Heredocs (#10) attempted but deferred — inline CompareOp < fires before
external tokenizer can see <<. Needs a different approach.

76 tests passing (4 new regex tests).

Author: jeanpaulsio

src/highlight.ts

@@ -29,6 +29,7 @@ export const rubyHighlighting = styleTags({
   StringEscape: t.escape,
   InterpolationStart: t.special(t.brace),
   InterpolationEnd: t.special(t.brace),
+  Regex: t.regexp,
   LineComment: t.lineComment,
   BlockComment: t.blockComment,
   "( )": t.paren,

src/syntax.grammar

@@ -182,6 +182,7 @@ expression {
   InterpolatedString |
   CharacterLiteral |
   PercentStringLiteral |
+  Regex |
   Symbol |
   Array |
   Hash |
@@ -228,6 +229,7 @@ Arg {
 BinaryExpression {
   expression !add ArithOp expression |
   expression !add "-" expression |
+  expression !mul divideOp expression |
   expression !mul "*" expression |
   expression !exp "**" expression |
   expression !compare CompareOp expression |
@@ -352,10 +354,9 @@ semi { ";" | newline }
     "%" $[wWiIqQrsx]? "{" (![\\}] | "\\" _)* "}" |
     "%" $[wWiIqQrsx]? "<" (![\\>] | "\\" _)* ">"
   }
-  // Regex token removed — it conflicts with division (/). Regex support
-  // requires an external tokenizer (Phase 4) that checks preceding context.
-
-  ArithOp { "+" | "/" | "%" }
+  // "/" is NOT in ArithOp — it's handled by the external tokenizer which
+  // emits either Regex or divideOp depending on context.
+  ArithOp { "+" | "%" }
   CompareOp { "==" | "!=" | "<" | ">" | "<=" | ">=" | "<=>" | "===" }
   AssignOp { "=" | "+=" | "-=" | "*=" | "/=" | "%=" | "||=" | "&&=" | "<<=" | ">>=" }
   LogicOp { "&&" | "||" }
@@ -391,4 +392,9 @@ semi { ";" | newline }
   }
 }
 
+// External tokenizers for context-dependent tokens
+// External tokenizer: handles / as either Regex or division (ArithOp)
+// based on parser state. Regex wins by longest-match when both are valid.
+@external tokens regexTokenizer from "./tokens" { Regex, divideOp[@name=ArithOp] }
+
 @detectDelim

src/syntax.grammar.terms.d.ts

@@ -0,0 +1,2 @@
+export declare const Regex: number
+export declare const divideOp: number

src/tokens.ts

@@ -0,0 +1,62 @@
+import {ExternalTokenizer} from "@lezer/lr"
+import {Regex, divideOp} from "./syntax.grammar.terms"
+
+// ============================================================
+// Regex / Division external tokenizer (#7)
+//
+// `/` is ambiguous: regex literal or division operator.
+// This tokenizer handles both:
+//
+// 1. Try to match /pattern/flags (regex)
+// 2. If the parser state allows Regex AND the pattern is valid, emit Regex
+// 3. Otherwise emit divideOp (aliased to ArithOp) for division
+//
+// The parser state determines which is valid:
+// - At expression-start (after =, (, [, ,, operator): Regex is valid
+// - After a value (identifier, number, ), ]): only divideOp is valid
+// ============================================================
+
+export const regexTokenizer = new ExternalTokenizer((input, stack) => {
+  if (input.next !== 47 /* '/' */) return
+
+  // Try to match a regex pattern
+  let pos = 1
+  let isRegex = true
+  while (true) {
+    const ch = input.peek(pos)
+    if (ch === -1 || ch === 10 /* \n */ || ch === 13 /* \r */) { isRegex = false; break }
+    if (ch === 92 /* \\ */) {
+      pos++
+      // Check the escaped char isn't newline/EOF
+      const next = input.peek(pos)
+      if (next === -1 || next === 10 || next === 13) { isRegex = false; break }
+      pos++
+      continue
+    }
+    if (ch === 47 /* '/' */) { pos++; break }
+    pos++
+  }
+
+  if (isRegex) {
+    // Consume optional flags: i, m, x, o, u, s, n
+    while (true) {
+      const ch = input.peek(pos)
+      if (ch === 105 || ch === 109 || ch === 120 || ch === 111 ||
+          ch === 117 || ch === 115 || ch === 110) {
+        pos++
+      } else {
+        break
+      }
+    }
+
+    if (stack.canShift(Regex)) {
+      input.acceptToken(Regex, pos)
+      return
+    }
+  }
+
+  // Fall back to division
+  if (stack.canShift(divideOp)) {
+    input.acceptToken(divideOp, 1)
+  }
+})

test/literals.txt

@@ -133,3 +133,35 @@ Program(ExpressionStatement(PercentStringLiteral))
 ==>
 
 Program(ExpressionStatement(PercentStringLiteral))
+
+# Regex literal
+
+/pattern/i
+
+==>
+
+Program(ExpressionStatement(Regex))
+
+# Regex after assignment
+
+x = /hello\sworld/m
+
+==>
+
+Program(Assignment(Identifier, AssignOp, Regex))
+
+# Division not regex
+
+10 / 2
+
+==>
+
+Program(ExpressionStatement(BinaryExpression(Integer, ArithOp, Integer)))
+
+# Regex after operator
+
+x + /test/
+
+==>
+
+Program(ExpressionStatement(BinaryExpression(Identifier, ArithOp, Regex)))