feat: add heredoc support via external tokenizer (#10) (#17)

Move < and <= out of CompareOp into an external tokenizer that also
handles heredoc detection. When << is followed by [-~]?IDENTIFIER
(or quoted string) and Heredoc is valid in the parser state, the
tokenizer scans to the matching closing delimiter and emits the
entire heredoc as one opaque token.

Supports:
- <<DELIM ... DELIM (strict)
- <<-DELIM ... DELIM (indented closing)
- <<~DELIM ... DELIM (squiggly — strips indent from closing match)
- Quoted delimiters: <<~"DELIM", <<~'DELIM'

Comparison operators still work:
- a < b → CompareOp (via lessThanOp external token)
- a <= b → CompareOp (via lessThanEqOp external token)

Known limitations:
- Heredoc body is one opaque token (no interpolation highlighting inside)
- << left-shift operator not distinguished from heredoc start
- <<~ indent stripping is delimiter-only (body whitespace preserved)

79 tests passing (3 new heredoc tests).

Author: jeanpaulsio

src/highlight.ts

@@ -30,6 +30,7 @@ export const rubyHighlighting = styleTags({
   InterpolationStart: t.special(t.brace),
   InterpolationEnd: t.special(t.brace),
   Regex: t.regexp,
+  Heredoc: t.string,
   LineComment: t.lineComment,
   BlockComment: t.blockComment,
   "( )": t.paren,

src/syntax.grammar

@@ -183,6 +183,7 @@ expression {
   CharacterLiteral |
   PercentStringLiteral |
   Regex |
+  Heredoc |
   Symbol |
   Array |
   Hash |
@@ -233,6 +234,8 @@ BinaryExpression {
   expression !mul "*" expression |
   expression !exp "**" expression |
   expression !compare CompareOp expression |
+  expression !compare lessThanOp expression |
+  expression !compare lessThanEqOp expression |
   expression !logic LogicOp expression |
   expression !logic kw<"and"> expression |
   expression !logic kw<"or"> expression
@@ -295,6 +298,11 @@ InterpolationEnd[openedBy=InterpolationStart] { "}" }
   @else StringContent
 }
 
+// Heredocs are matched as a single opaque token by the external tokenizer.
+// The tokenizer reads <<[-~]?DELIM, scans to the closing DELIM line, and
+// emits the entire heredoc as one Heredoc token. Interpolation inside
+// heredocs is not yet supported (would need @local tokens + ContextTracker).
+
 Array { "[" commaSep<expression> "]" }
 Hash { "{" (HashPair ("," HashPair)*)? "}" }
 HashPair { expression "=>" expression | Identifier ":" expression }
@@ -357,7 +365,9 @@ semi { ";" | newline }
   // "/" is NOT in ArithOp — it's handled by the external tokenizer which
   // emits either Regex or divideOp depending on context.
   ArithOp { "+" | "%" }
-  CompareOp { "==" | "!=" | "<" | ">" | "<=" | ">=" | "<=>" | "===" }
+  // "<" and "<=" are NOT in CompareOp — they're handled by the external
+  // tokenizer which also emits heredocStart for <<DELIM.
+  CompareOp { "==" | "!=" | ">" | ">=" | "<=>" | "===" }
   AssignOp { "=" | "+=" | "-=" | "*=" | "/=" | "%=" | "||=" | "&&=" | "<<=" | ">>=" }
   LogicOp { "&&" | "||" }
 
@@ -393,8 +403,12 @@ semi { ";" | newline }
 }
 
 // External tokenizers for context-dependent tokens
-// External tokenizer: handles / as either Regex or division (ArithOp)
-// based on parser state. Regex wins by longest-match when both are valid.
+// External tokenizers for context-dependent tokens
 @external tokens regexTokenizer from "./tokens" { Regex, divideOp[@name=ArithOp] }
+@external tokens lessThanTokenizer from "./tokens" {
+  Heredoc,
+  lessThanOp[@name=CompareOp],
+  lessThanEqOp[@name=CompareOp]
+}
 
 @detectDelim

src/syntax.grammar.terms.d.ts

@@ -1,2 +1,5 @@
 export declare const Regex: number
 export declare const divideOp: number
+export declare const Heredoc: number
+export declare const lessThanOp: number
+export declare const lessThanEqOp: number

src/tokens.ts

@@ -1,33 +1,27 @@
 import {ExternalTokenizer} from "@lezer/lr"
-import {Regex, divideOp} from "./syntax.grammar.terms"
+import {
+  Regex, divideOp,
+  Heredoc, lessThanOp, lessThanEqOp,
+} from "./syntax.grammar.terms"
 
 // ============================================================
 // Regex / Division external tokenizer (#7)
 //
 // `/` is ambiguous: regex literal or division operator.
-// This tokenizer handles both:
-//
-// 1. Try to match /pattern/flags (regex)
-// 2. If the parser state allows Regex AND the pattern is valid, emit Regex
-// 3. Otherwise emit divideOp (aliased to ArithOp) for division
-//
-// The parser state determines which is valid:
-// - At expression-start (after =, (, [, ,, operator): Regex is valid
-// - After a value (identifier, number, ), ]): only divideOp is valid
+// Tries to match /pattern/flags; falls back to divideOp.
+// Parser state (stack.canShift) determines which is valid.
 // ============================================================
 
 export const regexTokenizer = new ExternalTokenizer((input, stack) => {
   if (input.next !== 47 /* '/' */) return
 
-  // Try to match a regex pattern
   let pos = 1
   let isRegex = true
   while (true) {
     const ch = input.peek(pos)
-    if (ch === -1 || ch === 10 /* \n */ || ch === 13 /* \r */) { isRegex = false; break }
+    if (ch === -1 || ch === 10 || ch === 13) { isRegex = false; break }
     if (ch === 92 /* \\ */) {
       pos++
-      // Check the escaped char isn't newline/EOF
       const next = input.peek(pos)
       if (next === -1 || next === 10 || next === 13) { isRegex = false; break }
       pos++
@@ -38,7 +32,7 @@ export const regexTokenizer = new ExternalTokenizer((input, stack) => {
   }
 
   if (isRegex) {
-    // Consume optional flags: i, m, x, o, u, s, n
+    // Consume optional flags
     while (true) {
       const ch = input.peek(pos)
       if (ch === 105 || ch === 109 || ch === 120 || ch === 111 ||
@@ -48,15 +42,137 @@ export const regexTokenizer = new ExternalTokenizer((input, stack) => {
         break
       }
     }
-
     if (stack.canShift(Regex)) {
       input.acceptToken(Regex, pos)
       return
     }
   }
 
-  // Fall back to division
   if (stack.canShift(divideOp)) {
     input.acceptToken(divideOp, 1)
   }
 })
+
+// ============================================================
+// Less-than / Heredoc external tokenizer (#10)
+//
+// `<` is ambiguous: comparison, `<=`, `<<` left shift, or heredoc start.
+// When `<<` is followed by [-~]?IDENTIFIER (or quoted string), and the
+// parser allows Heredoc, we scan to the matching closing delimiter
+// and emit the entire heredoc as one token.
+// Otherwise emit lessThanOp or lessThanEqOp for comparison.
+// ============================================================
+
+export const lessThanTokenizer = new ExternalTokenizer((input, stack) => {
+  if (input.next !== 60 /* '<' */) return
+
+  const second = input.peek(1)
+
+  // Try heredoc: <<
+  if (second === 60 /* '<' */ && stack.canShift(Heredoc)) {
+    const heredocLen = tryMatchHeredoc(input)
+    if (heredocLen > 0) {
+      input.acceptToken(Heredoc, heredocLen)
+      return
+    }
+  }
+
+  // <=
+  if (second === 61 /* '=' */ && stack.canShift(lessThanEqOp)) {
+    input.acceptToken(lessThanEqOp, 2)
+    return
+  }
+
+  // Plain <
+  if (stack.canShift(lessThanOp)) {
+    input.acceptToken(lessThanOp, 1)
+  }
+})
+
+// Try to match a complete heredoc starting at the current position.
+// Returns the total length including the closing delimiter, or 0 if no match.
+function tryMatchHeredoc(input: {peek(offset: number): number}): number {
+  let pos = 2 // past <<
+
+  // Optional - or ~
+  const modifier = input.peek(pos)
+  if (modifier === 45 /* - */ || modifier === 126 /* ~ */) pos++
+
+  // Read the delimiter
+  let delimiter = ""
+  const quoteChar = input.peek(pos)
+
+  if (quoteChar === 39 /* ' */ || quoteChar === 34 /* " */ || quoteChar === 96 /* ` */) {
+    // Quoted delimiter: <<~"DELIM"
+    pos++
+    while (true) {
+      const ch = input.peek(pos)
+      if (ch === -1 || ch === 10) return 0 // unterminated quote
+      if (ch === quoteChar) { pos++; break }
+      delimiter += String.fromCharCode(ch)
+      pos++
+    }
+  } else {
+    // Bare identifier delimiter: <<~DELIM
+    if (!isIdentStart(input.peek(pos))) return 0
+    while (isIdentChar(input.peek(pos))) {
+      delimiter += String.fromCharCode(input.peek(pos))
+      pos++
+    }
+  }
+
+  if (!delimiter) return 0
+
+  // Must be followed by newline (or end of opening line)
+  // Skip to end of opening line
+  while (true) {
+    const ch = input.peek(pos)
+    if (ch === -1) return 0 // no newline after heredoc start
+    if (ch === 10) { pos++; break }
+    if (ch === 13) { pos++; if (input.peek(pos) === 10) pos++; break }
+    pos++
+  }
+
+  // Scan lines looking for the closing delimiter
+  const isIndented = modifier === 45 || modifier === 126 // <<- or <<~
+  while (true) {
+    let lineContent = ""
+
+    // For indented heredocs (<<- or <<~), skip leading whitespace
+    if (isIndented) {
+      while (input.peek(pos) === 32 || input.peek(pos) === 9) pos++
+    }
+
+    // Read the rest of the line
+    while (true) {
+      const ch = input.peek(pos)
+      if (ch === -1 || ch === 10 || ch === 13) break
+      lineContent += String.fromCharCode(ch)
+      pos++
+    }
+
+    // Check if this line matches the delimiter (trimmed)
+    if (lineContent === delimiter) {
+      // Include the delimiter line in the token
+      // Advance past newline if present
+      const ch = input.peek(pos)
+      if (ch === 10) pos++
+      else if (ch === 13) { pos++; if (input.peek(pos) === 10) pos++ }
+      return pos
+    }
+
+    // Advance past newline
+    const ch = input.peek(pos)
+    if (ch === -1) return pos // unterminated heredoc — emit what we have
+    if (ch === 10) pos++
+    else if (ch === 13) { pos++; if (input.peek(pos) === 10) pos++ }
+  }
+}
+
+function isIdentStart(ch: number): boolean {
+  return (ch >= 65 && ch <= 90) || (ch >= 97 && ch <= 122) || ch === 95
+}
+
+function isIdentChar(ch: number): boolean {
+  return isIdentStart(ch) || (ch >= 48 && ch <= 57)
+}

test/literals.txt

@@ -165,3 +165,33 @@ x + /test/
 ==>
 
 Program(ExpressionStatement(BinaryExpression(Identifier, ArithOp, Regex)))
+
+# Heredoc
+
+<<~SQL
+  SELECT * FROM users
+SQL
+
+==>
+
+Program(ExpressionStatement(Heredoc))
+
+# Heredoc with assignment
+
+x = <<~HTML
+  <div>hello</div>
+HTML
+
+==>
+
+Program(Assignment(Identifier, AssignOp, Heredoc))
+
+# Heredoc strict (no indent modifier)
+
+<<RUBY
+hello
+RUBY
+
+==>
+
+Program(ExpressionStatement(Heredoc))