From 72195b2ffb96f9ce56c4b530acbc3ed3d61b1fcc Mon Sep 17 00:00:00 2001
From: jeanpaulsio <hello@jpsio.me>
Date: Thu, 26 Mar 2026 22:57:49 -0700
Subject: [PATCH] fix: handle heredocs with trailing code (foo(<<~SQL),
 <<~HEREDOC.strip)

The heredoc tokenizer now uses a hybrid approach:
- No trailing code: full token spanning opener + body + delimiter (existing behavior)
- Trailing code after opener: opener-only token so the parser handles ), .method, etc.

This fixes parse errors for patterns like foo(<<~SQL) and <<~HEREDOC.strip
while keeping all 105 existing tests passing.
---
 src/syntax.grammar |   9 ++--
 src/tokens.ts      | 114 +++++++++++++++++++++++++--------------------
 2 files changed, 69 insertions(+), 54 deletions(-)

diff --git a/src/syntax.grammar b/src/syntax.grammar
index 6a63b44..c2b4482 100644
--- a/src/syntax.grammar
+++ b/src/syntax.grammar
@@ -439,10 +439,11 @@ InterpolationEnd[openedBy=InterpolationStart] { "}" }
   @else StringContent
 }
 
-// Heredocs are matched as a single opaque token by the external tokenizer.
-// The tokenizer reads <<[-~]?DELIM, scans to the closing DELIM line, and
-// emits the entire heredoc as one Heredoc token. Interpolation inside
-// heredocs is not yet supported (would need @local tokens + ContextTracker).
+// Heredocs: emitted as a single Heredoc token by lessThanTokenizer.
+// When there's no trailing code after the opener, the token spans the entire
+// heredoc (opener + body + closing delimiter). When there IS trailing code
+// (e.g. foo(<<~SQL) or <<~HEREDOC.strip), only the opener is emitted so
+// the parser can handle the trailing syntax correctly.
 
 Array { "[" commaSep<expression> "]" }
 Hash { ~blockOrHash "{" (HashPair ("," HashPair)* ","?)? "}" ~blockOrHash }
diff --git a/src/tokens.ts b/src/tokens.ts
index 75e1e2e..4fa9d2c 100644
--- a/src/tokens.ts
+++ b/src/tokens.ts
@@ -59,10 +59,15 @@ export const regexTokenizer = new ExternalTokenizer((input, stack) => {
 // ============================================================
 // Less-than / Heredoc external tokenizer (#10)
 //
-// `<` is ambiguous: comparison, `<=`, `<<` left shift, or heredoc start.
+// `<` is ambiguous: comparison, `<=`, `<<` left shift, or heredoc.
 // When `<<` is followed by [-~]?IDENTIFIER (or quoted string), and the
-// parser allows Heredoc, we scan to the matching closing delimiter
-// and emit the entire heredoc as one token.
+// parser allows Heredoc, we emit a Heredoc token.
+//
+// Two modes depending on trailing code after the opener:
+// 1. No trailing code: token spans opener + body + closing delimiter
+// 2. Trailing code (e.g. foo(<<~SQL) or <<~HEREDOC.strip): token spans
+//    only the opener, allowing the parser to handle trailing syntax
+//
 // Otherwise emit lessThanOp or lessThanEqOp for comparison.
 // ============================================================
 
@@ -74,9 +79,9 @@ export const lessThanTokenizer = new ExternalTokenizer((input, stack) => {
   // Try heredoc: <<
   if (second === 60 /* '<' */) {
     if (stack.canShift(Heredoc)) {
-      const heredocLen = tryMatchHeredoc(input)
-      if (heredocLen > 0) {
-        input.acceptToken(Heredoc, heredocLen)
+      const result = tryMatchHeredoc(input)
+      if (result) {
+        input.acceptToken(Heredoc, result)
         return
       }
     }
@@ -105,14 +110,21 @@ export const lessThanTokenizer = new ExternalTokenizer((input, stack) => {
   }
 })
 
-// Try to match a complete heredoc starting at the current position.
-// Returns the total length including the closing delimiter, or 0 if no match.
-function tryMatchHeredoc(input: {peek(offset: number): number}): number {
+// Try to match a heredoc at the current position.
+// Returns the total token length, or null if no match.
+//
+// Two modes:
+// 1. No trailing code after opener: token spans opener + body + closing delimiter
+//    e.g. <<~SQL\n  SELECT 1\nSQL → entire thing is one token
+// 2. Trailing code after opener: token spans only the opener
+//    e.g. <<~SQL in foo(<<~SQL) → just <<~SQL is the token
+function tryMatchHeredoc(input: {peek(offset: number): number}): number | null {
   let pos = 2 // past <<
 
   // Optional - or ~
   const modifier = input.peek(pos)
-  if (modifier === 45 /* - */ || modifier === 126 /* ~ */) pos++
+  const indented = modifier === 45 /* - */ || modifier === 126 /* ~ */
+  if (indented) pos++
 
   // Read the delimiter
   let delimiter = ""
@@ -123,80 +135,82 @@ function tryMatchHeredoc(input: {peek(offset: number): number}): number {
     pos++
     while (true) {
       const ch = input.peek(pos)
-      if (ch === -1 || ch === 10) return 0 // unterminated quote
+      if (ch === -1 || ch === 10) return null // unterminated quote
       if (ch === quoteChar) { pos++; break }
       delimiter += String.fromCharCode(ch)
       pos++
     }
   } else {
     // Bare identifier delimiter: <<~DELIM
-    if (!isIdentStart(input.peek(pos))) return 0
+    if (!isIdentStart(input.peek(pos))) return null
     while (isIdentChar(input.peek(pos))) {
       delimiter += String.fromCharCode(input.peek(pos))
       pos++
     }
   }
 
-  if (!delimiter) return 0
+  if (!delimiter) return null
+
+  const openerLength = pos // This is the length of just <<~DELIM
 
-  // After the delimiter, only whitespace/comments allowed on the rest of the line.
-  // If there's code (like .method or (args)), this is << left-shift, not a heredoc.
+  // Check what follows the delimiter on the same line.
+  // For bare identifiers without a modifier, non-whitespace trailing code
+  // means this is << operator, not heredoc. E.g. <<File.expand_path(...)
+  // With a modifier (<<~ or <<-) or quoted delimiter, trailing code is OK.
+  let scanPos = pos
+  let hasTrailingCode = false
   while (true) {
-    const ch = input.peek(pos)
-    if (ch === -1) return 0
-    if (ch === 10) { pos++; break }
-    if (ch === 13) { pos++; if (input.peek(pos) === 10) pos++; break }
-    if (ch === 32 || ch === 9) { pos++; continue } // whitespace OK
+    const ch = input.peek(scanPos)
+    if (ch === -1) return null // EOF on opener line, no body possible
+    if (ch === 10) { scanPos++; break }
+    if (ch === 13) { scanPos++; if (input.peek(scanPos) === 10) scanPos++; break }
+    if (ch === 32 || ch === 9) { scanPos++; continue }
     if (ch === 35 /* # */) { // comment — skip rest of line
       while (true) {
-        const c = input.peek(pos)
+        const c = input.peek(scanPos)
         if (c === -1 || c === 10 || c === 13) break
-        pos++
+        scanPos++
       }
       continue
     }
-    // For bare identifiers (not quoted), any non-whitespace after delimiter
-    // means this is << operator, not heredoc. E.g. <<File.expand_path(...)
-    if (quoteChar !== 39 && quoteChar !== 34 && quoteChar !== 96) return 0
-    pos++ // quoted delimiters can have trailing content (e.g. <<~"SQL", other_arg)
+    // Bare identifier without modifier + trailing code = not a heredoc
+    if (quoteChar !== 39 && quoteChar !== 34 && quoteChar !== 96 &&
+        !indented) return null
+    hasTrailingCode = true
+    scanPos++ // modifier/quoted → skip trailing code
   }
 
-  // Scan lines looking for the closing delimiter
-  const isIndented = modifier === 45 || modifier === 126 // <<- or <<~
+  // Verify the closing delimiter exists in the remaining input
+  const bodyStart = scanPos
   while (true) {
     let lineContent = ""
-
-    // For indented heredocs (<<- or <<~), skip leading whitespace
-    if (isIndented) {
-      while (input.peek(pos) === 32 || input.peek(pos) === 9) pos++
+    if (indented) {
+      while (input.peek(scanPos) === 32 || input.peek(scanPos) === 9) scanPos++
     }
-
-    // Read the rest of the line
     while (true) {
-      const ch = input.peek(pos)
+      const ch = input.peek(scanPos)
       if (ch === -1 || ch === 10 || ch === 13) break
       lineContent += String.fromCharCode(ch)
-      pos++
+      scanPos++
     }
-
-    // Check if this line matches the delimiter (trimmed)
     if (lineContent === delimiter) {
-      // Include the delimiter line in the token
-      // Advance past newline if present
-      const ch = input.peek(pos)
-      if (ch === 10) pos++
-      else if (ch === 13) { pos++; if (input.peek(pos) === 10) pos++ }
-      return pos
+      // Found closing delimiter
+      if (hasTrailingCode) {
+        // Trailing code present — emit only the opener so the parser
+        // can handle ), .method, etc. on the same line
+        return openerLength
+      }
+      // No trailing code — emit the full heredoc (opener + body + delimiter)
+      return scanPos
     }
-
-    // Advance past newline
-    const ch = input.peek(pos)
-    if (ch === -1) return pos // unterminated heredoc — emit what we have
-    if (ch === 10) pos++
-    else if (ch === 13) { pos++; if (input.peek(pos) === 10) pos++ }
+    const ch = input.peek(scanPos)
+    if (ch === -1) return null // EOF without closing delimiter
+    if (ch === 10) scanPos++
+    else if (ch === 13) { scanPos++; if (input.peek(scanPos) === 10) scanPos++ }
   }
 }
 
+
 // ============================================================
 // Percent literal external tokenizer (#11)
 //