Fix lexing for unterminated strings/heredocs etc.

When we hit EOF and still have lex modes left, it means some content was unterminated.
Heredocs specifically have logic that needs to happen when the body finished lexing.
If we don't reset the mode back to how it was before, it will not continue lexing at the correct place.

Author: Earlopain

src/prism.c

@@ -9783,6 +9783,12 @@ parser_lex(pm_parser_t *parser) {
     unsigned int semantic_token_seen = parser->semantic_token_seen;
     parser->semantic_token_seen = true;
 
+    // We'll jump to this label when we are about to encounter an EOF.
+    // If we still have lex_modes on the stack, we pop them so that cleanup
+    // can happen. For example, we should still continue parsing after a heredoc
+    // identifier, even if the heredoc body was syntax invalid.
+    switch_lex_mode:
+
     switch (parser->lex_modes.current->mode) {
         case PM_LEX_DEFAULT:
         case PM_LEX_EMBEXPR:
@@ -9856,6 +9862,14 @@ parser_lex(pm_parser_t *parser) {
             // We'll check if we're at the end of the file. If we are, then we
             // need to return the EOF token.
             if (parser->current.end >= parser->end) {
+                // We may be missing closing tokens. We should pop modes one by one
+                // to do the appropriate cleanup like moving next_start for heredocs.
+                // Only when no mode is remaining will we actually emit the EOF token.
+                if (parser->lex_modes.current->mode != PM_LEX_DEFAULT) {
+                    lex_mode_pop(parser);
+                    goto switch_lex_mode;
+                }
+
                 // If we hit EOF, but the EOF came immediately after a newline,
                 // set the start of the token to the newline.  This way any EOF
                 // errors will be reported as happening on that line rather than

test/prism/errors/unterminated_heredoc_and_embexpr.txt

@@ -0,0 +1,11 @@
+<<A+B
+  ^ unterminated heredoc; can't find string "A" anywhere before EOF
+   ^ unexpected '+', ignoring it
+  ^ unterminated heredoc; can't find string "A" anywhere before EOF
+#{C
+   ^ unexpected heredoc ending; expected an argument
+   ^ unexpected heredoc ending, expecting end-of-input
+   ^ unexpected heredoc ending, ignoring it
+   ^ unexpected end-of-input, assuming it is closing the parent top level context
+^ expected a `}` to close the embedded expression
+

test/prism/errors/unterminated_heredoc_and_embexpr_2.txt

@@ -0,0 +1,9 @@
+<<A+B
+  ^ unterminated heredoc; can't find string "A" anywhere before EOF
+#{C + "#{"}
+           ^ unterminated string meets end of file
+           ^ unexpected end-of-input, assuming it is closing the parent top level context
+           ^ expected a `}` to close the embedded expression
+      ^ unterminated string; expected a closing delimiter for the interpolated string
+           ^ expected a `}` to close the embedded expression
+

test/prism/lex_test.rb

@@ -48,11 +48,94 @@ def test_parse_lex_file
     end
 
     if RUBY_VERSION >= "3.3"
-      def test_lex_compare
-        prism = Prism.lex_compat(File.read(__FILE__), version: "current").value
-        ripper = Ripper.lex(File.read(__FILE__))
+      def test_lex_compat
+        source = "foo bar"
+        prism = Prism.lex_compat(source, version: "current").value
+        ripper = Ripper.lex(source)
         assert_equal(ripper, prism)
       end
     end
+
+    def test_lex_interpolation_unterminated
+      assert_lexed(<<~'RUBY'.strip, <<~'LEXED')
+        "#{
+      RUBY
+        STRING_BEGIN(1,0)-(1,1)("\"")
+        EMBEXPR_BEGIN(1,1)-(1,3)("\#{")
+        EOF(1,3)-(1,3)("")
+      LEXED
+
+      assert_lexed(<<~'RUBY', <<~'LEXED')
+        "#{
+      RUBY
+        STRING_BEGIN(1,0)-(1,1)("\"")
+        EMBEXPR_BEGIN(1,1)-(1,3)("\#{")
+        IGNORED_NEWLINE(1,3)-(2,0)("\n")
+        EOF(2,0)-(2,0)("")
+      LEXED
+    end
+
+    def test_lex_interpolation_unterminated_with_content
+      # No trailing newline.
+      # FIXME: Emits EOL twice.
+      assert_lexed(<<~'RUBY'.strip, <<~'LEXED')
+        "#{C
+      RUBY
+        STRING_BEGIN(1,0)-(1,1)("\"")
+        EMBEXPR_BEGIN(1,1)-(1,3)("\#{")
+        CONSTANT(1,3)-(1,4)("C")
+        EOF(1,4)-(1,4)("")
+        EOF(1,4)-(1,4)("")
+      LEXED
+
+      # Trailing newline
+      assert_lexed(<<~'RUBY', <<~'LEXED')
+        "#{C
+      RUBY
+        STRING_BEGIN(1,0)-(1,1)("\"")
+        EMBEXPR_BEGIN(1,1)-(1,3)("\#{")
+        CONSTANT(1,3)-(1,4)("C")
+        NEWLINE(1,4)-(2,0)("\n")
+        EOF(2,0)-(2,0)("")
+      LEXED
+    end
+
+    def test_lex_heredoc_unterminated
+      # No trailing newline
+      assert_lexed(<<~'RUBY'.strip, <<~'LEXED')
+        <<A+B
+        #{C
+      RUBY
+        HEREDOC_START(1,0)-(1,3)("<<A")
+        EMBEXPR_BEGIN(2,0)-(2,2)("\#{")
+        CONSTANT(2,2)-(2,3)("C")
+        HEREDOC_END(2,3)-(2,3)("")
+        PLUS(1,3)-(1,4)("+")
+        CONSTANT(1,4)-(1,5)("B")
+        NEWLINE(1,5)-(2,0)("\n")
+        EOF(2,3)-(2,3)("")
+      LEXED
+
+      # Trailing newline
+      assert_lexed(<<~'RUBY', <<~'LEXED')
+        <<A+B
+        #{C
+      RUBY
+        HEREDOC_START(1,0)-(1,3)("<<A")
+        EMBEXPR_BEGIN(2,0)-(2,2)("\#{")
+        CONSTANT(2,2)-(2,3)("C")
+        NEWLINE(2,3)-(3,0)("\n")
+        HEREDOC_END(3,0)-(3,0)("")
+        PLUS(1,3)-(1,4)("+")
+        CONSTANT(1,4)-(1,5)("B")
+        NEWLINE(1,5)-(2,0)("\n")
+        EOF(2,3)-(3,0)("\n")
+      LEXED
+    end
+
+    def assert_lexed(code, expected)
+      actual = Prism.lex(code).value.map { |token| token[0].pretty_inspect }
+      assert_equal(expected, actual.join)
+    end
   end
 end