Merge pull request #3978 from ruby/continuable

Refine continuable? with algorithm in C

Author: kddnewton

docs/serialization.md

@@ -106,6 +106,7 @@ The header is structured like the following table:
 | error* | errors |
 | varuint | number of warnings |
 | warning* | warnings |
+| `1` | `1` if the source is continuable (incomplete but could become valid with more input), `0` otherwise |
 | `4` | content pool offset |
 | varuint | content pool size |
 

ext/prism/extension.c

@@ -641,10 +641,11 @@ parse_result_create(VALUE class, const pm_parser_t *parser, VALUE value, rb_enco
         parser_data_loc(parser, source, freeze),
         parser_errors(parser, encoding, source, freeze),
         parser_warnings(parser, encoding, source, freeze),
+        parser->continuable ? Qtrue : Qfalse,
         source
     };
 
-    return rb_class_new_instance_freeze(7, result_argv, class, freeze);
+    return rb_class_new_instance_freeze(8, result_argv, class, freeze);
 }
 
 /******************************************************************************/

include/prism/parser.h

@@ -895,6 +895,14 @@ struct pm_parser {
     /** Whether or not we're currently recovering from a syntax error. */
     bool recovering;
 
+    /**
+     * Whether or not the source being parsed could become valid if more input
+     * were appended. This is set to false when the parser encounters a token
+     * that is definitively wrong (e.g., a stray `end` or `]`) as opposed to
+     * merely incomplete.
+     */
+    bool continuable;
+
     /**
      * This is very specialized behavior for when you want to parse in a context
      * that does not respect encoding comments. Its main use case is translating

java/org/ruby_lang/prism/ParseResult.java

@@ -69,14 +69,16 @@ public Warning(Nodes.WarningType type, String message, Nodes.Location location,
     public final Nodes.Location dataLocation;
     public final Error[] errors;
     public final Warning[] warnings;
+    public final boolean continuable;
     public final Nodes.Source source;
 
-    public ParseResult(Nodes.Node value, MagicComment[] magicComments, Nodes.Location dataLocation, Error[] errors, Warning[] warnings, Nodes.Source source) {
+    public ParseResult(Nodes.Node value, MagicComment[] magicComments, Nodes.Location dataLocation, Error[] errors, Warning[] warnings, boolean continuable, Nodes.Source source) {
         this.value = value;
         this.magicComments = magicComments;
         this.dataLocation = dataLocation;
         this.errors = errors;
         this.warnings = warnings;
+        this.continuable = continuable;
         this.source = source;
     }
 }

lib/prism/lex_compat.rb

@@ -43,10 +43,10 @@ class Result < Prism::Result
 
       # Create a new lex compat result object with the given values.
       #--
-      #: (Array[lex_compat_token] value, Array[Comment] comments, Array[MagicComment] magic_comments, Location? data_loc, Array[ParseError] errors, Array[ParseWarning] warnings, Source source) -> void
-      def initialize(value, comments, magic_comments, data_loc, errors, warnings, source)
+      #: (Array[lex_compat_token] value, Array[Comment] comments, Array[MagicComment] magic_comments, Location? data_loc, Array[ParseError] errors, Array[ParseWarning] warnings, bool continuable, Source source) -> void
+      def initialize(value, comments, magic_comments, data_loc, errors, warnings, continuable, source)
         @value = value
-        super(comments, magic_comments, data_loc, errors, warnings, source)
+        super(comments, magic_comments, data_loc, errors, warnings, continuable, source)
       end
 
       # Implement the hash pattern matching interface for Result.
@@ -825,7 +825,7 @@ def result
 
       tokens = post_process_tokens(tokens, source, result.data_loc, bom, eof_token)
 
-      Result.new(tokens, result.comments, result.magic_comments, result.data_loc, result.errors, result.warnings, source)
+      Result.new(tokens, result.comments, result.magic_comments, result.data_loc, result.errors, result.warnings, result.continuable?, source)
     end
 
     private

lib/prism/parse_result.rb

@@ -898,13 +898,14 @@ class Result
 
     # Create a new result object with the given values.
     #--
-    #: (Array[Comment] comments, Array[MagicComment] magic_comments, Location? data_loc, Array[ParseError] errors, Array[ParseWarning] warnings, Source source) -> void
-    def initialize(comments, magic_comments, data_loc, errors, warnings, source)
+    #: (Array[Comment] comments, Array[MagicComment] magic_comments, Location? data_loc, Array[ParseError] errors, Array[ParseWarning] warnings, bool continuable, Source source) -> void
+    def initialize(comments, magic_comments, data_loc, errors, warnings, continuable, source)
       @comments = comments
       @magic_comments = magic_comments
       @data_loc = data_loc
       @errors = errors
       @warnings = warnings
+      @continuable = continuable
       @source = source
     end
 
@@ -961,54 +962,8 @@ def failure?
     #--
     #: () -> bool
     def continuable?
-      return false if errors.empty?
-
-      offset = source.source.bytesize
-      errors.all? { |error| CONTINUABLE.include?(error.type) || error.location.start_offset == offset }
-    end
-
-    # The set of error types whose location the parser places at the opening
-    # token of an unclosed construct rather than at the end of the source. These
-    # errors always indicate incomplete input regardless of their byte position,
-    # so they are checked by type rather than by location.
-    #--
-    #: Array[Symbol]
-    CONTINUABLE = %i[
-      begin_term
-      begin_upcase_term
-      block_param_pipe_term
-      block_term_brace
-      block_term_end
-      case_missing_conditions
-      case_term
-      class_term
-      conditional_term
-      conditional_term_else
-      def_term
-      embdoc_term
-      end_upcase_term
-      for_term
-      hash_term
-      heredoc_term
-      lambda_term_brace
-      lambda_term_end
-      list_i_lower_term
-      list_i_upper_term
-      list_w_lower_term
-      list_w_upper_term
-      module_term
-      regexp_term
-      rescue_term
-      string_interpolated_term
-      string_literal_eof
-      symbol_term_dynamic
-      symbol_term_interpolated
-      until_term
-      while_term
-      xstring_term
-    ].freeze
-
-    private_constant :CONTINUABLE
+      @continuable
+    end
 
     # Create a code units cache for the given encoding.
     #--
@@ -1033,10 +988,10 @@ class ParseResult < Result
 
     # Create a new parse result object with the given values.
     #--
-    #: (ProgramNode value, Array[Comment] comments, Array[MagicComment] magic_comments, Location? data_loc, Array[ParseError] errors, Array[ParseWarning] warnings, Source source) -> void
-    def initialize(value, comments, magic_comments, data_loc, errors, warnings, source)
+    #: (ProgramNode value, Array[Comment] comments, Array[MagicComment] magic_comments, Location? data_loc, Array[ParseError] errors, Array[ParseWarning] warnings, bool continuable, Source source) -> void
+    def initialize(value, comments, magic_comments, data_loc, errors, warnings, continuable, source)
       @value = value
-      super(comments, magic_comments, data_loc, errors, warnings, source)
+      super(comments, magic_comments, data_loc, errors, warnings, continuable, source)
     end
 
     # Implement the hash pattern matching interface for ParseResult.
@@ -1077,10 +1032,10 @@ class LexResult < Result
 
     # Create a new lex result object with the given values.
     #--
-    #: (Array[[Token, Integer]] value, Array[Comment] comments, Array[MagicComment] magic_comments, Location? data_loc, Array[ParseError] errors, Array[ParseWarning] warnings, Source source) -> void
-    def initialize(value, comments, magic_comments, data_loc, errors, warnings, source)
+    #: (Array[[Token, Integer]] value, Array[Comment] comments, Array[MagicComment] magic_comments, Location? data_loc, Array[ParseError] errors, Array[ParseWarning] warnings, bool continuable, Source source) -> void
+    def initialize(value, comments, magic_comments, data_loc, errors, warnings, continuable, source)
       @value = value
-      super(comments, magic_comments, data_loc, errors, warnings, source)
+      super(comments, magic_comments, data_loc, errors, warnings, continuable, source)
     end
 
     # Implement the hash pattern matching interface for LexResult.
@@ -1099,10 +1054,10 @@ class ParseLexResult < Result
 
     # Create a new parse lex result object with the given values.
     #--
-    #: ([ProgramNode, Array[[Token, Integer]]] value, Array[Comment] comments, Array[MagicComment] magic_comments, Location? data_loc, Array[ParseError] errors, Array[ParseWarning] warnings, Source source) -> void
-    def initialize(value, comments, magic_comments, data_loc, errors, warnings, source)
+    #: ([ProgramNode, Array[[Token, Integer]]] value, Array[Comment] comments, Array[MagicComment] magic_comments, Location? data_loc, Array[ParseError] errors, Array[ParseWarning] warnings, bool continuable, Source source) -> void
+    def initialize(value, comments, magic_comments, data_loc, errors, warnings, continuable, source)
       @value = value
-      super(comments, magic_comments, data_loc, errors, warnings, source)
+      super(comments, magic_comments, data_loc, errors, warnings, continuable, source)
     end
 
     # Implement the hash pattern matching interface for ParseLexResult.