Merge pull request #2723 from ruby/split-parse-result

Split parse result based on type

Author: kddnewton

.github/dependabot.yml

@@ -40,7 +40,7 @@ updates:
     directory: '/gemfiles/truffleruby'
     schedule:
       interval: 'weekly'
-  # - package-ecosystem: 'bundler'
-  #   directory: '/gemfiles/typecheck'
-  #   schedule:
-  #     interval: 'weekly'
+  - package-ecosystem: 'bundler'
+    directory: '/gemfiles/typecheck'
+    schedule:
+      interval: 'weekly'

.github/workflows/main.yml

@@ -38,23 +38,23 @@ jobs:
       env:
         LANG: "C"
 
-  # typecheck:
-  #   runs-on: ubuntu-latest
-  #   env:
-  #     BUNDLE_GEMFILE: gemfiles/typecheck/Gemfile
-  #   steps:
-  #   - uses: actions/checkout@v4
-  #   - name: Set up Ruby
-  #     uses: ruby/setup-ruby@v1
-  #     with:
-  #       ruby-version: "3.3"
-  #       bundler-cache: true
-  #   - name: Check Sorbet
-  #     run: bundle exec rake typecheck:tapioca typecheck:sorbet
-  #   - name: Check Steep
-  #     run: bundle exec rake typecheck:steep
-  #   - name: Check field kinds
-  #     run: rm lib/prism/node.rb && CHECK_FIELD_KIND=true bundle exec rake
+  typecheck:
+    runs-on: ubuntu-latest
+    env:
+      BUNDLE_GEMFILE: gemfiles/typecheck/Gemfile
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Ruby
+      uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: "3.3"
+        bundler-cache: true
+    # - name: Check Sorbet
+    #   run: bundle exec rake typecheck:tapioca typecheck:sorbet
+    - name: Check Steep
+      run: bundle exec rake typecheck:steep
+    - name: Check field kinds
+      run: rm lib/prism/node.rb && CHECK_FIELD_KIND=true bundle exec rake
 
   build:
     strategy:

ext/prism/extension.c

@@ -19,7 +19,9 @@ VALUE rb_cPrismEmbDocComment;
 VALUE rb_cPrismMagicComment;
 VALUE rb_cPrismParseError;
 VALUE rb_cPrismParseWarning;
+VALUE rb_cPrismResult;
 VALUE rb_cPrismParseResult;
+VALUE rb_cPrismParseLexResult;
 
 VALUE rb_cPrismDebugEncoding;
 
@@ -515,7 +517,7 @@ parser_warnings(pm_parser_t *parser, rb_encoding *encoding, VALUE source) {
  * Create a new parse result from the given parser, value, encoding, and source.
  */
 static VALUE
-parse_result_create(pm_parser_t *parser, VALUE value, rb_encoding *encoding, VALUE source) {
+parse_result_create(VALUE class, pm_parser_t *parser, VALUE value, rb_encoding *encoding, VALUE source) {
     VALUE result_argv[] = {
         value,
         parser_comments(parser, source),
@@ -526,7 +528,7 @@ parse_result_create(pm_parser_t *parser, VALUE value, rb_encoding *encoding, VAL
         source
     };
 
-    return rb_class_new_instance(7, result_argv, rb_cPrismParseResult);
+    return rb_class_new_instance(7, result_argv, class);
 }
 
 /******************************************************************************/
@@ -635,7 +637,7 @@ parse_lex_input(pm_string_t *input, const pm_options_t *options, bool return_nod
         value = parse_lex_data.tokens;
     }
 
-    VALUE result = parse_result_create(&parser, value, parse_lex_data.encoding, source);
+    VALUE result = parse_result_create(rb_cPrismParseLexResult, &parser, value, parse_lex_data.encoding, source);
     pm_node_destroy(&parser, node);
     pm_parser_free(&parser);
 
@@ -700,7 +702,7 @@ parse_input(pm_string_t *input, const pm_options_t *options) {
 
     VALUE source = pm_source_new(&parser, encoding);
     VALUE value = pm_ast_new(&parser, node, encoding, source);
-    VALUE result = parse_result_create(&parser, value, encoding, source) ;
+    VALUE result = parse_result_create(rb_cPrismParseResult, &parser, value, encoding, source) ;
 
     pm_node_destroy(&parser, node);
     pm_parser_free(&parser);
@@ -804,7 +806,7 @@ parse_stream(int argc, VALUE *argv, VALUE self) {
 
     VALUE source = pm_source_new(&parser, encoding);
     VALUE value = pm_ast_new(&parser, node, encoding, source);
-    VALUE result = parse_result_create(&parser, value, encoding, source);
+    VALUE result = parse_result_create(rb_cPrismParseResult, &parser, value, encoding, source);
 
     pm_node_destroy(&parser, node);
     pm_buffer_free(&buffer);
@@ -1362,7 +1364,10 @@ Init_prism(void) {
     rb_cPrismMagicComment = rb_define_class_under(rb_cPrism, "MagicComment", rb_cObject);
     rb_cPrismParseError = rb_define_class_under(rb_cPrism, "ParseError", rb_cObject);
     rb_cPrismParseWarning = rb_define_class_under(rb_cPrism, "ParseWarning", rb_cObject);
-    rb_cPrismParseResult = rb_define_class_under(rb_cPrism, "ParseResult", rb_cObject);
+
+    rb_cPrismResult = rb_define_class_under(rb_cPrism, "Result", rb_cObject);
+    rb_cPrismParseResult = rb_define_class_under(rb_cPrism, "ParseResult", rb_cPrismResult);
+    rb_cPrismParseLexResult = rb_define_class_under(rb_cPrism, "ParseLexResult", rb_cPrismResult);
 
     // Intern all of the options that we support so that we don't have to do it
     // every time we parse.

gemfiles/typecheck/Gemfile.lock

@@ -29,7 +29,7 @@ GEM
     fileutils (1.7.2)
     i18n (1.14.4)
       concurrent-ruby (~> 1.0)
-    json (2.7.1)
+    json (2.7.2)
     language_server-protocol (3.17.0.3)
     listen (3.9.0)
       rb-fsevent (~> 0.10, >= 0.10.3)
@@ -51,8 +51,8 @@ GEM
     rb-fsevent (0.11.2)
     rb-inotify (0.10.1)
       ffi (~> 1.0)
-    rbi (0.1.10)
-      prism (>= 0.18.0, < 0.25)
+    rbi (0.1.11)
+      prism (>= 0.18.0, < 0.27)
       sorbet-runtime (>= 0.5.9204)
     rbs (3.4.4)
       abbrev
@@ -61,15 +61,15 @@ GEM
       sexp_processor (~> 4.16)
     securerandom (0.3.1)
     sexp_processor (4.17.1)
-    sorbet (0.5.11319)
-      sorbet-static (= 0.5.11319)
-    sorbet-runtime (0.5.11319)
-    sorbet-static (0.5.11319-aarch64-linux)
-    sorbet-static (0.5.11319-universal-darwin)
-    sorbet-static (0.5.11319-x86_64-linux)
-    sorbet-static-and-runtime (0.5.11319)
-      sorbet (= 0.5.11319)
-      sorbet-runtime (= 0.5.11319)
+    sorbet (0.5.11351)
+      sorbet-static (= 0.5.11351)
+    sorbet-runtime (0.5.11351)
+    sorbet-static (0.5.11351-aarch64-linux)
+    sorbet-static (0.5.11351-universal-darwin)
+    sorbet-static (0.5.11351-x86_64-linux)
+    sorbet-static-and-runtime (0.5.11351)
+      sorbet (= 0.5.11351)
+      sorbet-runtime (= 0.5.11351)
     spoom (1.3.0)
       erubi (>= 1.10.0)
       prism (>= 0.19.0)
@@ -91,7 +91,7 @@ GEM
       strscan (>= 1.0.0)
       terminal-table (>= 2, < 4)
     strscan (3.1.0)
-    tapioca (0.13.1)
+    tapioca (0.13.3)
       bundler (>= 2.2.25)
       netrc (>= 0.11.0)
       parallel (>= 1.21.0)

lib/prism.rb

@@ -37,7 +37,7 @@ module Prism
   private_constant :LexRipper
 
   # :call-seq:
-  #   Prism::lex_compat(source, **options) -> ParseResult
+  #   Prism::lex_compat(source, **options) -> LexCompat::Result
   #
   # Returns a parse result whose value is an array of tokens that closely
   # resembles the return value of Ripper::lex. The main difference is that the

lib/prism/ffi.rb

@@ -350,7 +350,7 @@ def parse_lex_common(string, code, options) # :nodoc:
         node, comments, magic_comments, data_loc, errors, warnings = loader.load_nodes
         tokens.each { |token,| token.value.force_encoding(loader.encoding) }
 
-        ParseResult.new([node, tokens], comments, magic_comments, data_loc, errors, warnings, source)
+        ParseLexResult.new([node, tokens], comments, magic_comments, data_loc, errors, warnings, source)
       end
     end
 

lib/prism/lex_compat.rb

@@ -10,6 +10,23 @@ module Prism
   # generally lines up. However, there are a few cases that require special
   # handling.
   class LexCompat # :nodoc:
+    # A result class specialized for holding tokens produced by the lexer.
+    class Result < Prism::Result
+      # The list of tokens that were produced by the lexer.
+      attr_reader :value
+
+      # Create a new lex compat result object with the given values.
+      def initialize(value, comments, magic_comments, data_loc, errors, warnings, source)
+        @value = value
+        super(comments, magic_comments, data_loc, errors, warnings, source)
+      end
+
+      # Implement the hash pattern matching interface for Result.
+      def deconstruct_keys(keys)
+        super.merge!(value: value)
+      end
+    end
+
     # This is a mapping of prism token types to Ripper token types. This is a
     # many-to-one mapping because we split up our token types, whereas Ripper
     # tends to group them.
@@ -844,7 +861,7 @@ def result
       # We sort by location to compare against Ripper's output
       tokens.sort_by!(&:location)
 
-      ParseResult.new(tokens, result.comments, result.magic_comments, result.data_loc, result.errors, result.warnings, Source.new(source))
+      Result.new(tokens, result.comments, result.magic_comments, result.data_loc, result.errors, result.warnings, Source.new(source))
     end
   end
 

lib/prism/parse_result.rb

@@ -438,14 +438,9 @@ def inspect
   end
 
   # This represents the result of a call to ::parse or ::parse_file. It contains
-  # the AST, any comments that were encounters, and any errors that were
-  # encountered.
-  class ParseResult
-    # The value that was generated by parsing. Normally this holds the AST, but
-    # it can sometimes how a list of tokens or other results passed back from
-    # the parser.
-    attr_reader :value
-
+  # the requested structure, any comments that were encounters, and any errors
+  # that were encountered.
+  class Result
     # The list of comments that were encountered during parsing.
     attr_reader :comments
 
@@ -466,9 +461,8 @@ class ParseResult
     # A Source instance that represents the source code that was parsed.
     attr_reader :source
 
-    # Create a new parse result object with the given values.
-    def initialize(value, comments, magic_comments, data_loc, errors, warnings, source)
-      @value = value
+    # Create a new result object with the given values.
+    def initialize(comments, magic_comments, data_loc, errors, warnings, source)
       @comments = comments
       @magic_comments = magic_comments
       @data_loc = data_loc
@@ -477,9 +471,9 @@ def initialize(value, comments, magic_comments, data_loc, errors, warnings, sour
       @source = source
     end
 
-    # Implement the hash pattern matching interface for ParseResult.
+    # Implement the hash pattern matching interface for Result.
     def deconstruct_keys(keys)
-      { value: value, comments: comments, magic_comments: magic_comments, data_loc: data_loc, errors: errors, warnings: warnings }
+      { comments: comments, magic_comments: magic_comments, data_loc: data_loc, errors: errors, warnings: warnings }
     end
 
     # Returns the encoding of the source code that was parsed.
@@ -500,6 +494,58 @@ def failure?
     end
   end
 
+  # This is a result specific to the `parse` and `parse_file` methods.
+  class ParseResult < Result
+    # The syntax tree that was parsed from the source code.
+    attr_reader :value
+
+    # Create a new parse result object with the given values.
+    def initialize(value, comments, magic_comments, data_loc, errors, warnings, source)
+      @value = value
+      super(comments, magic_comments, data_loc, errors, warnings, source)
+    end
+
+    # Implement the hash pattern matching interface for ParseResult.
+    def deconstruct_keys(keys)
+      super.merge!(value: value)
+    end
+  end
+
+  # This is a result specific to the `lex` and `lex_file` methods.
+  class LexResult < Result
+    # The list of tokens that were parsed from the source code.
+    attr_reader :value
+
+    # Create a new lex result object with the given values.
+    def initialize(value, comments, magic_comments, data_loc, errors, warnings, source)
+      @value = value
+      super(comments, magic_comments, data_loc, errors, warnings, source)
+    end
+
+    # Implement the hash pattern matching interface for LexResult.
+    def deconstruct_keys(keys)
+      super.merge!(value: value)
+    end
+  end
+
+  # This is a result specific to the `parse_lex` and `parse_lex_file` methods.
+  class ParseLexResult < Result
+    # A tuple of the syntax tree and the list of tokens that were parsed from
+    # the source code.
+    attr_reader :value
+
+    # Create a new parse lex result object with the given values.
+    def initialize(value, comments, magic_comments, data_loc, errors, warnings, source)
+      @value = value
+      super(comments, magic_comments, data_loc, errors, warnings, source)
+    end
+
+    # Implement the hash pattern matching interface for ParseLexResult.
+    def deconstruct_keys(keys)
+      super.merge!(value: value)
+    end
+  end
+
   # This represents a token from the Ruby source.
   class Token
     # The Source object that represents the source this token came from.

lib/prism/parse_result/newlines.rb

@@ -58,8 +58,6 @@ def visit_statements_node(node)
 
     # Walk the tree and mark nodes that are on a new line.
     def mark_newlines!
-      value = self.value
-      raise "This method should only be called on a parse result that contains a node" unless Node === value
       value.accept(Newlines.new(Array.new(1 + source.offsets.size, false))) # steep:ignore
     end
   end

rbi/prism.rbi

@@ -7,28 +7,28 @@ module Prism
   sig { params(filepath: String, command_line: T.nilable(String), encoding: T.nilable(T.any(String, Encoding)), frozen_string_literal: T.nilable(T::Boolean), line: T.nilable(Integer), scopes: T.nilable(T::Array[T::Array[Symbol]]), version: T.nilable(String)).returns(String) }
   def self.dump_file(filepath, command_line: nil, encoding: nil, frozen_string_literal: nil, line: nil, scopes: nil, version: nil); end
 
-  sig { params(source: String, command_line: T.nilable(String), encoding: T.nilable(T.any(String, Encoding)), filepath: T.nilable(String), frozen_string_literal: T.nilable(T::Boolean), line: T.nilable(Integer), scopes: T.nilable(T::Array[T::Array[Symbol]]), version: T.nilable(String)).returns(Prism::ParseResult[T::Array[T.untyped]]) }
+  sig { params(source: String, command_line: T.nilable(String), encoding: T.nilable(T.any(String, Encoding)), filepath: T.nilable(String), frozen_string_literal: T.nilable(T::Boolean), line: T.nilable(Integer), scopes: T.nilable(T::Array[T::Array[Symbol]]), version: T.nilable(String)).returns(Prism::LexResult) }
   def self.lex(source, command_line: nil, encoding: nil, filepath: nil, frozen_string_literal: nil, line: nil, scopes: nil, version: nil); end
 
-  sig { params(filepath: String, command_line: T.nilable(String), encoding: T.nilable(T.any(String, Encoding)), frozen_string_literal: T.nilable(T::Boolean), line: T.nilable(Integer), scopes: T.nilable(T::Array[T::Array[Symbol]]), version: T.nilable(String)).returns(Prism::ParseResult[T::Array[T.untyped]]) }
+  sig { params(filepath: String, command_line: T.nilable(String), encoding: T.nilable(T.any(String, Encoding)), frozen_string_literal: T.nilable(T::Boolean), line: T.nilable(Integer), scopes: T.nilable(T::Array[T::Array[Symbol]]), version: T.nilable(String)).returns(Prism::LexResult) }
   def self.lex_file(filepath, command_line: nil, encoding: nil, frozen_string_literal: nil, line: nil, scopes: nil, version: nil); end
 
-  sig { params(source: String, options: T::Hash[Symbol, T.untyped]).returns(Prism::ParseResult[T::Array[T.untyped]]) }
+  sig { params(source: String, options: T::Hash[Symbol, T.untyped]).returns(Prism::LexCompat::Result) }
   def self.lex_compat(source, **options); end
 
   sig { params(source: String).returns(T::Array[T.untyped]) }
   def self.lex_ripper(source); end
 
-  sig { params(source: String, serialized: String).returns(Prism::ParseResult[Prism::ProgramNode]) }
+  sig { params(source: String, serialized: String).returns(Prism::ParseResult) }
   def self.load(source, serialized); end
 
-  sig { params(source: String, command_line: T.nilable(String), encoding: T.nilable(T.any(String, Encoding)), filepath: T.nilable(String), frozen_string_literal: T.nilable(T::Boolean), line: T.nilable(Integer), scopes: T.nilable(T::Array[T::Array[Symbol]]), version: T.nilable(String)).returns(Prism::ParseResult[Prism::ProgramNode]) }
+  sig { params(source: String, command_line: T.nilable(String), encoding: T.nilable(T.any(String, Encoding)), filepath: T.nilable(String), frozen_string_literal: T.nilable(T::Boolean), line: T.nilable(Integer), scopes: T.nilable(T::Array[T::Array[Symbol]]), version: T.nilable(String)).returns(Prism::ParseResult) }
   def self.parse(source, command_line: nil, encoding: nil, filepath: nil, frozen_string_literal: nil, line: nil, scopes: nil, version: nil); end
 
-  sig { params(filepath: String, command_line: T.nilable(String), encoding: T.nilable(T.any(String, Encoding)), frozen_string_literal: T.nilable(T::Boolean), line: T.nilable(Integer), scopes: T.nilable(T::Array[T::Array[Symbol]]), version: T.nilable(String)).returns(Prism::ParseResult[Prism::ProgramNode]) }
+  sig { params(filepath: String, command_line: T.nilable(String), encoding: T.nilable(T.any(String, Encoding)), frozen_string_literal: T.nilable(T::Boolean), line: T.nilable(Integer), scopes: T.nilable(T::Array[T::Array[Symbol]]), version: T.nilable(String)).returns(Prism::ParseResult) }
   def self.parse_file(filepath, command_line: nil, encoding: nil, frozen_string_literal: nil, line: nil, scopes: nil, version: nil); end
 
-  sig { params(stream: T.any(IO, StringIO), command_line: T.nilable(String), encoding: T.nilable(T.any(String, Encoding)), filepath: T.nilable(String), frozen_string_literal: T.nilable(T::Boolean), line: T.nilable(Integer), scopes: T.nilable(T::Array[T::Array[Symbol]]), version: T.nilable(String)).returns(Prism::ParseResult[Prism::ProgramNode]) }
+  sig { params(stream: T.any(IO, StringIO), command_line: T.nilable(String), encoding: T.nilable(T.any(String, Encoding)), filepath: T.nilable(String), frozen_string_literal: T.nilable(T::Boolean), line: T.nilable(Integer), scopes: T.nilable(T::Array[T::Array[Symbol]]), version: T.nilable(String)).returns(Prism::ParseResult) }
   def self.parse_stream(stream, command_line: nil, encoding: nil, filepath: nil, frozen_string_literal: nil, line: nil, scopes: nil, version: nil); end
 
   sig { params(source: String, command_line: T.nilable(String), encoding: T.nilable(T.any(String, Encoding)), filepath: T.nilable(String), frozen_string_literal: T.nilable(T::Boolean), line: T.nilable(Integer), scopes: T.nilable(T::Array[T::Array[Symbol]]), version: T.nilable(String)).returns(T::Array[Prism::Comment]) }
@@ -37,10 +37,10 @@ module Prism
   sig { params(filepath: String, command_line: T.nilable(String), encoding: T.nilable(T.any(String, Encoding)), frozen_string_literal: T.nilable(T::Boolean), line: T.nilable(Integer), scopes: T.nilable(T::Array[T::Array[Symbol]]), version: T.nilable(String)).returns(T::Array[Prism::Comment]) }
   def self.parse_file_comments(filepath, command_line: nil, encoding: nil, frozen_string_literal: nil, line: nil, scopes: nil, version: nil); end
 
-  sig { params(source: String, command_line: T.nilable(String), encoding: T.nilable(T.any(String, Encoding)), filepath: T.nilable(String), frozen_string_literal: T.nilable(T::Boolean), line: T.nilable(Integer), scopes: T.nilable(T::Array[T::Array[Symbol]]), version: T.nilable(String)).returns(Prism::ParseResult[[Prism::ProgramNode, T::Array[T.untyped]]]) }
+  sig { params(source: String, command_line: T.nilable(String), encoding: T.nilable(T.any(String, Encoding)), filepath: T.nilable(String), frozen_string_literal: T.nilable(T::Boolean), line: T.nilable(Integer), scopes: T.nilable(T::Array[T::Array[Symbol]]), version: T.nilable(String)).returns(Prism::ParseLexResult) }
   def self.parse_lex(source, command_line: nil, encoding: nil, filepath: nil, frozen_string_literal: nil, line: nil, scopes: nil, version: nil); end
 
-  sig { params(filepath: String, command_line: T.nilable(String), encoding: T.nilable(T.any(String, Encoding)), frozen_string_literal: T.nilable(T::Boolean), line: T.nilable(Integer), scopes: T.nilable(T::Array[T::Array[Symbol]]), version: T.nilable(String)).returns(Prism::ParseResult[[Prism::ProgramNode, T::Array[T.untyped]]]) }
+  sig { params(filepath: String, command_line: T.nilable(String), encoding: T.nilable(T.any(String, Encoding)), frozen_string_literal: T.nilable(T::Boolean), line: T.nilable(Integer), scopes: T.nilable(T::Array[T::Array[Symbol]]), version: T.nilable(String)).returns(Prism::ParseLexResult) }
   def self.parse_lex_file(filepath, command_line: nil, encoding: nil, frozen_string_literal: nil, line: nil, scopes: nil, version: nil); end
 
   sig { params(source: String, command_line: T.nilable(String), encoding: T.nilable(T.any(String, Encoding)), filepath: T.nilable(String), frozen_string_literal: T.nilable(T::Boolean), line: T.nilable(Integer), scopes: T.nilable(T::Array[T::Array[Symbol]]), version: T.nilable(String)).returns(T::Boolean) }