Merge pull request #2828 from Earlopain/no-ripper

Use prism to parse comments

Author: soutaro

Gemfile.lock

@@ -3,7 +3,7 @@ PATH
   specs:
     rbs (4.0.0.dev.5)
       logger
-      prism (>= 1.3.0)
+      prism (>= 1.6.0)
       tsort
 
 PATH

lib/rbs.rb

@@ -6,7 +6,6 @@
 require "json"
 require "pathname" unless defined?(Pathname)
 require "pp"
-require "ripper"
 require "logger"
 require "tsort"
 require "strscan"

lib/rbs/prototype/helpers.rb

@@ -5,6 +5,63 @@ module Prototype
     module Helpers
       private
 
+      # Prism can't parse Ruby 3.2 code
+      if RUBY_VERSION >= "3.3"
+        def parse_comments(string, include_trailing:)
+          Prism.parse_comments(string, version: "current").yield_self do |prism_comments| # steep:ignore UnexpectedKeywordArgument
+            prism_comments.each_with_object({}) do |comment, hash| #$ Hash[Integer, AST::Comment]
+              # Skip EmbDoc comments
+              next unless comment.is_a?(Prism::InlineComment)
+              # skip like `module Foo # :nodoc:`
+              next if comment.trailing? && !include_trailing
+
+              line = comment.location.start_line
+              body = "#{comment.location.slice}\n"
+              body = body[2..-1] or raise
+              body = "\n" if body.empty?
+
+              comment = AST::Comment.new(string: body, location: nil)
+              if prev_comment = hash.delete(line - 1)
+                hash[line] = AST::Comment.new(string: prev_comment.string + comment.string,
+                                              location: nil)
+              else
+                hash[line] = comment
+              end
+            end
+          end
+        end
+      else
+        require "ripper"
+        def parse_comments(string, include_trailing:)
+          Ripper.lex(string).yield_self do |tokens|
+            code_lines = {} #: Hash[Integer, bool]
+            tokens.each.with_object({}) do |token, hash| #$ Hash[Integer, AST::Comment]
+              case token[1]
+              when :on_sp, :on_ignored_nl
+                # skip
+              when :on_comment
+                line = token[0][0]
+                # skip like `module Foo # :nodoc:`
+                next if code_lines[line] && !include_trailing
+                body = token[2][2..-1] or raise
+
+                body = "\n" if body.empty?
+
+                comment = AST::Comment.new(string: body, location: nil)
+                if prev_comment = hash.delete(line - 1)
+                  hash[line] = AST::Comment.new(string: prev_comment.string + comment.string,
+                                                location: nil)
+                else
+                  hash[line] = comment
+                end
+              else
+                code_lines[token[0][0]] = true
+              end
+            end
+          end
+        end
+      end
+
       def block_from_body(node)
         _, args_node, body_node = node.children
         _pre_num, _pre_init, _opt, _first_post, _post_num, _post_init, _rest, _kw, _kwrest, block_var = args_from_node(args_node)

lib/rbs/prototype/rb.rb

@@ -74,32 +74,7 @@ def decls
 
       def parse(string)
         # @type var comments: Hash[Integer, AST::Comment]
-        comments = Ripper.lex(string).yield_self do |tokens|
-          code_lines = {} #: Hash[Integer, bool]
-          tokens.each.with_object({}) do |token, hash| #$ Hash[Integer, AST::Comment]
-            case token[1]
-            when :on_sp, :on_ignored_nl
-              # skip
-            when :on_comment
-              line = token[0][0]
-              # skip like `module Foo # :nodoc:`
-              next if code_lines[line]
-              body = token[2][2..-1] or raise
-
-              body = "\n" if body.empty?
-
-              comment = AST::Comment.new(string: body, location: nil)
-              if prev_comment = hash.delete(line - 1)
-                hash[line] = AST::Comment.new(string: prev_comment.string + comment.string,
-                                              location: nil)
-              else
-                hash[line] = comment
-              end
-            else
-              code_lines[token[0][0]] = true
-            end
-          end
-        end
+        comments = parse_comments(string, include_trailing: false)
 
         process RubyVM::AbstractSyntaxTree.parse(string), decls: source_decls, comments: comments, context: Context.initial
       end

lib/rbs/prototype/rbi.rb

@@ -16,26 +16,7 @@ def initialize
       end
 
       def parse(string)
-        comments = Ripper.lex(string).yield_self do |tokens|
-          tokens.each.with_object({}) do |token, hash| #$ Hash[Integer, AST::Comment]
-            if token[1] == :on_comment
-              line = token[0][0]
-              body = token[2][2..-1] or raise
-
-              body = "\n" if body.empty?
-
-              comment = AST::Comment.new(string: body, location: nil)
-              if (prev_comment = hash.delete(line - 1))
-                hash[line] = AST::Comment.new(
-                  string: prev_comment.string + comment.string,
-                  location: nil
-                )
-              else
-                hash[line] = comment
-              end
-            end
-          end
-        end
+        comments = parse_comments(string, include_trailing: true)
         process RubyVM::AbstractSyntaxTree.parse(string), comments: comments
       end
 

rbs.gemspec

@@ -46,6 +46,6 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
   spec.required_ruby_version = ">= 3.2"
   spec.add_dependency "logger"
-  spec.add_dependency "prism", ">= 1.3.0"
+  spec.add_dependency "prism", ">= 1.6.0"
   spec.add_dependency "tsort"
 end

sig/prototype/helpers.rbs

@@ -3,6 +3,8 @@ module RBS
     module Helpers
       type node = RubyVM::AbstractSyntaxTree::Node
 
+      def parse_comments: (String, include_trailing: bool) -> Hash[Integer, AST::Comment]
+
       def block_from_body: (node) -> Types::Block?
 
       def each_node: (Array[untyped] nodes) { (node) -> void } -> void