Merge pull request #3871 from Earlopain/lex-ripper-plain

Remove `Prism.lex_ripper`

Author: kddnewton

Steepfile

@@ -10,7 +10,6 @@ target :lib do
   # TODO: Type-checking these files is still WIP
   ignore "lib/prism/desugar_compiler.rb"
   ignore "lib/prism/lex_compat.rb"
-  ignore "lib/prism/lex_ripper.rb"
   ignore "lib/prism/serialize.rb"
   ignore "lib/prism/ffi.rb"
   ignore "lib/prism/translation"

bin/prism

@@ -210,18 +210,10 @@ module Prism
 
     # bin/prism lex_compat [source]
     def lex_compat(argv)
+      require "ripper"
       source, filepath = read_source(argv)
 
-      ripper_value =
-        begin
-          Prism.lex_ripper(source)
-        rescue ArgumentError, SyntaxError
-          # If Ripper raises a syntax error, we want to continue as if it didn't
-          # return any tokens at all. prism won't raise a syntax error, so it's
-          # nicer to still be able to see the tokens that prism generated.
-          []
-        end
-
+      ripper_value = Ripper.lex(source)
       prism_compat = Prism.lex_compat(source, filepath: filepath)
       prism = Prism.lex(source, filepath: filepath)
 

lib/prism.rb

@@ -20,7 +20,6 @@ module Prism
   autoload :DSL, "prism/dsl"
   autoload :InspectVisitor, "prism/inspect_visitor"
   autoload :LexCompat, "prism/lex_compat"
-  autoload :LexRipper, "prism/lex_ripper"
   autoload :MutationCompiler, "prism/mutation_compiler"
   autoload :Pack, "prism/pack"
   autoload :Pattern, "prism/pattern"
@@ -35,7 +34,6 @@ module Prism
   # private here.
 
   private_constant :LexCompat
-  private_constant :LexRipper
 
   # Raised when requested to parse as the currently running Ruby version but Prism has no support for it.
   class CurrentVersionError < ArgumentError
@@ -68,15 +66,6 @@ def self.lex_compat(source, **options)
     LexCompat.new(source, **options).result # steep:ignore
   end
 
-  # :call-seq:
-  #   Prism::lex_ripper(source) -> Array
-  #
-  # This wraps the result of Ripper.lex. It produces almost exactly the
-  # same tokens. Raises SyntaxError if the syntax in source is invalid.
-  def self.lex_ripper(source)
-    LexRipper.new(source).result # steep:ignore
-  end
-
   # :call-seq:
   #   Prism::load(source, serialized, freeze) -> ParseResult
   #

lib/prism/lex_ripper.rb

@@ -77,7 +77,6 @@ Gem::Specification.new do |spec|
     "lib/prism/ffi.rb",
     "lib/prism/inspect_visitor.rb",
     "lib/prism/lex_compat.rb",
-    "lib/prism/lex_ripper.rb",
     "lib/prism/mutation_compiler.rb",
     "lib/prism/node_ext.rb",
     "lib/prism/node.rb",

prism.gemspec

@@ -1,9 +1,11 @@
 # frozen_string_literal: true
 # typed: ignore
 
+require "ripper"
+
 module Prism
-  # This class is responsible for lexing files with both lex_compat and
-  # lex_ripper and ensuring they match up. It keeps track of the files which
+  # This class is responsible for lexing files with both prism and
+  # ripper and ensuring they match up. It keeps track of the files which
   # failed to match up, and the files which passed.
   class LexTask
     attr_reader :failing_files, :passing_file_count
@@ -28,7 +30,7 @@ module Prism
       end
 
       result = Prism.lex_compat(source)
-      if result.errors.empty? && Prism.lex_ripper(source) == result.value
+      if result.errors.empty? && Ripper.lex(source) == result.value
         @passing_file_count += 1
         true
       else
@@ -98,6 +100,10 @@ TARGETS = {
 
       # Requires an implicit -x, which ripper does not respect
       "tool/merger.rb",
+
+      # Contains `"x#$%"` which looks like bare interpolation but $% is not a valid global.
+      # This confuses ripper, it emits two string tokens. https://bugs.ruby-lang.org/issues/21849
+      "ext/psych/lib/psych/scalar_scanner.rb",
     ]
   },
   discourse: {
@@ -265,6 +271,16 @@ TOP_100_GEMS_INVALID_SYNTAX_PREFIXES = %w[
   top-100-gems/devise-4.9.2/lib/generators/templates/controllers/
   top-100-gems/fastlane-2.212.1/fastlane/lib/assets/custom_action_template.rb
 ]
+TOP_100_GEMS_LEX_RIPPER_BUG = [
+  # Contains code like `"x#$%"` which looks like bare interpolation but $% is not a valid global.
+  # This confuses ripper, it emits two string tokens. https://bugs.ruby-lang.org/issues/21849
+  "faker-3.1.1/lib/faker/default/internet.rb",
+  "ruby_parser-3.20.0/test/test_ruby_parser.rb",
+  "rouge-4.1.0/lib/rouge/lexers/cisco_ios.rb",
+  "rouge-4.1.0/lib/rouge/lexers/ghc_cmm.rb",
+  "rouge-4.1.0/lib/rouge/lexers/nasm.rb",
+  "rouge-4.1.0/lib/rouge/lexers/velocity.rb",
+]
 
 namespace :download do
   directory TOP_100_GEMS_DIR
@@ -346,7 +362,10 @@ task "lex:topgems": ["download:topgems", :compile] do
       lex_task.compare(filepath)
     end
 
-    gem_failing_files = lex_task.failing_files.map { |todo| todo.delete_prefix("#{directory}/") }
+    gem_failing_files = lex_task.failing_files.filter_map do |todo|
+      next if TOP_100_GEMS_LEX_RIPPER_BUG.any? { |path| todo.end_with?(path) }
+      todo.delete_prefix("#{directory}/")
+    end
     failing_files[gem_name] = gem_failing_files if gem_failing_files.any?
   end
 

rakelib/lex.rake

@@ -20,7 +20,6 @@ namespace :typecheck do
     File.write("sorbet/typed_overrides.yml", ERB.new(<<~YAML, trim_mode: "-").result_with_hash(locals))
       false:
         - ./lib/prism/lex_compat.rb
-        - ./lib/prism/lex_ripper.rb
         - ./lib/prism/node_ext.rb
         - ./lib/prism/parse_result.rb
         - ./lib/prism/visitor.rb

rakelib/typecheck.rake

@@ -16,9 +16,6 @@ module Prism
   sig { params(source: String, options: T::Hash[Symbol, T.untyped]).returns(Prism::LexCompat::Result) }
   def self.lex_compat(source, **options); end
 
-  sig { params(source: String).returns(T::Array[T.untyped]) }
-  def self.lex_ripper(source); end
-
   sig { params(source: String, serialized: String, freeze: T.nilable(T::Boolean)).returns(Prism::ParseResult) }
   def self.load(source, serialized, freeze = false); end
 

rbi/prism.rbi

@@ -42,10 +42,6 @@ module Prism
     ?bool freeze
   ) -> ParseResult
 
-  def self.lex_ripper: (
-    String source
-  ) -> Array[[[Integer, Integer], Symbol, String, untyped]]
-
   # Methods taking a path to a Ruby file:
   <%-
     {

templates/sig/prism.rbs.erb

@@ -5,6 +5,7 @@
 return if RUBY_ENGINE != "ruby"
 
 require_relative "test_helper"
+require "ripper"
 
 module Prism
   class BOMTest < TestCase
@@ -53,7 +54,7 @@ def test_string
 
     def assert_bom(source)
       bommed = "\xEF\xBB\xBF#{source}"
-      assert_equal Prism.lex_ripper(bommed), Prism.lex_compat(bommed).value
+      assert_equal Ripper.lex(bommed), Prism.lex_compat(bommed).value
     end
   end
 end