Rename lex to token where appropriate

I had a bit of trouble following what is going on.
Lexing produces tokens, this renames to reflect that.

Author: Earlopain

lib/syntax_suggest/clean_document.rb

@@ -182,8 +182,8 @@ def join_heredoc!
       start_index_stack = []
       heredoc_beg_end_index = []
       lines.each do |line|
-        line.lex.each do |lex_value|
-          case lex_value.type
+        line.tokens.each do |token|
+          case token.type
           when :on_heredoc_beg
             start_index_stack << line.index
           when :on_heredoc_end
@@ -273,7 +273,7 @@ def join_groups(groups)
 
         # Join group into the first line
         @document[line.index] = CodeLine.new(
-          lex: lines.map(&:lex).flatten,
+          tokens: lines.map(&:tokens).flatten,
           line: lines.join,
           index: line.index
         )
@@ -282,7 +282,7 @@ def join_groups(groups)
         lines[1..].each do |line|
           # The above lines already have newlines in them, if add more
           # then there will be double newline, use an empty line instead
-          @document[line.index] = CodeLine.new(line: "", index: line.index, lex: [])
+          @document[line.index] = CodeLine.new(line: "", index: line.index, tokens: [])
         end
       end
       self

lib/syntax_suggest/code_line.rb

@@ -27,19 +27,20 @@ class CodeLine
     # Returns an array of CodeLine objects
     # from the source string
     def self.from_source(source)
-      lex_array_for_line = LexAll.new(source: source).each_with_object(Hash.new { |h, k| h[k] = [] }) { |lex, hash| hash[lex.line] << lex }
+      tokens = LexAll.new(source: source)
+      tokens_for_line = tokens.each_with_object(Hash.new { |h, k| h[k] = [] }) { |token, hash| hash[token.line] << token }
       source.lines.map.with_index do |line, index|
         CodeLine.new(
           line: line,
           index: index,
-          lex: lex_array_for_line[index + 1]
+          tokens: tokens_for_line[index + 1]
         )
       end
     end
 
-    attr_reader :line, :index, :lex, :line_number, :indent
-    def initialize(line:, index:, lex:)
-      @lex = lex
+    attr_reader :line, :index, :tokens, :line_number, :indent
+    def initialize(line:, index:, tokens:)
+      @tokens = tokens
       @line = line
       @index = index
       @original = line
@@ -180,12 +181,12 @@ def ignore_newline_not_beg?
     #     expect(lines.first.trailing_slash?).to eq(true)
     #
     def trailing_slash?
-      last = @lex.last
+      last = @tokens.last
 
       # Older versions of prism diverged slightly from Ripper in compatibility mode
       case last&.type
       when :on_sp
-        last.token == TRAILING_SLASH
+        last.value == TRAILING_SLASH
       when :on_tstring_end
         true
       else
@@ -209,21 +210,21 @@ def trailing_slash?
       end_count = 0
 
       @ignore_newline_not_beg = false
-      @lex.each do |lex|
-        kw_count += 1 if lex.is_kw?
-        end_count += 1 if lex.is_end?
+      @tokens.each do |token|
+        kw_count += 1 if token.is_kw?
+        end_count += 1 if token.is_end?
 
-        if lex.type == :on_ignored_nl
-          @ignore_newline_not_beg = !lex.expr_beg?
+        if token.type == :on_ignored_nl
+          @ignore_newline_not_beg = !token.expr_beg?
         end
 
         if in_oneliner_def.nil?
-          in_oneliner_def = :ENDFN if lex.state.allbits?(Ripper::EXPR_ENDFN)
-        elsif lex.state.allbits?(Ripper::EXPR_ENDFN)
+          in_oneliner_def = :ENDFN if token.state.allbits?(Ripper::EXPR_ENDFN)
+        elsif token.state.allbits?(Ripper::EXPR_ENDFN)
           # Continue
-        elsif lex.state.allbits?(Ripper::EXPR_BEG)
-          in_oneliner_def = :BODY if lex.token == "="
-        elsif lex.state.allbits?(Ripper::EXPR_END)
+        elsif token.state.allbits?(Ripper::EXPR_BEG)
+          in_oneliner_def = :BODY if token.value == "="
+        elsif token.state.allbits?(Ripper::EXPR_END)
           # We found an endless method, count it
           oneliner_count += 1 if in_oneliner_def == :BODY
 

lib/syntax_suggest/explain_syntax.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require_relative "left_right_lex_count"
+require_relative "left_right_token_count"
 
 module SyntaxSuggest
   class GetParseErrors
@@ -45,14 +45,14 @@ class ExplainSyntax
 
     def initialize(code_lines:)
       @code_lines = code_lines
-      @left_right = LeftRightLexCount.new
+      @left_right = LeftRightTokenCount.new
       @missing = nil
     end
 
     def call
       @code_lines.each do |line|
-        line.lex.each do |lex|
-          @left_right.count_lex(lex)
+        line.tokens.each do |token|
+          @left_right.count_token(token)
         end
       end
 

…b/syntax_suggest/left_right_lex_count.rb …syntax_suggest/left_right_token_count.rblib/syntax_suggest/left_right_lex_count.rb renamed to lib/syntax_suggest/left_right_token_count.rb lib/syntax_suggest/left_right_lex_count.rb renamed to lib/syntax_suggest/left_right_token_count.rb

@@ -9,19 +9,19 @@ module SyntaxSuggest
   #
   # Example:
   #
-  #   left_right = LeftRightLexCount.new
+  #   left_right = LeftRightTokenCount.new
   #   left_right.count_kw
   #   left_right.missing.first
   #   # => "end"
   #
-  #   left_right = LeftRightLexCount.new
+  #   left_right = LeftRightTokenCount.new
   #   source = "{ a: b, c: d" # Note missing '}'
-  #   LexAll.new(source: source).each do |lex|
-  #     left_right.count_lex(lex)
+  #   LexAll.new(source: source).each do |token|
+  #     left_right.count_token(token)
   #   end
   #   left_right.missing.first
   #   # => "}"
-  class LeftRightLexCount
+  class LeftRightTokenCount
     def initialize
       @kw_count = 0
       @end_count = 0
@@ -49,14 +49,14 @@ def count_end
     #
     # Example:
     #
-    #   left_right = LeftRightLexCount.new
-    #   left_right.count_lex(LexValue.new(1, :on_lbrace, "{", Ripper::EXPR_BEG))
+    #   left_right = LeftRightTokenCount.new
+    #   left_right.count_token(Token.new(1, :on_lbrace, "{", Ripper::EXPR_BEG))
     #   left_right.count_for_char("{")
     #   # => 1
     #   left_right.count_for_char("}")
     #   # => 0
-    def count_lex(lex)
-      case lex.type
+    def count_token(token)
+      case token.type
       when :on_tstring_content
         # ^^^
         # Means it's a string or a symbol `"{"` rather than being
@@ -70,7 +70,7 @@ def count_lex(lex)
         # The start token will be the full thing `%Q{` but we
         # need to count it as if it's a `{`. Any token
         # can be used
-        char = lex.token[-1]
+        char = token.value[-1]
         @count_for_char[char] += 1 if @count_for_char.key?(char)
       when :on_embexpr_beg
         # ^^^
@@ -87,14 +87,14 @@ def count_lex(lex)
         # When we see `#{` count it as a `{` or we will
         # have a mis-match count.
         #
-        case lex.token
+        case token.value
         when "\#{"
           @count_for_char["{"] += 1
         end
       else
-        @end_count += 1 if lex.is_end?
-        @kw_count += 1 if lex.is_kw?
-        @count_for_char[lex.token] += 1 if @count_for_char.key?(lex.token)
+        @end_count += 1 if token.is_end?
+        @kw_count += 1 if token.is_kw?
+        @count_for_char[token.value] += 1 if @count_for_char.key?(token.value)
       end
     end
 

lib/syntax_suggest/lex_all.rb

@@ -5,18 +5,18 @@ module SyntaxSuggest
   #
   # Example usage:
   #
-  #   lex = LexAll.new(source: source)
-  #   lex.each do |value|
-  #     puts value.line
+  #   tokens = LexAll.new(source: source)
+  #   tokens.each do |token|
+  #     puts token.line
   #   end
   class LexAll
     include Enumerable
 
     def initialize(source:)
-      @lex = self.class.lex(source, 1)
-      last_lex = nil
-      @lex.map! { |elem|
-        last_lex = LexValue.new(elem[0].first, elem[1], elem[2], elem[3], last_lex)
+      @tokens = self.class.lex(source, 1)
+      last_token = nil
+      @tokens.map! { |elem|
+        last_token = Token.new(elem[0].first, elem[1], elem[2], elem[3], last_token)
       }
     end
 
@@ -25,24 +25,24 @@ def self.lex(source, line_number)
     end
 
     def to_a
-      @lex
+      @tokens
     end
 
     def each
-      return @lex.each unless block_given?
-      @lex.each do |x|
-        yield x
+      return @tokens.each unless block_given?
+      @tokens.each do |token|
+        yield token
       end
     end
 
     def [](index)
-      @lex[index]
+      @tokens[index]
     end
 
     def last
-      @lex.last
+      @tokens.last
     end
   end
 end
 
-require_relative "lex_value"
+require_relative "token"

lib/syntax_suggest/lex_value.rb lib/syntax_suggest/token.rblib/syntax_suggest/lex_value.rb renamed to lib/syntax_suggest/token.rb

@@ -9,29 +9,29 @@ module SyntaxSuggest
   #
   # Would translate into:
   #
-  #  lex.line # => 1
-  #  lex.type # => :on_indent
-  #  lex.token # => "describe"
-  class LexValue
-    attr_reader :line, :type, :token, :state
+  #  token.line # => 1
+  #  token.type # => :on_indent
+  #  token.value # => "describe"
+  class Token
+    attr_reader :line, :type, :value, :state
 
-    def initialize(line, type, token, state, last_lex = nil)
+    def initialize(line, type, value, state, last_token = nil)
       @line = line
       @type = type
-      @token = token
+      @value = value
       @state = state
 
-      set_kw_end(last_lex)
+      set_kw_end(last_token)
     end
 
-    private def set_kw_end(last_lex)
+    private def set_kw_end(last_token)
       @is_end = false
       @is_kw = false
       return if type != :on_kw
 
-      return if last_lex && last_lex.fname? # https://github.com/ruby/ruby/commit/776759e300e4659bb7468e2b97c8c2d4359a2953
+      return if last_token && last_token.fname? # https://github.com/ruby/ruby/commit/776759e300e4659bb7468e2b97c8c2d4359a2953
 
-      case token
+      case value
       when "if", "unless", "while", "until"
         # Only count if/unless when it's not a "trailing" if/unless
         # https://github.com/ruby/ruby/blob/06b44f819eb7b5ede1ff69cecb25682b56a1d60c/lib/irb/ruby-lex.rb#L374-L375

spec/unit/code_block_spec.rb

@@ -33,7 +33,7 @@ def foo
       array = [block_2, block_1, block_0].sort
       expect(array.last).to eq(block_2)
 
-      block = CodeBlock.new(lines: CodeLine.new(line: " " * 8 + "foo", index: 4, lex: []))
+      block = CodeBlock.new(lines: CodeLine.new(line: " " * 8 + "foo", index: 4, tokens: []))
       array.prepend(block)
       expect(array.max).to eq(block)
     end

spec/unit/lex_all_spec.rb

@@ -17,10 +17,10 @@ module SyntaxSuggest
         end            # 9
       EOM
 
-      lex = LexAll.new(source: source)
-      expect(lex.map(&:token).to_s).to include("dog")
-      expect(lex.first.line).to eq(1)
-      expect(lex.last.line).to eq(9)
+      tokens = LexAll.new(source: source)
+      expect(tokens.map(&:value)).to include("dog")
+      expect(tokens.first.line).to eq(1)
+      expect(tokens.last.line).to eq(9)
     end
   end
 end