Don't create tokens by hand

There is some awkward code that dances around the fact
that the tokens for a method actually contain a 3 extra tokens
that don't exist in the source code.

Now `RipperStateLex` is only referenced to actually parse, rest is kept internal

Author: Earlopain

lib/rdoc/generator/markup.rb

@@ -86,57 +86,56 @@ class RDoc::CodeObject
 class RDoc::MethodAttr
 
   ##
-  # Prepend +src+ with line numbers.  Relies on the first line of a source
-  # code listing having:
-  #
-  #   # File xxxxx, line dddd
-  #
-  # If it has this comment then line numbers are added to +src+ and the <tt>,
-  # line dddd</tt> portion of the comment is removed.
+  # Prepend +src+ with line numbers.
 
-  def add_line_numbers(src)
-    return unless src.sub!(/\A(.*)(, line (\d+))/, '\1')
-    first = $3.to_i - 1
-    last  = first + src.count("\n")
-    size = last.to_s.length
+  def add_line_numbers(src, token_stream)
+    start_line = token_stream.first[:line_no]
+    end_line  = start_line + src.count("\n")
+    number_digits = end_line.to_s.length
 
-    line = first
+    line = start_line
     src.gsub!(/^/) do
-      res = if line == first then
-              " " * (size + 1)
-            else
-              "<span class=\"line-num\">%2$*1$d</span> " % [size, line]
-            end
+      res = "<span class=\"line-num\">#{line.to_s.rjust(number_digits)}</span> "
 
       line += 1
       res
     end
   end
 
+  ##
+  # Prepend +src+ with a comment that declares its location in the source.
+
+  def add_location_comment(src, token_stream)
+    if options.line_numbers
+      src.prepend("<span class=\"ruby-comment\"># File #{file.relative_name}</span>\n")
+    else
+      src.prepend("<span class=\"ruby-comment\"># File #{file.relative_name}, line #{token_stream.first.line_no}</span>\n")
+    end
+  end
+
   ##
   # Turns the method's token stream into HTML.
   #
   # Prepends line numbers if +options.line_numbers+ is true.
 
   def markup_code
-    return '' unless @token_stream
+    return '' if !@token_stream || @token_stream.empty?
 
     src = RDoc::TokenStream.to_html @token_stream
 
+    # add initial whitespace so that the ident gets calculated correctly
+    src.prepend(' ' * @token_stream.first[:char_no])
+
     # dedent the source
-    indent = src.length
-    lines = src.lines.to_a
-    lines.shift if src =~ /\A.*#\ *File/i # remove '# File' comment
-    lines.each do |line|
-      if line =~ /^ *(?=\S)/
-        n = $~.end(0)
-        indent = n if n < indent
-        break if n == 0
-      end
+    common_ident = src.length
+    src.scan(/^ *(?=\S)/) do |whitespace|
+      common_ident = whitespace.length if whitespace.length < common_ident
+      break if common_ident == 0
     end
-    src.gsub!(/^#{' ' * indent}/, '') if indent > 0
+    src.gsub!(/^#{' ' * common_ident}/, '') if common_ident > 0
 
-    add_line_numbers(src) if options.line_numbers
+    add_line_numbers(src, @token_stream) if options.line_numbers
+    add_location_comment(src, @token_stream)
 
     src
   end

lib/rdoc/parser/ruby.rb

@@ -314,7 +314,7 @@ def parse_comment_tomdoc(container, comment, line_no, start_line)
 
     meth.start_collecting_tokens(:ruby)
     node = @line_nodes[line_no]
-    tokens = node ? visible_tokens_from_location(node.location) : [file_line_comment_token(start_line)]
+    tokens = node ? visible_tokens_from_location(node.location) : []
     tokens.each { |token| meth.token_stream << token }
 
     container.add_method meth
@@ -385,7 +385,7 @@ def handle_meta_method_comment(comment, directives, node)
         tokens = visible_tokens_from_location(node.location)
         line_no = node.location.start_line
       else
-        tokens = [file_line_comment_token(line_no)]
+        tokens = []
       end
       internal_add_method(
         method_name,
@@ -498,23 +498,13 @@ def slice_tokens(start_pos, end_pos) # :nodoc:
     tokens
   end
 
-  def file_line_comment_token(line_no) # :nodoc:
-    position_comment = RDoc::Parser::RipperStateLex::Token.new(line_no - 1, 0, :on_comment)
-    position_comment[:text] = "# File #{@top_level.relative_name}, line #{line_no}"
-    position_comment
-  end
-
   # Returns tokens from the given location
 
   def visible_tokens_from_location(location)
-    position_comment = file_line_comment_token(location.start_line)
-    newline_token = RDoc::Parser::RipperStateLex::Token.new(0, 0, :on_nl, "\n")
-    indent_token = RDoc::Parser::RipperStateLex::Token.new(location.start_line, 0, :on_sp, ' ' * location.start_character_column)
-    tokens = slice_tokens(
+    slice_tokens(
       [location.start_line, location.start_character_column],
       [location.end_line, location.end_character_column]
     )
-    [position_comment, newline_token, indent_token, *tokens]
   end
 
   # Handles `public :foo, :bar` `private :foo, :bar` and `protected :foo, :bar`

test/rdoc/code_object/any_method_test.rb

@@ -149,42 +149,44 @@ def test_call_seq_returns_nil_if_alias_is_missing_from_call_seq
   end
 
   def test_markup_code
-    tokens = [
-      { :line_no => 0, :char_no => 0, :kind => :on_const, :text => 'CONSTANT' },
-    ]
-
+    tokens = RDoc::Parser::RipperStateLex.parse("A\nB")
     @c2_a.collect_tokens(:ruby)
     @c2_a.add_tokens(tokens)
 
-    expected = '<span class="ruby-constant">CONSTANT</span>'
-
-    assert_equal expected, @c2_a.markup_code
+    assert_equal <<~EXPECTED.chomp, @c2_a.markup_code
+      <span class="ruby-comment"># File xref_data.rb, line 1</span>
+      <span class="ruby-constant">A</span>
+      <span class="ruby-constant">B</span>
+    EXPECTED
   end
 
   def test_markup_code_with_line_numbers
-    position_comment = "# File #{@file_name}, line 1"
-    tokens = [
-      { :line_no => 1, :char_no => 0, :kind => :on_comment, :text => position_comment },
-      { :line_no => 1, :char_no => position_comment.size, :kind => :on_nl, :text => "\n" },
-      { :line_no => 2, :char_no => 0, :kind => :on_const, :text => 'A' },
-      { :line_no => 2, :char_no => 1, :kind => :on_nl, :text => "\n" },
-      { :line_no => 3, :char_no => 0, :kind => :on_const, :text => 'B' }
-    ]
-
+    tokens = RDoc::Parser::RipperStateLex.parse("A\nB")
     @c2_a.collect_tokens(:ruby)
     @c2_a.add_tokens(tokens)
 
-    assert_equal <<-EXPECTED.chomp, @c2_a.markup_code
-<span class="ruby-comment"># File xref_data.rb, line 1</span>
-<span class="ruby-constant">A</span>
-<span class="ruby-constant">B</span>
+    @c2_a.options.line_numbers = true
+    assert_equal <<~EXPECTED.chomp, @c2_a.markup_code
+      <span class="ruby-comment"># File xref_data.rb</span>
+      <span class="line-num">1</span> <span class="ruby-constant">A</span>
+      <span class="line-num">2</span> <span class="ruby-constant">B</span>
     EXPECTED
+  end
 
-    @c2_a.options.line_numbers = true
-    assert_equal <<-EXPECTED.chomp, @c2_a.markup_code
-  <span class="ruby-comment"># File xref_data.rb</span>
-<span class="line-num">1</span> <span class="ruby-constant">A</span>
-<span class="line-num">2</span> <span class="ruby-constant">B</span>
+  def test_markup_code_dedent
+    tokens = RDoc::Parser::RipperStateLex.parse(<<-RUBY.rstrip)
+      foo
+        bar
+          baz
+    RUBY
+    @c2_a.collect_tokens(:ruby)
+    @c2_a.add_tokens(tokens)
+
+    assert_equal <<~EXPECTED.chomp, @c2_a.markup_code
+      <span class="ruby-comment"># File xref_data.rb, line 1</span>
+      <span class="ruby-identifier">foo</span>
+        <span class=\"ruby-identifier\">bar</span>
+          <span class="ruby-identifier">baz</span>
     EXPECTED
   end