Don't create tokens by hand (#1696)

There is some awkward code that dances around the fact that the tokens
for a method actually contain a 3 extra tokens that don't exist in the
source code.

Now `RipperStateLex` is only referenced to actually parse, rest is kept
internal

Author: Earlopain

lib/rdoc/generator/markup.rb

@@ -86,57 +86,60 @@ class RDoc::CodeObject
 class RDoc::MethodAttr
 
   ##
-  # Prepend +src+ with line numbers.  Relies on the first line of a source
-  # code listing having:
-  #
-  #   # File xxxxx, line dddd
-  #
-  # If it has this comment then line numbers are added to +src+ and the <tt>,
-  # line dddd</tt> portion of the comment is removed.
+  # Prepend +src+ with line numbers.
 
   def add_line_numbers(src)
-    return unless src.sub!(/\A(.*)(, line (\d+))/, '\1')
-    first = $3.to_i - 1
-    last  = first + src.count("\n")
-    size = last.to_s.length
+    return if src.empty? || !line
+    start_line = line
+    end_line  = start_line + src.count("\n")
+    number_digits = end_line.to_s.length
 
-    line = first
+    current_line = start_line
     src.gsub!(/^/) do
-      res = if line == first then
-              " " * (size + 1)
-            else
-              "<span class=\"line-num\">%2$*1$d</span> " % [size, line]
-            end
+      res = "<span class=\"line-num\">#{current_line.to_s.rjust(number_digits)}</span> "
 
-      line += 1
+      current_line += 1
       res
     end
   end
 
+  ##
+  # Prepend +src+ with a comment that declares its location in the source.
+
+  def add_location_comment(src)
+    path = CGI.escapeHTML(file.relative_name)
+    if options.line_numbers && !src.empty?
+      src.prepend("<span class=\"ruby-comment\"># File #{path}</span>\n")
+    else
+      src.prepend("<span class=\"ruby-comment\"># File #{path}, line #{line}</span>\n")
+    end
+  end
+
   ##
   # Turns the method's token stream into HTML.
   #
   # Prepends line numbers if +options.line_numbers+ is true.
 
   def markup_code
-    return '' unless @token_stream
+    return '' if !@token_stream
 
     src = RDoc::TokenStream.to_html @token_stream
 
+    # add initial whitespace so that the indent gets calculated correctly
+    src.prepend(' ' * @token_stream.first[:char_no]) if source_language == 'ruby' && @token_stream.first
+
     # dedent the source
-    indent = src.length
-    lines = src.lines.to_a
-    lines.shift if src =~ /\A.*#\ *File/i # remove '# File' comment
-    lines.each do |line|
-      if line =~ /^ *(?=\S)/
-        n = $~.end(0)
-        indent = n if n < indent
-        break if n == 0
-      end
+    common_indent = src.length
+    src.scan(/^ *(?=\S)/) do |whitespace|
+      common_indent = whitespace.length if whitespace.length < common_indent
+      break if common_indent == 0
     end
-    src.gsub!(/^#{' ' * indent}/, '') if indent > 0
+    src.gsub!(/^#{' ' * common_indent}/, '') if common_indent > 0
 
-    add_line_numbers(src) if options.line_numbers
+    if source_language == 'ruby'
+      add_line_numbers(src) if options.line_numbers
+      add_location_comment(src)
+    end
 
     src
   end

lib/rdoc/parser/ruby.rb

@@ -314,7 +314,7 @@ def parse_comment_tomdoc(container, comment, line_no, start_line)
 
     meth.start_collecting_tokens(:ruby)
     node = @line_nodes[line_no]
-    tokens = node ? visible_tokens_from_location(node.location) : [file_line_comment_token(start_line)]
+    tokens = node ? visible_tokens_from_location(node.location) : []
     tokens.each { |token| meth.token_stream << token }
 
     container.add_method meth
@@ -385,7 +385,7 @@ def handle_meta_method_comment(comment, directives, node)
         tokens = visible_tokens_from_location(node.location)
         line_no = node.location.start_line
       else
-        tokens = [file_line_comment_token(line_no)]
+        tokens = []
       end
       internal_add_method(
         method_name,
@@ -498,23 +498,13 @@ def slice_tokens(start_pos, end_pos) # :nodoc:
     tokens
   end
 
-  def file_line_comment_token(line_no) # :nodoc:
-    position_comment = RDoc::Parser::RipperStateLex::Token.new(line_no - 1, 0, :on_comment)
-    position_comment[:text] = "# File #{@top_level.relative_name}, line #{line_no}"
-    position_comment
-  end
-
   # Returns tokens from the given location
 
   def visible_tokens_from_location(location)
-    position_comment = file_line_comment_token(location.start_line)
-    newline_token = RDoc::Parser::RipperStateLex::Token.new(0, 0, :on_nl, "\n")
-    indent_token = RDoc::Parser::RipperStateLex::Token.new(location.start_line, 0, :on_sp, ' ' * location.start_character_column)
-    tokens = slice_tokens(
+    slice_tokens(
       [location.start_line, location.start_character_column],
       [location.end_line, location.end_character_column]
     )
-    [position_comment, newline_token, indent_token, *tokens]
   end
 
   # Handles `public :foo, :bar` `private :foo, :bar` and `protected :foo, :bar`

test/rdoc/code_object/any_method_test.rb

@@ -148,51 +148,11 @@ def test_call_seq_returns_nil_if_alias_is_missing_from_call_seq
     assert_nil(alias_to_method.call_seq)
   end
 
-  def test_markup_code
-    tokens = [
-      { :line_no => 0, :char_no => 0, :kind => :on_const, :text => 'CONSTANT' },
-    ]
-
-    @c2_a.collect_tokens(:ruby)
-    @c2_a.add_tokens(tokens)
-
-    expected = '<span class="ruby-constant">CONSTANT</span>'
-
-    assert_equal expected, @c2_a.markup_code
-  end
-
-  def test_markup_code_with_line_numbers
-    position_comment = "# File #{@file_name}, line 1"
-    tokens = [
-      { :line_no => 1, :char_no => 0, :kind => :on_comment, :text => position_comment },
-      { :line_no => 1, :char_no => position_comment.size, :kind => :on_nl, :text => "\n" },
-      { :line_no => 2, :char_no => 0, :kind => :on_const, :text => 'A' },
-      { :line_no => 2, :char_no => 1, :kind => :on_nl, :text => "\n" },
-      { :line_no => 3, :char_no => 0, :kind => :on_const, :text => 'B' }
-    ]
-
-    @c2_a.collect_tokens(:ruby)
-    @c2_a.add_tokens(tokens)
-
-    assert_equal <<-EXPECTED.chomp, @c2_a.markup_code
-<span class="ruby-comment"># File xref_data.rb, line 1</span>
-<span class="ruby-constant">A</span>
-<span class="ruby-constant">B</span>
-    EXPECTED
-
-    @c2_a.options.line_numbers = true
-    assert_equal <<-EXPECTED.chomp, @c2_a.markup_code
-  <span class="ruby-comment"># File xref_data.rb</span>
-<span class="line-num">1</span> <span class="ruby-constant">A</span>
-<span class="line-num">2</span> <span class="ruby-constant">B</span>
-    EXPECTED
-  end
-
   def test_markup_code_empty
     assert_equal '', @c2_a.markup_code
   end
 
-  def test_markup_code_with_variable_expansion
+  def test_param_seq_with_variable_expansion
     m = RDoc::AnyMethod.new nil, 'method'
     m.parent = @c1
     m.block_params = '"Hello, #{world}", yield_arg'

test/rdoc/parser/c_test.rb

@@ -2215,6 +2215,33 @@ def test_markup_format_override
     assert_equal("markdown", klass.attributes.find {|a| a.name == "default_format"}.comment.format)
   end
 
+  def test_markup_code
+    # Should not generate line numbers
+    @top_level.store.options.line_numbers = true
+    parser = util_parser <<~C
+      static VALUE
+      rb_hash_has_value(VALUE hash, VALUE val) {
+        return Qtrue;
+      }
+
+      Init_Hash(void)
+      {
+        rb_define_method(rb_cHash, "value?", rb_hash_has_value, 1);
+      }
+    C
+    parser.scan
+
+    hash = @store.classes_hash['Hash']
+    value_method = hash.method_list.find { |m| m.name == 'value?' }
+
+    assert_equal(<<~EXPECTED.chomp, value_method.markup_code)
+      static VALUE
+      rb_hash_has_value(VALUE hash, VALUE val) {
+        return Qtrue;
+      }
+    EXPECTED
+  end
+
   def test_clear_file_contributions_removes_c_methods
     content = <<~C
       /* Document-class: Foo */

test/rdoc/parser/ruby_test.rb

@@ -2529,6 +2529,105 @@ def m2; end
     assert_equal "ARGF.readlines(a)\nARGF.readlines(b)\nARGF.readlines(c)\nARGF.readlines(d)", m2.call_seq.chomp
   end
 
+  def test_markup_code
+    util_parser <<~RUBY
+      class Foo
+        def bar
+        end
+      end
+    RUBY
+
+    m1, = @top_level.classes.first.method_list
+
+    assert_equal <<~EXPECTED.chomp, m1.markup_code
+      <span class="ruby-comment"># File #{@filename}, line 2</span>
+      <span class="ruby-keyword">def</span> <span class="ruby-identifier">bar</span>
+      <span class="ruby-keyword">end</span>
+    EXPECTED
+  end
+
+  def test_markup_code_with_line_numbers
+    @top_level.store.options.line_numbers = true
+    util_parser <<~RUBY
+      class Foo
+        def bar
+        end
+      end
+    RUBY
+
+    m1, = @top_level.classes.first.method_list
+
+    assert_equal <<~EXPECTED.chomp, m1.markup_code
+      <span class="ruby-comment"># File #{@filename}</span>
+      <span class="line-num">2</span> <span class="ruby-keyword">def</span> <span class="ruby-identifier">bar</span>
+      <span class="line-num">3</span> <span class="ruby-keyword">end</span>
+    EXPECTED
+  end
+
+  def test_markup_code_dedent
+    util_parser <<~RUBY
+      class Foo
+        def bar
+        end
+
+        private
+          def baz
+          end
+      end
+    RUBY
+    m1, m2 = @top_level.classes.first.method_list
+
+    assert_equal(<<~EXPECTED.chomp, m1.markup_code)
+      <span class="ruby-comment"># File #{@filename}, line 2</span>
+      <span class="ruby-keyword">def</span> <span class="ruby-identifier">bar</span>
+      <span class="ruby-keyword">end</span>
+    EXPECTED
+    assert_equal(<<~EXPECTED.chomp, m2.markup_code)
+      <span class="ruby-comment"># File #{@filename}, line 6</span>
+      <span class="ruby-keyword">def</span> <span class="ruby-identifier">baz</span>
+      <span class="ruby-keyword">end</span>
+    EXPECTED
+  end
+
+  def test_markup_code_empty
+    util_parser <<~RUBY
+      class Foo
+        ##
+        # :method: ghost_method
+
+        ##
+        # :method:
+        # :call-seq: ghost_method2() -> Integer
+      end
+    RUBY
+
+    m1, m2 = @top_level.classes.first.method_list
+    assert_equal(
+      "<span class=\"ruby-comment\"># File #{@filename}, line 3</span>",
+      m1.markup_code.chomp
+    )
+    assert_equal(
+      "<span class=\"ruby-comment\"># File #{@filename}, line 6</span>",
+      m2.markup_code.chomp
+    )
+  end
+
+  def test_markup_code_empty_line_number
+    @top_level.store.options.line_numbers = true
+    util_parser <<~RUBY
+      class Foo
+        ##
+        # :method: ghost_method
+      end
+    RUBY
+
+    m, = @top_level.classes.first.method_list
+    assert_equal(
+      "<span class=\"ruby-comment\"># File #{@filename}, line 3</span>",
+      m.markup_code.chomp
+    )
+  end
+
   def util_parser(content)
     @parser = RDoc::Parser::Ruby.new @top_level, content, @options, @stats
     @parser.scan