#{CGI.escapeHTML text}\ndef foo [ - '\\', - '\'', - "'", - "\'\"\`", - "\#", - "\#{}", - "#", - "#{}", + '\\', + '\'', + "'", + "\'\"\`", + "\#", + "\#{1}", + "#", + "#{1}", /'"/, /\'\"/, /\//, /\\/, /\#/, - /\#{}/, + /\#{1}/, /#/, - /#{}/ + /#{1}/ ] end def bar @@ -603,9 +603,9 @@ def foo `\\`, `\'\"\``, `\#`, - `\#{}`, + `\#{1}`, `#`, - `#{}` + `#{1}` ] end def bar @@ -620,12 +620,12 @@ def bardef foo [ - `\\`, - `\'\"\``, - `\#`, - `\#{}`, - `#`, - `#{}` + `\\`, + `\'\"\``, + `\#`, + `\#{1}`, + `#`, + `#{1}` ] end def bar diff --git a/test/rdoc/parser/ruby_colorizer_test.rb b/test/rdoc/parser/ruby_colorizer_test.rb new file mode 100644 index 0000000000..b3fd1b71de --- /dev/null +++ b/test/rdoc/parser/ruby_colorizer_test.rb @@ -0,0 +1,202 @@ +# frozen_string_literal: true +require_relative '../helper' +require 'rdoc/parser/ruby_colorizer' + +class RDocParserRubyColorizerTest < RDoc::TestCase + def token(kind, text) + RDoc::Parser::RubyColorizer::ColoredToken.new(kind, text) + end + + def test_partial_colorize + code = <<~RUBY + class A + def m + # comment + 42 + end + end + RUBY + parse_result = Prism.parse_lex(code) + program_node, unordered_tokens = parse_result.value + prism_tokens = unordered_tokens.map(&:first).sort_by! { |token| token.location.start_offset } + def_node = program_node.statements.body[0].body.body[0] + tokens = RDoc::Parser::RubyColorizer.partial_colorize(code, def_node, prism_tokens) + expected = [' ', 'def', ' ', 'm', "\n", ' ', "# comment\n", ' ', '42', "\n", ' ', 'end'] + assert_equal(expected, tokens.map(&:text)) + end + + def test_comment + code = <<~RUBY + # comment1 + class A + =begin + comment2 + =end + def m + 42 # comment3 + end + end + RUBY + tokens = RDoc::Parser::RubyColorizer.colorize(code) + assert_equal(code, tokens.map(&:text).join) + assert_include(tokens, token(:comment, "# comment1\n")) + assert_include(tokens, token(:comment, "=begin\n")) + assert_include(tokens, token(:comment, "comment2\n")) + assert_include(tokens, token(:comment, "=end\n")) + assert_include(tokens, token(:comment, "# comment3\n")) + end + + def test_interpolated_node + code = <<~'RUBY' + def m + "string#{interpolation1}example#@embvar" + /regexp#{interpolation2}example#$embvar/ + `xstring#{interpolation3}example#@embvar` + :"symbol#{interpolation4}example#$embvar" + end + RUBY + tokens = RDoc::Parser::RubyColorizer.colorize(code) + assert_equal(code, tokens.map(&:text).join) + + assert_include(tokens, token(:string, '"')) + assert_include(tokens, token(:string, 'string')) + assert_include(tokens, token(:string, '#{')) + assert_include(tokens, token(:identifier, 'interpolation1')) + assert_include(tokens, token(:string, '}')) + assert_include(tokens, token(:string, 'example')) + assert_include(tokens, token(:string, '#')) + + assert_include(tokens, token(:regexp, '/')) + assert_include(tokens, token(:regexp, 'regexp')) + assert_include(tokens, token(:regexp, '#{')) + assert_include(tokens, token(:identifier, 'interpolation2')) + assert_include(tokens, token(:regexp, '}')) + assert_include(tokens, token(:regexp, 'example')) + assert_include(tokens, token(:regexp, '#')) + + assert_include(tokens, token(:x_string, '`')) + assert_include(tokens, token(:x_string, 'xstring')) + assert_include(tokens, token(:x_string, '#{')) + assert_include(tokens, token(:identifier, 'interpolation3')) + assert_include(tokens, token(:x_string, '}')) + assert_include(tokens, token(:x_string, 'example')) + assert_include(tokens, token(:x_string, '#')) + + assert_include(tokens, token(:symbol, ':"')) + assert_include(tokens, token(:symbol, 'symbol')) + assert_include(tokens, token(:symbol, '#{')) + assert_include(tokens, token(:identifier, 'interpolation4')) + assert_include(tokens, token(:symbol, '}')) + assert_include(tokens, token(:symbol, 'example')) + assert_include(tokens, token(:symbol, '#')) + assert_include(tokens, token(:symbol, '"')) + end + + def test_percent_literal_arrays + code = <<~'RUBY' + def m + %w[1 2 3] + %W[one #{two} three] + %i[4 5 6] + %I[four #{five} six] + end + RUBY + tokens = RDoc::Parser::RubyColorizer.colorize(code) + assert_equal(code, tokens.map(&:text).join) + assert_include(tokens, token(:string, '%w[')) + assert_include(tokens, token(:string, '%W[')) + assert_include(tokens, token(:string, ']')) + assert_include(tokens, token(:string, '1')) + assert_include(tokens, token(:string, 'one')) + assert_include(tokens, token(:string, '#{')) + assert_include(tokens, token(:identifier, 'two')) + assert_include(tokens, token(:string, '}')) + assert_include(tokens, token(:symbol, '%i[')) + assert_include(tokens, token(:symbol, '%I[')) + assert_include(tokens, token(:symbol, ']')) + assert_include(tokens, token(:symbol, '4')) + assert_include(tokens, token(:symbol, 'four')) + assert_include(tokens, token(:symbol, '#{')) + assert_include(tokens, token(:identifier, 'five')) + assert_include(tokens, token(:symbol, '}')) + end + + def test_multibyte + code = <<~RUBY + def f(s = '💎') + # comment 💎 + puts '💎' + s + end + RUBY + tokens = RDoc::Parser::RubyColorizer.colorize(code) + assert_equal(code, tokens.map(&:text).join) + end + + def test_string + code = <<~'RUBY' + # string without closing + ?S + # interpolated string node may not have opening/closing + # parts may have opening/closing + %[s3] 's4'\ + "s5#{[?s]}s6" + RUBY + tokens = RDoc::Parser::RubyColorizer.colorize(code) + assert_equal(code, tokens.map(&:text).join) + string_token_texts = tokens.select { |t| t[:kind] == :string }.map(&:text) + expected_string_token_texts = %w[? S %[ s3 ] ' s4 ' " s5 #{ ? s } s6 "] + assert_equal(expected_string_token_texts, string_token_texts) + end + + def test_symbol + code = <<~'RUBY' + # symbol without closing + :sym1 + # symbol with opening/closing + :"sym2" + %s[sym3] + # opening and content has gap + <<~A; :\ + A + sym4 + RUBY + tokens = RDoc::Parser::RubyColorizer.colorize(code) + assert_equal(code, tokens.map(&:text).join) + symbol_token_texts = tokens.select { |t| t[:kind] == :symbol }.map(&:text) + expected_symbol_token_texts = %w[: sym1 :" sym2 " %s[ sym3 ] : sym4] + assert_equal(expected_symbol_token_texts, symbol_token_texts) + end + + def test_heredoc + code = <<~'RUBY' + def f + str1 = <<~AA + single-line-heredoc + AA + str2 = <<~`BB` # comment + x-string-heredoc + BB + str3 = <<~CC.itself + multi-line + #{embed} + heredoc + CC + end + RUBY + tokens = RDoc::Parser::RubyColorizer.colorize(code) + assert_equal(code, tokens.map(&:text).join) + assert_include(tokens, token(:string, '<<~AA')) + assert_include(tokens, token(:x_string, '<<~`BB`')) + assert_include(tokens, token(:string, '<<~CC')) + assert_include(tokens, token(:string, " AA\n")) + assert_include(tokens, token(:x_string, " BB\n")) + assert_include(tokens, token(:string, " CC\n")) + assert_include(tokens, token(:string, " single-line-heredoc\n")) + assert_include(tokens, token(:x_string, " x-string-heredoc\n")) + assert_include(tokens, token(:string, " multi-line\n")) + assert_include(tokens, token(:string, '#{')) + assert_include(tokens, token(:identifier, 'embed')) + assert_include(tokens, token(:string, '}')) + assert_include(tokens, token(:string, " heredoc\n")) + end +end diff --git a/test/rdoc/parser/ruby_test.rb b/test/rdoc/parser/ruby_test.rb index b3cef3a341..a5c79071ba 100644 --- a/test/rdoc/parser/ruby_test.rb +++ b/test/rdoc/parser/ruby_test.rb @@ -2406,6 +2406,23 @@ def test_read_directive_linear_performance end end + def test_code_object_token_stream + util_parser <<~RUBY + class Foo + def foo + 42 + end + + private def bar + baz + end + end + RUBY + + foo, bar = @top_level.classes.first.method_list + assert_equal([' ', 'def', ' ', 'foo', "\n", ' ', '42', "\n", ' ', 'end'], foo.token_stream.map(&:text)) + assert_equal([' ', 'def', ' ', 'bar', "\n", ' ', 'baz', "\n", ' ', 'end'], bar.token_stream.map(&:text)) + end def test_markup_first_comment util_parser <<~RUBY diff --git a/test/rdoc/rdoc_token_stream_test.rb b/test/rdoc/rdoc_token_stream_test.rb index ed5e124cc6..254811c72a 100644 --- a/test/rdoc/rdoc_token_stream_test.rb +++ b/test/rdoc/rdoc_token_stream_test.rb @@ -5,17 +5,18 @@ class RDocTokenStreamTest < RDoc::TestCase def test_class_to_html tokens = [ - { :line_no => 0, :char_no => 0, :kind => :on_const, :text => 'CONSTANT' }, - { :line_no => 0, :char_no => 0, :kind => :on_kw, :text => 'KW' }, - { :line_no => 0, :char_no => 0, :kind => :on_ivar, :text => 'IVAR' }, - { :line_no => 0, :char_no => 0, :kind => :on_op, :text => 'Op' }, - { :line_no => 0, :char_no => 0, :kind => :on_ident, :text => 'Id' }, - { :line_no => 0, :char_no => 0, :kind => :on_backref, :text => 'Node' }, - { :line_no => 0, :char_no => 0, :kind => :on_comment, :text => 'COMMENT' }, - { :line_no => 0, :char_no => 0, :kind => :on_regexp, :text => 'REGEXP' }, - { :line_no => 0, :char_no => 0, :kind => :on_tstring, :text => 'STRING' }, - { :line_no => 0, :char_no => 0, :kind => :on_int, :text => 'Val' }, - { :line_no => 0, :char_no => 0, :kind => :on_unknown, :text => '\\' } + { kind: :constant, text: 'CONSTANT' }, + { kind: :keyword, text: 'KW' }, + { kind: :ivar, text: 'IVAR' }, + { kind: :operator, text: 'Op' }, + { kind: :identifier, text: 'Id' }, + { kind: :symbol, text: 'Symbol' }, + { kind: :x_string, text: 'XString' }, + { kind: :comment, text: 'COMMENT' }, + { kind: :regexp, text: 'REGEXP' }, + { kind: :string, text: 'STRING' }, + { kind: :value, text: 'Val' }, + { kind: :plain, text: '\\' } ] expected = [ @@ -24,7 +25,8 @@ def test_class_to_html 'IVAR', 'Op', 'Id', - 'Node', + 'Symbol', + 'XString', 'COMMENT', 'REGEXP', 'STRING',