Enable break_on_newline extension by default for Markdown

Enable the existing break_on_newline extension in DEFAULT_EXTENSIONS so
the Markdown parser converts soft line breaks to HardBreak objects. This
produces visible <br> line breaks in HTML output, matching GFM rendering.

The conversion happens in the Markdown parser's paragraph() method,
which is the proper place for Markdown-specific behavior. The generic
accept_paragraph in ToHtml is unchanged and has no Markdown-specific
logic.

Also remove the CJK-aware newline-to-space gsub from accept_paragraph
that is no longer needed. The RDoc markup parser already handles newline
joining at parse time in build_paragraph.

Author: st0012

lib/rdoc/markup/to_html.rb

@@ -313,9 +313,6 @@ def accept_block_quote(block_quote)
   def accept_paragraph(paragraph)
     @res << "\n<p>"
     text = paragraph.text @hard_break
-    text = text.gsub(/(#{SPACE_SEPARATED_LETTER_CLASS})?\K\r?\n(?=(?(1)(#{SPACE_SEPARATED_LETTER_CLASS})?))/o) {
-      defined?($2) && ' '
-    }
     @res << to_html(text)
     @res << "</p>\n"
   end

test/rdoc/markup/to_html_test.rb

@@ -236,7 +236,7 @@ def accept_paragraph_br
   end
 
   def accept_paragraph_break
-    assert_equal "\n<p>hello<br> world</p>\n", @to.res.join
+    assert_equal "\n<p>hello<br>\nworld</p>\n", @to.res.join
   end
 
   def accept_paragraph_i
@@ -411,43 +411,13 @@ def test_accept_heading_dedup_resets_on_start_accepting
   end
 
   def test_accept_paragraph_newline
-    hellos = ["hello", "\u{393 3b5 3b9 3ac} \u{3c3 3bf 3c5}"]
-    worlds = ["world", "\u{3ba 3cc 3c3 3bc 3bf 3c2}"]
-    ohayo, sekai = %W"\u{304a 306f 3088 3046} \u{4e16 754c}"
-
-    hellos.product(worlds) do |hello, world|
-      @to.start_accepting
-      @to.accept_paragraph para("#{hello}\n", "#{world}\n")
-      assert_equal "\n<p>#{hello} #{world}</p>\n", @to.res.join
-    end
-
-    hellos.each do |hello|
-      @to.start_accepting
-      @to.accept_paragraph para("#{hello}\n", "#{sekai}\n")
-      assert_equal "\n<p>#{hello}#{sekai}</p>\n", @to.res.join
-    end
-
-    worlds.each do |world|
-      @to.start_accepting
-      @to.accept_paragraph para("#{ohayo}\n", "#{world}\n")
-      assert_equal "\n<p>#{ohayo}#{world}</p>\n", @to.res.join
-    end
-
     @to.start_accepting
-    @to.accept_paragraph para("#{ohayo}\n", "#{sekai}\n")
-    assert_equal "\n<p>#{ohayo}#{sekai}</p>\n", @to.res.join
+    @to.accept_paragraph para("hello\n", "world\n")
+    assert_equal "\n<p>hello\nworld\n</p>\n", @to.res.join
 
     @to.start_accepting
     @to.accept_paragraph para("+hello+\n", "world\n")
-    assert_equal "\n<p><code>hello</code> world</p>\n", @to.res.join
-
-    @to.start_accepting
-    @to.accept_paragraph para("hello\n", "+world+\n")
-    assert_equal "\n<p>hello <code>world</code></p>\n", @to.res.join
-
-    @to.start_accepting
-    @to.accept_paragraph para("+hello+\n", "+world+\n")
-    assert_equal "\n<p><code>hello</code> <code>world</code></p>\n", @to.res.join
+    assert_equal "\n<p><code>hello</code>\nworld\n</p>\n", @to.res.join
   end
 
   def test_accept_heading_output_decoration

test/rdoc/parser/changelog_test.rb

@@ -421,8 +421,8 @@ def test_scan_git
                 'Masataka Pocke Kuwabara', 'kuwabara@pocke.me', '2021-01-01 14:25:08 +0900',
                 [head(4, 'Make args info for RubyVM::AST to available on endless method without parens'),
                  head(5, 'Problem'),
-                 para("Arguments information is missing for endless method without parens.\n" +
-                      "For example:"),
+                 para("Arguments information is missing for endless method without parens.",
+                      hard_break, "For example:"),
                  verb("# ok\n").tap {|v| v.format = :ruby},
                  para('It causes an error if a program expects <code>args</code> node exists.'),
                  head(5, 'Solution'),

test/rdoc/rdoc_markdown_test.rb

@@ -56,7 +56,7 @@ def test_parse_block_quote
     expected =
       doc(
         block(
-          para("this is\na block quote")))
+          para("this is", hard_break, "a block quote")))
 
     assert_equal expected, doc
   end
@@ -70,11 +70,22 @@ def test_parse_block_quote_continue
     expected =
       doc(
         block(
-          para("this is\na block quote")))
+          para("this is", hard_break, "a block quote")))
 
     assert_equal expected, doc
   end
 
+  def test_parse_block_quote_continue_html
+    doc = parse <<-BLOCK_QUOTE
+> this is
+a block quote
+    BLOCK_QUOTE
+
+    html = doc.accept(RDoc::Markup::ToHtml.new)
+
+    assert_include html, "<p>this is<br>\na block quote</p>"
+  end
+
   def test_parse_block_quote_list
     doc = parse <<-BLOCK_QUOTE
 > text
@@ -104,7 +115,7 @@ def test_parse_block_quote_newline
     expected =
       doc(
         block(
-          para("this is\na block quote")))
+          para("this is", hard_break, "a block quote")))
 
     assert_equal expected, doc
   end
@@ -120,12 +131,77 @@ def test_parse_block_quote_separate
     expected =
       doc(
         block(
-          para("this is\na block quote"),
+          para("this is", hard_break, "a block quote")),
+        block(
           para("that continues")))
 
     assert_equal expected, doc
   end
 
+  def test_parse_block_quote_no_lazy_continuation_for_list
+    doc = parse <<-BLOCK_QUOTE
+> foo
+- bar
+    BLOCK_QUOTE
+
+    expected =
+      doc(
+        block(
+          para("foo")),
+        list(:BULLET,
+          item(nil, para("bar"))))
+
+    assert_equal expected, doc
+  end
+
+  def test_parse_block_quote_no_lazy_continuation_for_ordered_list
+    doc = parse <<-BLOCK_QUOTE
+> foo
+1. bar
+    BLOCK_QUOTE
+
+    expected =
+      doc(
+        block(
+          para("foo")),
+        list(:NUMBER,
+          item(nil, para("bar"))))
+
+    assert_equal expected, doc
+  end
+
+  def test_parse_block_quote_no_lazy_continuation_for_heading
+    doc = parse <<-BLOCK_QUOTE
+> foo
+# bar
+    BLOCK_QUOTE
+
+    expected =
+      doc(
+        block(
+          para("foo")),
+        head(1, "bar"))
+
+    assert_equal expected, doc
+  end
+
+  def test_parse_block_quote_no_lazy_continuation_for_code_fence
+    doc = parse <<~BLOCK_QUOTE
+      > foo
+      ```
+      code
+      ```
+    BLOCK_QUOTE
+
+    expected =
+      doc(
+        block(
+          para("foo")),
+        verb("code\n"))
+
+    assert_equal expected, doc
+  end
+
   def test_parse_char_entity
     doc = parse '&pi; &nn;'
 
@@ -262,7 +338,7 @@ def test_parse_code_github
     assert_equal expected, parse(doc)
 
     expected =
-      doc(para("Example:\n<code>\n""code goes here\n</code>"))
+      doc(para("Example:", hard_break, "<code>\n""code goes here\n</code>"))
 
     assert_equal expected, parse(doc.sub(/^\n/, ''))
   end
@@ -296,7 +372,7 @@ def test_parse_code_github_format
     assert_equal expected, parse(doc)
 
     expected =
-      doc(para("Example:\n<code>ruby\n""code goes here\n</code>"))
+      doc(para("Example:", hard_break, "<code>ruby\n""code goes here\n</code>"))
 
     assert_equal expected, parse(doc.sub(/^\n/, ''))
   end
@@ -343,7 +419,7 @@ def test_parse_definition_list_indents
         item(%w[one],   para("Indented one characters")),
         item(%w[two],   para("Indented two characters")),
         item(%w[three], para("Indented three characters"))),
-      para("four\n : Indented four characters"))
+      para("four", hard_break, " : Indented four characters"))
 
     assert_equal expected, doc
   end
@@ -392,9 +468,9 @@ def test_parse_definition_list_multi_line
     expected = doc(
       list(:NOTE,
         item(%w[one],
-          para("This is a definition\nthat extends to two lines")),
+          para("This is a definition", hard_break, "that extends to two lines")),
         item(%w[two],
-          para("This is another definition\nthat also extends to two lines"))))
+          para("This is another definition", hard_break, "that also extends to two lines"))))
 
     assert_equal expected, doc
   end
@@ -430,8 +506,8 @@ def test_parse_definition_list_no
     MD
 
     expected = doc(
-        para("one\n: This is a definition"),
-        para("two\n: This is another definition"))
+        para("one", hard_break, ": This is a definition"),
+        para("two", hard_break, ": This is another definition"))
 
     assert_equal expected, doc
   end
@@ -779,7 +855,7 @@ def test_parse_list_bullet_multiline
 
     expected = doc(
       list(:BULLET,
-        item(nil, para("one\n two"))))
+        item(nil, para("one", hard_break, " two"))))
 
     assert_equal expected, doc
   end
@@ -832,7 +908,7 @@ def test_parse_list_bullet_nest_continue
           para("outer"),
           list(:BULLET,
             item(nil,
-              para("inner\n continue inner")))),
+              para("inner", hard_break, " continue inner")))),
         item(nil,
           para("outer 2"))))
 
@@ -899,7 +975,7 @@ def test_parse_note_indent
     expected = doc(
       para("Some text.{*1}[rdoc-label:foottext-1:footmark-1]"),
       rule(1),
-      para("{^1}[rdoc-label:footmark-1:foottext-1] With a footnote\n\nmore"))
+      para("{^1}[rdoc-label:footmark-1:foottext-1] With a footnote", hard_break, "more"))
 
     assert_equal expected, doc
   end
@@ -940,8 +1016,10 @@ def test_parse_note_multiple
     MD
 
     expected = doc(
-      para("Some text{*1}[rdoc-label:foottext-1:footmark-1]\n" +
-           "with inline notes{*2}[rdoc-label:foottext-2:footmark-2]\n" +
+      para("Some text{*1}[rdoc-label:foottext-1:footmark-1]",
+           hard_break,
+           "with inline notes{*2}[rdoc-label:foottext-2:footmark-2]",
+           hard_break,
            "and an extra note.{*3}[rdoc-label:foottext-3:footmark-3]"),
 
       rule(1),
@@ -1040,7 +1118,7 @@ def test_parse_paragraph_indent_three
   def test_parse_paragraph_multiline
     doc = parse "one\ntwo"
 
-    expected = doc(para("one\ntwo"))
+    expected = doc(para("one", hard_break, "two"))
 
     assert_equal expected, doc
   end