Reintroduce escaped trailing newlines in heredocs in Macro.to_string/2

Closes #15354.

Author: josevalim

lib/elixir/lib/code/normalizer.ex

@@ -68,7 +68,7 @@ defmodule Code.Normalizer do
 
   # Bit containers
   defp do_normalize({:<<>>, _, args} = quoted, state) when is_list(args) do
-    normalize_bitstring(quoted, state)
+    normalize_bitstring(quoted, state, false)
   end
 
   # Atoms with interpolations
@@ -89,13 +89,7 @@ defmodule Code.Normalizer do
         normalize_literal(:utf8, [], state)
       end
 
-    string =
-      if state.escape do
-        normalize_bitstring(string, state, true)
-      else
-        normalize_bitstring(string, state)
-      end
-
+    string = normalize_bitstring(string, state, state.escape)
     {{:., dot_meta, [:erlang, :binary_to_atom]}, call_meta, [string, utf8]}
   end
 
@@ -118,6 +112,7 @@ defmodule Code.Normalizer do
             end
         end)
 
+      parts = maybe_add_trailing_newline(call_meta, parts, state)
       {{:., dot_meta, [List, :to_charlist]}, call_meta, [parts]}
     else
       normalize_call(quoted, state)
@@ -405,7 +400,8 @@ defmodule Code.Normalizer do
   defp allow_keyword?(:{}, _), do: false
   defp allow_keyword?(op, arity), do: not is_atom(op) or not Macro.operator?(op, arity)
 
-  defp normalize_bitstring({:<<>>, meta, parts}, state, escape_interpolation \\ false) do
+  defp normalize_bitstring({:<<>>, meta, parts}, state, escape_interpolation) do
+    parts = maybe_add_trailing_newline(meta, parts, state)
     meta = patch_meta_line(meta, state.parent_meta)
 
     parts =
@@ -427,6 +423,17 @@ defmodule Code.Normalizer do
     {:<<>>, meta, parts}
   end
 
+  defp maybe_add_trailing_newline(meta, parts, state) do
+    with true <- state.escape and Keyword.get(meta, :delimiter) in ~w(""" '''),
+         last = List.last(parts),
+         true <- is_binary(last) and not String.ends_with?(last, "\n") do
+      [_last | rest] = Enum.reverse(parts)
+      Enum.reverse([last <> "\n" | rest])
+    else
+      _ -> parts
+    end
+  end
+
   defp normalize_interpolation_parts(parts, state, escape_interpolation) do
     Enum.map(parts, fn
       {:"::", interpolation_meta,

lib/elixir/test/elixir/code_normalizer/formatted_ast_test.exs

@@ -30,7 +30,7 @@ defmodule Code.Normalizer.FormatterASTTest do
 
     {quoted, comments} = Code.string_to_quoted_with_comments!(good, to_quoted_opts)
 
-    to_algebra_opts = [comments: comments, escape: false] ++ opts
+    to_algebra_opts = Keyword.merge([comments: comments, escape: false], opts)
 
     quoted
     |> Code.quoted_to_algebra(to_algebra_opts)
@@ -289,13 +289,53 @@ defmodule Code.Normalizer.FormatterASTTest do
     end
 
     test "with escaped new lines" do
-      assert_same ~S'''
-      """
-      one\
-      #{"two"}\
-      three\
-      """
-      '''
+      source =
+        String.trim(~S'''
+        """
+        one\
+        #{"two"}\
+        three\
+        """
+        ''')
+
+      assert_same source
+
+      # This is the same default as assert_same
+      assert string_to_string(source, unescape: false, escape: false) == source
+
+      # If we unescape, we cannot keep the original representation
+      # but we must keep it as a valid heredoc
+      assert string_to_string(source, unescape: true, escape: true) ==
+               String.trim(~S'''
+               """
+               one#{"two"}three
+               """
+               ''')
+    end
+
+    test "with empty escaped new lines" do
+      source =
+        String.trim(~S'''
+        """
+        one\
+        #{"two"}\
+        three\
+        """
+        ''')
+
+      assert_same source
+
+      # This is the same default as assert_same
+      assert string_to_string(source, unescape: false, escape: false) == source
+
+      # If we unescape, we cannot keep the original representation
+      # but we must keep it as a valid heredoc
+      assert string_to_string(source, unescape: true, escape: true) ==
+               String.trim(~S'''
+               """
+               one#{"two"}three
+               """
+               ''')
     end
   end
 
@@ -341,6 +381,31 @@ defmodule Code.Normalizer.FormatterASTTest do
       '''
       """
     end
+
+    test "with empty escaped new lines" do
+      source =
+        String.trim(~S"""
+        '''
+        one\
+        #{"two"}\
+        three\
+        '''
+        """)
+
+      assert_same source
+
+      # This is the same default as assert_same
+      assert string_to_string(source, unescape: false, escape: false) == source
+
+      # If we unescape, we cannot keep the original representation
+      # but we must keep it as a valid heredoc
+      assert string_to_string(source, unescape: true, escape: true) ==
+               String.trim(~S"""
+               '''
+               one#{"two"}three
+               '''
+               """)
+    end
   end
 
   describe "keyword list" do
@@ -467,6 +532,23 @@ defmodule Code.Normalizer.FormatterASTTest do
       '''rsa
       """
     end
+
+    test "with empty escaped new lines" do
+      source =
+        String.trim(~S"""
+        ~c'''
+        one\
+        #{"two"}\
+        three\
+        '''
+        """)
+
+      assert_same source
+
+      # Heredoc in sigils always preserves newlines regardless of escape/unescape
+      assert string_to_string(source, unescape: false, escape: false) == source
+      assert string_to_string(source, unescape: true, escape: true) == source
+    end
   end
 
   describe "preserves comments formatting" do