Handle escapes in docstrings more correctly

HISTORY_v2.md

@@ -1,3 +1,7 @@
+# v2.11.3
+
+Fixed a bug where Julia code in docstrings would be formatted incorrectly if the code itself contained characters that had to be escaped. (#1224, #1228)
+
 # v2.11.2
 
 Fixed a bug where YASStyle would insert a trailing comma for a function call with zero arguments, which is invalid Julia syntax. (#1225, #1226)

Project.toml

@@ -1,6 +1,6 @@
 name = "JuliaFormatter"
 uuid = "98e50ef6-434e-11e9-1051-2b60c6c9e899"
-version = "2.11.2"
+version = "2.11.3"
 authors = ["Dominique Luna <dluna132@gmail.com> and contributors"]
 
 [workspace]

src/format_docstring.jl

@@ -1,14 +1,76 @@
+# When formatting Julia code inside docstrings, we need to know whether it's inside
+# a single-quoted Julia docstring, a triple-quoted Julia docstring, or just a Markdown
+# file. This affects how we handle escape sequences.
+@enum FormatDocstringContext SingleQuotedDocstring TripleQuotedDocstring MarkdownFile
+
 struct FormatRule
     style::AbstractStyle
     opts::Options{Union{}}
+    context::FormatDocstringContext
+end
+
+const _REPLACEMENT_PAIRS = (
+    # Not sure what other replacements are needed?
+    "\\\\" => "\\",
+    "\\\$" => "\$",
+    "\\\"" => "\"",
+)
+function unescape_docstring_code(text::AbstractString, context::FormatDocstringContext)
+    return if context === MarkdownFile
+        text
+    else
+        replace(text, (k => v for (k, v) in _REPLACEMENT_PAIRS)...)
+    end
+end
+function escape_docstring_code(text::AbstractString, context::FormatDocstringContext)
+    return if context === MarkdownFile
+        text
+    elseif context === TripleQuotedDocstring
+        # If the output is within a triple-quoted docstring, then we don't need to
+        # fully escape `"""`: we can get away with just doing `\\"""`.
+        text_chunks = split(text, "\"\"\"")
+        text_chunks = map(x -> escape_docstring_code(x, SingleQuotedDocstring), text_chunks)
+        join(text_chunks, "\\\"\"\"")
+    else
+        replace(text, (v => k for (k, v) in _REPLACEMENT_PAIRS)...)
+    end
 end
 
-function format_docstring_julia(text::AbstractString, fr::FormatRule)
+function format_docstring_code(text::AbstractString, fr::FormatRule)
     try
-        _format_text(text, fr.style, fr.opts)
+        # Julia code within docstrings may contain escape sequences. For example if we want
+        # to write `A \ b` in a docstring, it has to be written as (for example)
+        # 
+        #     """
+        #     [...]
+        #     julia> A \\ b
+        #
+        #     [...]
+        #     """
+        #     function foo end
+        #
+        # However, this means that the `text` we receive here is actually doubly
+        # escaped because it's the literal contents of the docstring:
+        #
+        #     text = "A \\\\ b"
+        #
+        # which is not the same thing as the code we want to format, which is
+        #
+        #     code = "A \\ b"
+        #
+        # So we need to unescape it before formatting, and then re-escape it afterwards.
+        # Note that docstrings have to be `"""..."""`: prefixed strings (e.g.
+        # `raw"""..."""`) are not valid docstrings, so we don't have to worry about them.
+        #
+        # https://github.com/JuliaEditorSupport/JuliaFormatter.jl/issues/1224
+        escape_docstring_code(
+            _format_text(unescape_docstring_code(text, fr.context), fr.style, fr.opts),
+            fr.context,
+        )
     catch e
         if e isa JuliaSyntax.ParseError
-            # Original code was invalid. Just pass it through.
+            # Original code was invalid Julia (not through any fault of ours). Just pass it
+            # through.
             return text
         else
             rethrow(e)
@@ -28,7 +90,7 @@ function block_modifier(rule::FormatRule)
                 for (i, (an_input, output)) in enumerate(chunks)
                     write(doctests, "julia> ")
                     for (j, line) in
-                        enumerate(split(format_docstring_julia(an_input, rule), '\n'))
+                        enumerate(split(format_docstring_code(an_input, rule), '\n'))
                         if j > 1
                             if line == ""
                                 write(doctests, "\n")
@@ -54,12 +116,12 @@ function block_modifier(rule::FormatRule)
             elseif occursin(r"\n+# output\n+", code)
                 input, output = split(code, r"\n+# output\n+"; limit = 2)
                 string(
-                    format_docstring_julia(String(input), rule),
+                    format_docstring_code(String(input), rule),
                     "\n\n# output\n\n",
                     output,
                 )
             else
-                format_docstring_julia(code, rule)
+                format_docstring_code(code, rule)
             end
         end
     end
@@ -128,7 +190,11 @@ function format_docstring(style::AbstractStyle, state::State, text::AbstractStri
                     MathRule(),
                     TableRule(),
                     FrontMatterRule(),
-                    FormatRule(style, state.opts),
+                    FormatRule(style, state.opts, if is_triple_quoted
+                        TripleQuotedDocstring
+                    else
+                        SingleQuotedDocstring
+                    end),
                 ],
             )(
                 deindented_string,
@@ -139,7 +205,7 @@ function format_docstring(style::AbstractStyle, state::State, text::AbstractStri
         formatted = ""
     end
     # Render into text lines, taking care of original indentation,
-    quot = is_triple_quoted ? "\"\"\"" : '"'
+    quot = is_triple_quoted ? "\"\"\"" : "\""
     indentation = " "^state.indent
     indent(line) = indentation * line
     clean(line) = all(isspace, line) ? "" : line # don't write empty lines #667

src/markdown.jl

@@ -28,7 +28,7 @@ function _format_md(text::AbstractString, style::AbstractStyle, opts::Options{Un
                 MathRule(),
                 TableRule(),
                 FrontMatterRule(),
-                FormatRule(style, opts),
+                FormatRule(style, opts, MarkdownFile),
             ],
         )(
             text,

test/options/format_docstrings.jl

@@ -388,6 +388,58 @@ using Test
             test_format(str, str; format_docstrings = true)
         end
     end
+
+    @testset "1224 escape sequences" begin
+        for escaped_julia_code in (
+            raw"@macro a.\$s",
+            raw"A \\ b",
+            "A = \\\"hello\\\"",
+        )
+            s = """
+            \"""
+            ```julia
+            $(escaped_julia_code)
+            ```
+            \"""
+            f
+            """
+            test_format(s, s; format_docstrings=true)
+        end
+
+        @testset "normalisation of triple quotes" begin
+            # Regardless of how the user has escaped their triple quotes inside docstrings,
+            # we want to always normalise them to `\"""`
+            s_ = """
+            \"""
+            ```julia
+            s = \\\"\\\"\\\"hello\\\"\\\"\\\"
+            ```
+            \"""
+            f
+            """
+
+            s = """
+            \"""
+            ```julia
+            s = \\\"\"\"hello\\\"\"\"
+            ```
+            \"""
+            f
+            """
+            test_format(s_, s; format_docstrings=true)
+            test_format(s, s; format_docstrings=true)
+
+            # However if the docstring is single quoted we need to normalise to `\"\"\"`
+            # instead
+            s = """
+            \"```julia
+            s = \\\"\\\"\\\"hello\\\"\\\"\\\"
+            ```\"
+            f
+            """ 
+            test_format(s, s; format_docstrings=true, enforce_triplequoted_docstrings=false)
+        end
+    end
 end
 
 end # module OptionsFormatDocstringsTests