Pass escape=False for pprint to keep real control chars

Author: hugovk

Lib/_pyrepl/utils.py

@@ -307,9 +307,14 @@ def iter_display_chars(
     buffer: str,
     colors: list[ColorSpan] | None = None,
     start_index: int = 0,
+    *,
+    escape: bool = True,
 ) -> Iterator[StyledChar]:
     """Yield visible display characters with widths and semantic color tags.
 
+    With ``escape=True`` (default) ASCII control chars are rewritten to caret
+    notation (``\\n`` -> ``^J``); pass ``escape=False`` to keep them verbatim.
+
     Note: ``colors`` is consumed in place as spans are processed -- callers
     that split a buffer across multiple calls rely on this mutation to track
     which spans have already been handled.
@@ -331,7 +336,7 @@ def iter_display_chars(
         if colors and color_idx < len(colors) and colors[color_idx].span.start == i:
             active_tag = colors[color_idx].tag
 
-        if control := _ascii_control_repr(c):
+        if escape and (control := _ascii_control_repr(c)):
             text = control
             width = len(control)
         elif ord(c) < 128:
@@ -363,6 +368,8 @@ def disp_str(
     colors: list[ColorSpan] | None = None,
     start_index: int = 0,
     force_color: bool = False,
+    *,
+    escape: bool = True,
 ) -> tuple[CharBuffer, CharWidths]:
     r"""Decompose the input buffer into a printable variant with applied colors.
 
@@ -374,6 +381,9 @@ def disp_str(
     - the second list is the visible width of each character in the input
       buffer.
 
+    With ``escape=True`` (default) ASCII control chars are rewritten to caret
+    notation (``\\n`` -> ``^J``); pass ``escape=False`` to keep them verbatim.
+
     Note on colors:
     - The `colors` list, if provided, is partially consumed within. We're using
       a list and not a generator since we need to hold onto the current
@@ -393,7 +403,9 @@ def disp_str(
     (['\x1b[1;34mw', 'h', 'i', 'l', 'e\x1b[0m', ' ', '1', ':'], [1, 1, 1, 1, 1, 1, 1, 1])
 
     """
-    styled_chars = list(iter_display_chars(buffer, colors, start_index))
+    styled_chars = list(
+        iter_display_chars(buffer, colors, start_index, escape=escape)
+    )
     chars: CharBuffer = []
     char_widths: CharWidths = []
     theme = THEME(force_color=force_color)

Lib/pprint.py

@@ -134,7 +134,7 @@ def _safe_tuple(t):
 def _colorize_output(text):
     """Apply syntax highlighting."""
     colors = list(gen_colors(text))
-    chars, _ = disp_str(text, colors=colors, force_color=True)
+    chars, _ = disp_str(text, colors=colors, force_color=True, escape=False)
     return "".join(chars)
 
 

Lib/test/test_pprint.py

@@ -214,6 +214,27 @@ def test_color_prettyprinter(self):
             pp.pprint(obj)
             self.assertNotIn("\x1b[", stream.getvalue())
 
+    def test_color_preserves_newlines(self):
+        """Color multiline output must use real newlines, not '^J'."""
+        obj = {"a": 1, "b": 2, "c": 3, "d": [10, 20, 30, 40, 50, 60, 70, 80]}
+
+        plain_stream = io.StringIO()
+        pprint.pprint(obj, stream=plain_stream, width=20, color=False)
+        plain = plain_stream.getvalue()
+        self.assertIn("\n", plain)
+
+        with unittest.mock.patch.dict(
+            "os.environ", {"FORCE_COLOR": "1", "NO_COLOR": ""}
+        ):
+            color_stream = io.StringIO()
+            pprint.pprint(obj, stream=color_stream, width=20, color=True)
+            color = color_stream.getvalue()
+
+        self.assertIn("\x1b[", color)  # has color
+        self.assertNotIn("^J", color)
+        stripped = re.sub(r"\x1b\[[0-9;]*m", "", color)
+        self.assertEqual(stripped, plain)
+
     def test_basic(self):
         # Verify .isrecursive() and .isreadable() w/o recursion
         pp = pprint.PrettyPrinter()

Lib/test/test_pyrepl/test_utils.py

@@ -1,6 +1,12 @@
 from unittest import TestCase
 
-from _pyrepl.utils import str_width, wlen, prev_next_window, gen_colors
+from _pyrepl.utils import (
+    disp_str,
+    gen_colors,
+    prev_next_window,
+    str_width,
+    wlen,
+)
 
 
 class TestUtils(TestCase):
@@ -135,3 +141,12 @@ def test_gen_colors_keyword_highlighting(self):
                     span_text = code[color.span.start:color.span.end + 1]
                     actual_highlights.append((span_text, color.tag))
                 self.assertEqual(actual_highlights, expected_highlights)
+
+    def test_disp_str_escape(self):
+        # default: control chars become caret notation
+        chars, _ = disp_str("a\nb\tc\x1bd")
+        self.assertEqual("".join(chars), "a^Jb^Ic^[d")
+
+        # escape=False: control chars pass through verbatim
+        chars, _ = disp_str("a\nb\tc\x1bd", escape=False)
+        self.assertEqual("".join(chars), "a\nb\tc\x1bd")