fix: Preserve Line Breaks in Code Blocks During Chunking (#4196)

## Problem

When using `chunk_elements()` on markdown files containing code blocks,
line breaks within the code were being discarded, resulting in
unreadable code:

Fixes #4095 

```python
# Before fix - code becomes flattened:
"def hello(): print('Hello') return True"

# Expected - preserve formatting:
"def hello():\n    print('Hello')\n    return True"
```

## Root Cause

Two issues were identified:

1. **HTML Parser**: `<pre>` elements generated generic `Text` elements
instead of `CodeSnippet` elements
2. **Chunking**: The `_iter_text_segments()` method normalized all
whitespace to single spaces, destroying newlines

## Solution

### 1. HTML Parser Change (`unstructured/partition/html/parser.py`)

Made `<pre>` elements generate `CodeSnippet` elements:

```python
class Pre(BlockItem):
    """Custom element-class for `<pre>` element.

    Can only contain phrasing content. Generates CodeSnippet elements to preserve
    code formatting including whitespace and line breaks.
    """

    _ElementCls = CodeSnippet  # Added this line
```

### 2. Chunking Change (`unstructured/chunking/base.py`)

Modified `_iter_text_segments()` to preserve whitespace for
`CodeSnippet` elements:

```python
def _iter_text_segments(self) -> Iterator[str]:
    """Generate overlap text and each element text segment in order.

    Empty text segments are not included. CodeSnippet elements preserve their
    original whitespace (including newlines) to maintain code formatting.
    """
    if self._overlap_prefix:
        yield self._overlap_prefix
    for e in self._elements:
        if e.text and len(e.text):
            # -- preserve whitespace for code snippets to maintain formatting --
            if isinstance(e, CodeSnippet):
                text = e.text.strip()
            else:
                text = " ".join(e.text.strip().split())
            if text:
                yield text
```

## Files Changed

| File | Change |
|------|--------|
| `unstructured/partition/html/parser.py` | Added `CodeSnippet` import,
set `_ElementCls = CodeSnippet` in `Pre` class |
| `unstructured/chunking/base.py` | Added `CodeSnippet` import, special
handling in `_iter_text_segments()` |
| `test_unstructured/partition/html/test_parser.py` | Added test for
`CodeSnippet` generation, updated existing test |
| `test_unstructured/chunking/test_base.py` | Added 2 tests for
whitespace preservation |

Contribution by Gittensor, see my contribution statistics at
https://gittensor.io/miners/details?githubId=42954461

Author: eureka0928

CHANGELOG.md

@@ -1,4 +1,4 @@
-## 0.18.31-dev1
+## 0.18.31-dev2
 
 ### Enhancements
 - Changed default DPI to 350
@@ -7,6 +7,7 @@
 ### Fixes
 - **Fix Pandoc exitcode 97 during ODT conversion**: Try with sandbox=True first, fallback without sandbox only if `ALLOW_PANDOC_NO_SANDBOX=true` env var is set (fixes #3997)
 - **Fix `coordinates=True` causing TypeError in hi_res PDF processing**: Filter out `coordinates` and `coordinate_system` from kwargs before passing to `add_element_metadata()` to prevent conflict with explicit parameters (fixes #4126)
+- **Preserve line breaks in code blocks during chunking**: `<pre>` elements now generate `CodeSnippet` elements instead of `Text`, and chunking preserves internal whitespace for code snippets. (fixes #4095)
 
 ## 0.18.30 
 

test_unstructured/chunking/test_base.py

@@ -29,6 +29,7 @@
 from unstructured.common.html_table import HtmlCell, HtmlRow, HtmlTable
 from unstructured.documents.elements import (
     CheckBox,
+    CodeSnippet,
     CompositeElement,
     Element,
     ElementMetadata,
@@ -802,6 +803,37 @@ def it_knows_the_concatenated_text_of_the_pre_chunk_to_help(
         pre_chunk = PreChunk(elements, overlap_prefix=overlap_prefix, opts=ChunkingOptions())
         assert pre_chunk._text == expected_value
 
+    def it_preserves_whitespace_in_CodeSnippet_elements(self):
+        """CodeSnippet elements should preserve their internal whitespace including newlines.
+
+        This is important for code blocks where formatting (indentation, line breaks) is
+        semantically meaningful.
+        """
+        code_text = "def hello():\n    print('Hello')\n    return True"
+        pre_chunk = PreChunk([CodeSnippet(code_text)], overlap_prefix="", opts=ChunkingOptions())
+
+        # The text should preserve newlines, not collapse them to spaces
+        assert "\n" in pre_chunk._text
+        assert pre_chunk._text == code_text
+
+    def it_preserves_whitespace_in_CodeSnippet_when_mixed_with_other_elements(self):
+        """CodeSnippet whitespace is preserved even when mixed with regular Text elements."""
+        code_text = "for i in range(10):\n    print(i)"
+        pre_chunk = PreChunk(
+            [
+                Text("Here is some code:"),
+                CodeSnippet(code_text),
+                Text("That was the code."),
+            ],
+            overlap_prefix="",
+            opts=ChunkingOptions(),
+        )
+
+        # The combined text should have the code with preserved newlines
+        assert "for i in range(10):\n    print(i)" in pre_chunk._text
+        # Regular text elements are still joined with blank line separators
+        assert "Here is some code:\n\n" in pre_chunk._text
+
 
 # ================================================================================================
 # CHUNKING HELPER/SPLITTERS

test_unstructured/partition/html/test_parser.py

@@ -10,7 +10,15 @@
 import pytest
 from lxml import etree
 
-from unstructured.documents.elements import Address, Element, ListItem, NarrativeText, Text, Title
+from unstructured.documents.elements import (
+    Address,
+    CodeSnippet,
+    Element,
+    ListItem,
+    NarrativeText,
+    Text,
+    Title,
+)
 from unstructured.partition.html.parser import (
     Annotation,
     DefaultElement,
@@ -536,7 +544,7 @@ def it_preserves_the_whitespace_of_its_phrasing_only_contents(self):
         elements = pre.iter_elements()
 
         e = next(elements)
-        assert e == Text(
+        assert e == CodeSnippet(
             "  The Answer to the Great Question...   Of Life, the Universe and Everything...\n"
             "  Is... Forty-two, said Deep Thought, with infinite majesty and calm."
         )
@@ -585,6 +593,19 @@ def it_assigns_emphasis_and_link_metadata_when_contents_have_those_phrasing_elem
         assert e.metadata.link_texts == ["penguin"]
         assert e.metadata.link_urls == ["http://eie.io"]
 
+    def it_generates_CodeSnippet_elements_to_preserve_code_formatting(self):
+        """Pre elements should generate CodeSnippet elements, not generic Text elements.
+
+        This ensures code formatting (whitespace, line breaks) is preserved during chunking.
+        """
+        html_text = "<pre>def hello():\n    print('Hello')\n    return True</pre>"
+        pre = etree.fromstring(html_text, html_parser).xpath(".//pre")[0]
+
+        e = next(pre.iter_elements())
+
+        assert isinstance(e, CodeSnippet)
+        assert e.text == "def hello():\n    print('Hello')\n    return True"
+
 
 class DescribeRemovedBlock:
     """Isolated unit-test suite for `unstructured.partition.html.parser.RemovedBlock`.

test_unstructured/partition/html/test_partition.py

@@ -25,6 +25,7 @@
 from unstructured.cleaners.core import clean_extra_whitespace
 from unstructured.documents.elements import (
     Address,
+    CodeSnippet,
     CompositeElement,
     ElementType,
     ListItem,
@@ -627,7 +628,7 @@ def test_partition_html_with_widely_encompassing_pre_tag():
     print(f"{len(elements)=}")
     assert len(elements) > 0
     assert clean_extra_whitespace(elements[0].text).startswith("[107th Congress Public Law 56]")
-    assert isinstance(elements[0], NarrativeText)
+    assert isinstance(elements[0], CodeSnippet)
     assert elements[0].metadata.filetype == "text/html"
     assert elements[0].metadata.filename == "fake-html-pre.htm"
 
@@ -641,9 +642,9 @@ def test_pre_tag_parsing_respects_order():
             "<div>The Big Blue Bear</div>\n"
         )
     ) == [
-        Text("The Big Brown Bear"),
+        CodeSnippet("The Big Brown Bear"),
         NarrativeText("The big brown bear is growling."),
-        NarrativeText("The big brown bear is sleeping."),
+        CodeSnippet("The big brown bear is sleeping."),
         Text("The Big Blue Bear"),
     ]
 
@@ -672,7 +673,7 @@ def test_partition_html_br_tag_parsing():
         Title("Header 1"),
         Text("Text"),
         Title("Header 2"),
-        Text(
+        CodeSnippet(
             "    Param1 = Y\nParam2 = 1\nParam3 = 2\nParam4 = A\n    \nParam5 = A,B,C,D,E\n"
             "Param6 = 7\nParam7 = Five\n\n  "
         ),

test_unstructured/partition/test_auto.py

@@ -32,6 +32,7 @@
 from unstructured.cleaners.core import clean_extra_whitespace
 from unstructured.documents.elements import (
     Address,
+    CodeSnippet,
     CompositeElement,
     Element,
     ElementMetadata,
@@ -272,7 +273,7 @@ def test_auto_partition_html_pre_from_file():
     assert len(elements) > 0
     assert "PageBreak" not in [elem.category for elem in elements]
     assert clean_extra_whitespace(elements[0].text).startswith("[107th Congress Public Law 56]")
-    assert isinstance(elements[0], NarrativeText)
+    assert isinstance(elements[0], CodeSnippet)
     assert all(e.metadata.filetype == "text/html" for e in elements)
     assert all(e.metadata.filename == "fake-html-pre.htm" for e in elements)
 

test_unstructured_ingest/expected-structured-output-html/local-single-file-with-encoding/fake-html-cp1252.html.html

@@ -16,13 +16,13 @@ <h1 class="Title" id="a59f117741c76dca0bc8f5ee72e2010b">
   <p class="UncategorizedText" id="d536ba7636a9a4603a81b358d1fe2590">
    Some text with CP1252-specific characters:
   </p>
-  <p class="NarrativeText" id="3b8ca5305e52587b8fbbfcd994de0667">
+  <div class="CodeSnippet" id="3b8ca5305e52587b8fbbfcd994de0667">
    Die schöne Frau hat einen Kaffee mit Kuchen gegessen. Sie sagte: "Das war köstlich!" und lächelte dabei. Der Preis betrug 15,50 €.
 L'été était trčs chaud cette année. J'ai acheté un café au lait pour 3,50 €. C'était délicieux ! L'homme a dit : "C'est parfait !"
 El nińo comió paella con ńoquis. La seńora dijo: "ˇQué rico!" y pagó 25,75 €. El restaurante tenía un menú del día.
 Kvinnan ĺt köttbullar med lingonsylt. Hon sa: "Det var fantastiskt!" och betalade 45,90 €. Mannen frĺgade: "Vill du ha mer?"
 O Joăo comprou um café por 2,50 €. Ele disse: "Está ótimo!" e sorriu. A mulher perguntou: "Quer mais alguma coisa?"
 De vrouw dronk koffie met koekjes. Ze zei: "Het was heerlijk!" en betaalde 4,25 €. Het kind vroeg: "Mag ik ook wat?"
-  </p>
+  </div>
  </body>
 </html>

test_unstructured_ingest/expected-structured-output/local-single-file-with-encoding/fake-html-cp1252.html.json

@@ -63,7 +63,7 @@
     }
   },
   {
-    "type": "NarrativeText",
+    "type": "CodeSnippet",
     "element_id": "3b8ca5305e52587b8fbbfcd994de0667",
     "text": "Die schöne Frau hat einen Kaffee mit Kuchen gegessen. Sie sagte: \"Das war köstlich!\" und lächelte dabei. Der Preis betrug 15,50 €.\nL'été était trčs chaud cette année. J'ai acheté un café au lait pour 3,50 €. C'était délicieux ! L'homme a dit : \"C'est parfait !\"\nEl nińo comió paella con ńoquis. La seńora dijo: \"ˇQué rico!\" y pagó 25,75 €. El restaurante tenía un menú del día.\nKvinnan ĺt köttbullar med lingonsylt. Hon sa: \"Det var fantastiskt!\" och betalade 45,90 €. Mannen frĺgade: \"Vill du ha mer?\"\nO Joăo comprou um café por 2,50 €. Ele disse: \"Está ótimo!\" e sorriu. A mulher perguntou: \"Quer mais alguma coisa?\"\nDe vrouw dronk koffie met koekjes. Ze zei: \"Het was heerlijk!\" en betaalde 4,25 €. Het kind vroeg: \"Mag ik ook wat?\"",
     "metadata": {

unstructured/__version__.py

@@ -1 +1 @@
-__version__ = "0.18.31-dev1"  # pragma: no cover
+__version__ = "0.18.31-dev2"  # pragma: no cover

unstructured/chunking/base.py

@@ -11,6 +11,7 @@
 
 from unstructured.common.html_table import HtmlCell, HtmlRow, HtmlTable
 from unstructured.documents.elements import (
+    CodeSnippet,
     CompositeElement,
     ConsolidationStrategy,
     Element,
@@ -610,15 +611,20 @@ def overlap_tail(self) -> str:
     def _iter_text_segments(self) -> Iterator[str]:
         """Generate overlap text and each element text segment in order.
 
-        Empty text segments are not included.
+        Empty text segments are not included. CodeSnippet elements preserve their
+        original whitespace (including newlines) to maintain code formatting.
         """
         if self._overlap_prefix:
             yield self._overlap_prefix
         for e in self._elements:
             if e.text and len(e.text):
-                text = " ".join(e.text.strip().split())
-                if text:
-                    yield text
+                # -- preserve all whitespace for code snippets to maintain formatting --
+                if isinstance(e, CodeSnippet):
+                    yield e.text
+                else:
+                    text = " ".join(e.text.strip().split())
+                    if text:
+                        yield text
 
     @lazyproperty
     def _text(self) -> str:

unstructured/partition/html/parser.py

@@ -87,6 +87,7 @@
 from unstructured.common.html_table import htmlify_matrix_of_cell_texts
 from unstructured.documents.elements import (
     Address,
+    CodeSnippet,
     Element,
     ElementMetadata,
     EmailAddress,
@@ -470,9 +471,12 @@ class ListItemBlock(Flow):
 class Pre(BlockItem):
     """Custom element-class for `<pre>` element.
 
-    Can only contain phrasing content.
+    Can only contain phrasing content. Generates CodeSnippet elements to preserve
+    code formatting including whitespace and line breaks.
     """
 
+    _ElementCls = CodeSnippet
+
     @lazyproperty
     def _element_accum(self) -> _ElementAccumulator:
         """Text-segment accumulator suitable for this block-element."""