feat: add option for table chunking (#4354)

## Summary

Adds `isolate_tables` (default `True`) to `chunk_elements` and
`chunk_by_title`, making the PR #4307 table-isolation behavior optional.

- `True` (default): preserves current behavior — `Table`/`TableChunk`
elements are always staged alone and never combined with neighbors by
`PreChunkCombiner`.
- `False`: restores the pre-#4307 behavior — tables can share pre-chunks
with adjacent text and be combined into a `CompositeElement`.

Rejects `skip_table_chunking=True` + `isolate_tables=False` with a
`ValueError`, since the pass-through path only fires when a table is
alone in its pre-chunk.

<!-- This is an auto-generated description by cubic. -->
---
## Summary by cubic
Adds `isolate_tables` to `chunk_elements` and `chunk_by_title` to toggle
table isolation; default `True` preserves current behavior, while
`False` lets tables share pre-chunks and merge with adjacent text into
`CompositeElement`s. Validation now uses bool-coerced options and raises
`ValueError` when `skip_table_chunking=True` with `isolate_tables=False`
(including falsy values like `0`) for predictable behavior.

<sup>Written for commit 1da0506.
Summary will update on new commits. <a
href="https://cubic.dev/pr/Unstructured-IO/unstructured/pull/4354?utm_source=github">Review
in cubic</a></sup>

<!-- End of auto-generated description by cubic. -->

---------

Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

Author: badGarnet

CHANGELOG.md

@@ -1,3 +1,9 @@
+## 0.22.30
+
+### Enhancements
+
+- **Toggle table isolation in chunking**: Add `isolate_tables` to basic/title chunking options. Defaults to `True` (the post-#4307 behavior: `Table`/`TableChunk` elements always staged alone). Set to `False` to allow tables to share pre-chunks with adjacent non-table elements and be combined by `PreChunkCombiner`.
+
 ## 0.22.29
 
 ### Fixes

test_unstructured/chunking/test_base.py

@@ -127,6 +127,34 @@ def it_knows_whether_to_repeat_table_headers_by_default(
     def it_knows_whether_to_skip_table_chunking(self, kwargs: dict[str, Any], expected_value: bool):
         assert ChunkingOptions(**kwargs).skip_table_chunking is expected_value
 
+    @pytest.mark.parametrize(
+        ("kwargs", "expected_value"),
+        [
+            ({"isolate_tables": True}, True),
+            ({"isolate_tables": False}, False),
+            ({"isolate_tables": None}, True),
+            ({}, True),
+        ],
+    )
+    def it_knows_whether_to_isolate_tables(self, kwargs: dict[str, Any], expected_value: bool):
+        assert ChunkingOptions(**kwargs).isolate_tables is expected_value
+
+    @pytest.mark.parametrize(
+        ("skip", "isolate"),
+        [
+            (True, False),
+            (1, 0),
+            (True, 0),
+            (1, False),
+        ],
+    )
+    def it_rejects_skip_table_chunking_when_isolation_is_disabled(self, skip: Any, isolate: Any):
+        with pytest.raises(
+            ValueError,
+            match="'skip_table_chunking=True' requires 'isolate_tables=True'",
+        ):
+            ChunkingOptions(skip_table_chunking=skip, isolate_tables=isolate)._validate()
+
     @pytest.mark.parametrize("n_chars", [-1, -42])
     def it_rejects_new_after_n_chars_for_n_less_than_zero(self, n_chars: int):
         with pytest.raises(

test_unstructured/chunking/test_basic.py

@@ -284,6 +284,23 @@ def it_supports_the_skip_table_chunking_option(
         _, opts = _chunk_elements_.call_args.args
         assert opts.skip_table_chunking is expected_value
 
+    @pytest.mark.parametrize(
+        ("kwargs", "expected_value"),
+        [
+            ({"isolate_tables": True}, True),
+            ({"isolate_tables": False}, False),
+            ({"isolate_tables": None}, True),
+            ({}, True),
+        ],
+    )
+    def it_supports_the_isolate_tables_option(
+        self, kwargs: dict[str, Any], expected_value: bool, _chunk_elements_: Mock
+    ):
+        chunk_elements([], **kwargs)
+
+        _, opts = _chunk_elements_.call_args.args
+        assert opts.isolate_tables is expected_value
+
     # -- fixtures --------------------------------------------------------------------------------
 
     @pytest.fixture()

test_unstructured/chunking/test_table_isolation.py

@@ -238,6 +238,62 @@ def it_never_produces_a_composite_element_that_lists_a_table_in_orig_elements(
             assert not any(isinstance(e, Table) for e in orig)
 
 
+class DescribeTableIsolationDisabled:
+    """When `isolate_tables=False`, the pre-#4307 behavior is restored."""
+
+    def it_lets_a_table_share_a_pre_chunk_with_adjacent_text_when_disabled(self):
+        opts = ChunkingOptions(max_characters=500, isolate_tables=False)
+        builder = PreChunkBuilder(opts=opts)
+        builder.add_element(Text("Short preamble."))
+
+        assert builder.will_fit(Table("Heading\nCell text"))
+
+    def it_lets_text_follow_a_table_in_the_same_pre_chunk_when_disabled(self):
+        opts = ChunkingOptions(max_characters=500, isolate_tables=False)
+        builder = PreChunkBuilder(opts=opts)
+        builder.add_element(Table("Heading\nCell text"))
+
+        assert builder.will_fit(Text("Follow-up paragraph."))
+
+    def it_combines_a_table_pre_chunk_with_text_neighbors_when_disabled(self):
+        opts = ChunkingOptions(
+            max_characters=500, combine_text_under_n_chars=500, isolate_tables=False
+        )
+        stream = [
+            PreChunk([Text("Hello world.")], overlap_prefix="", opts=opts),
+            PreChunk([Table("H\nC")], overlap_prefix="", opts=opts),
+            PreChunk([Text("Goodbye world.")], overlap_prefix="", opts=opts),
+        ]
+
+        combined = list(PreChunkCombiner(stream, opts=opts).iter_combined_pre_chunks())
+
+        assert len(combined) == 1
+        assert combined[0]._elements == [
+            Text("Hello world."),
+            Table("H\nC"),
+            Text("Goodbye world."),
+        ]
+
+    def it_wraps_a_table_into_a_composite_element_with_neighbors_when_disabled(self):
+        """End-to-end: opting out collapses tiny table + text into one CompositeElement."""
+        elements = [
+            Text("preamble"),
+            Table("H\nC"),
+            Text("post"),
+        ]
+        chunks = chunk_elements(
+            elements,
+            max_characters=500,
+            isolate_tables=False,
+        )
+
+        assert len(chunks) == 1
+        assert isinstance(chunks[0], CompositeElement)
+        text = chunks[0].text
+        assert "preamble" in text
+        assert "post" in text
+
+
 class DescribeTableIsolationOverlapAll:
     """With overlap_all=True, overlap must not cross table / narrative boundaries."""
 

test_unstructured/chunking/test_title.py

@@ -609,6 +609,23 @@ def it_supports_the_skip_table_chunking_option(
         _, opts = _chunk_by_title_.call_args.args
         assert opts.skip_table_chunking is expected_value
 
+    @pytest.mark.parametrize(
+        ("kwargs", "expected_value"),
+        [
+            ({"isolate_tables": True}, True),
+            ({"isolate_tables": False}, False),
+            ({"isolate_tables": None}, True),
+            ({}, True),
+        ],
+    )
+    def it_supports_the_isolate_tables_option(
+        self, kwargs: dict[str, Any], expected_value: bool, _chunk_by_title_: Mock
+    ):
+        chunk_by_title([], **kwargs)
+
+        _, opts = _chunk_by_title_.call_args.args
+        assert opts.isolate_tables is expected_value
+
     # -- fixtures --------------------------------------------------------------------------------
 
     @pytest.fixture()

unstructured/__version__.py

@@ -1 +1 @@
-__version__ = "0.22.29"  # pragma: no cover
+__version__ = "0.22.30"  # pragma: no cover

unstructured/chunking/base.py

@@ -125,6 +125,12 @@ class ChunkingOptions:
     repeat_table_headers
         Default: `True`. When `True`, repeated table-header behavior is enabled for chunked table
         continuations. Specify `False` to opt out and preserve legacy table-chunk behavior.
+    isolate_tables
+        Default: `True`. When `True`, `Table` and `TableChunk` elements are always staged in
+        their own pre-chunk and never combined with adjacent non-table elements. Specify
+        `False` to allow tables to share pre-chunks with adjacent elements (the pre-#4307
+        behavior), which is sometimes useful when downstream consumers expect mixed-content
+        composite chunks.
     text_splitting_separators
         A sequence of strings like `("\n", " ")` to be used as target separators during
         text-splitting. Text-splitting only applies to splitting an oversized element into two or
@@ -208,6 +214,17 @@ def skip_table_chunking(self) -> bool:
         arg_value = self._kwargs.get("skip_table_chunking")
         return False if arg_value is None else bool(arg_value)
 
+    @cached_property
+    def isolate_tables(self) -> bool:
+        """When True, `Table`/`TableChunk` elements are staged in their own pre-chunk.
+
+        Default value is `True`. When `False`, table-family elements are allowed to share a
+        pre-chunk with adjacent non-table elements (and may be merged by `PreChunkCombiner`),
+        restoring the pre-#4307 behavior.
+        """
+        arg_value = self._kwargs.get("isolate_tables")
+        return True if arg_value is None else bool(arg_value)
+
     @cached_property
     def inter_chunk_overlap(self) -> int:
         """Characters of overlap to add between chunks.
@@ -349,6 +366,17 @@ def _validate(self) -> None:
         if new_after_n_chars is not None and new_after_n_chars < 0:
             raise ValueError(f"'new_after_n_chars' argument must be >= 0, got {new_after_n_chars}")
 
+        # -- `skip_table_chunking` requires `isolate_tables` because the pass-through path only
+        # -- fires when the pre-chunk contains a single `Table` element. With isolation disabled,
+        # -- tables can fold into `CompositeElement` alongside neighbors and the skip would be
+        # -- silently ignored, breaking the contract.
+        if self.skip_table_chunking and not self.isolate_tables:
+            raise ValueError(
+                "'skip_table_chunking=True' requires 'isolate_tables=True' (the default);"
+                " tables cannot be passed through unchanged while also sharing a pre-chunk with"
+                " adjacent elements"
+            )
+
         # -- overlap must be less than max-chars or the chunk text will never be consumed --
         if self.overlap >= hard_max:
             raise ValueError(
@@ -502,7 +530,11 @@ def __init__(self, opts: ChunkingOptions) -> None:
     def add_element(self, element: Element) -> None:
         """Add `element` to this section."""
         # -- do not prefix a table-only pre-chunk with narrative overlap from the prior chunk --
-        if len(self._elements) == 0 and _element_is_table_family(element):
+        if (
+            self._opts.isolate_tables
+            and len(self._elements) == 0
+            and _element_is_table_family(element)
+        ):
             self._overlap_prefix = ""
             self._text_segments = []
             self._text_len = 0
@@ -531,7 +563,11 @@ def flush(self) -> Iterator[PreChunk]:
         # -- iterator is exhausted and can add elements for the next pre-chunk immediately.
         overlap_for_next = pre_chunk.overlap_tail
         # -- table tails must not prefix the following narrative pre-chunk (overlap_all) --
-        if len(elements) == 1 and _element_is_table_family(elements[0]):
+        if (
+            self._opts.isolate_tables
+            and len(elements) == 1
+            and _element_is_table_family(elements[0])
+        ):
             overlap_for_next = ""
         self._reset_state(overlap_for_next)
         yield pre_chunk
@@ -548,13 +584,15 @@ def will_fit(self, element: Element) -> bool:
         - A text-element will not fit when together with the elements already present it would
           exceed the hard-max (aka. max_characters/max_tokens).
         """
-        # -- a `Table` can only start a pre-chunk; it is never appended to a non-empty pre-chunk --
-        if _element_is_table_family(element):
-            return len(self._elements) == 0
+        if self._opts.isolate_tables:
+            # -- a `Table` can only start a pre-chunk; it is never appended to a non-empty
+            # -- pre-chunk --
+            if _element_is_table_family(element):
+                return len(self._elements) == 0
 
-        # -- no non-table element may share a pre-chunk with a `Table` --
-        if _elements_contain_table_family(self._elements):
-            return False
+            # -- no non-table element may share a pre-chunk with a `Table` --
+            if _elements_contain_table_family(self._elements):
+                return False
 
         # -- an empty pre-chunk will accept any element (including an oversized-element) --
         if len(self._elements) == 0:
@@ -637,7 +675,9 @@ def __eq__(self, other: Any) -> bool:
 
     def can_combine(self, pre_chunk: PreChunk) -> bool:
         """True when `pre_chunk` can be combined with this one without exceeding size limits."""
-        if _table_isolation_forbids_side_by_side_merge(self._elements, pre_chunk._elements):
+        if self._opts.isolate_tables and _table_isolation_forbids_side_by_side_merge(
+            self._elements, pre_chunk._elements
+        ):
             return False
         if len(self._text) >= self._opts.combine_text_under_n_chars:
             return False

unstructured/chunking/basic.py

@@ -34,6 +34,7 @@ def chunk_elements(
     tokenizer: Optional[str] = None,
     repeat_table_headers: Optional[bool] = None,
     skip_table_chunking: Optional[bool] = None,
+    isolate_tables: Optional[bool] = None,
 ) -> list[Element]:
     """Combine sequential `elements` into chunks, respecting specified text-length limits.
 
@@ -84,6 +85,11 @@ def chunk_elements(
     skip_table_chunking
         Default: `False`. When `True`, `Table` elements are passed through unchanged without
         being split into `TableChunk` elements, regardless of their size.
+    isolate_tables
+        Default: `True`. When `True`, `Table` and `TableChunk` elements are always staged in
+        their own pre-chunk and never combined with adjacent non-table elements. Specify
+        `False` to allow tables to share pre-chunks with adjacent elements (the pre-#4307
+        behavior).
     """
     # -- raises ValueError on invalid parameters --
     opts = _BasicChunkingOptions.new(
@@ -97,6 +103,7 @@ def chunk_elements(
         tokenizer=tokenizer,
         repeat_table_headers=repeat_table_headers,
         skip_table_chunking=skip_table_chunking,
+        isolate_tables=isolate_tables,
     )
 
     return _chunk_elements(elements, opts)

unstructured/chunking/dispatch.py

@@ -74,6 +74,10 @@ def add_chunking_strategy(func: Callable[_P, list[Element]]) -> Callable[_P, lis
             + "\n\t\tskip_table_chunking"
             + "\n\t\t\tDefault: False. When True, Table elements are passed through"
             + "\n\t\t\tunchanged without being split into TableChunk elements."
+            + "\n\t\tisolate_tables"
+            + "\n\t\t\tDefault: True. When True, Table/TableChunk elements are always"
+            + "\n\t\t\tstaged in their own pre-chunk. Set to False to allow tables to"
+            + "\n\t\t\tshare pre-chunks with adjacent non-table elements."
         )
 
     @functools.wraps(func)

unstructured/chunking/title.py

@@ -35,6 +35,7 @@ def chunk_by_title(
     tokenizer: Optional[str] = None,
     repeat_table_headers: Optional[bool] = None,
     skip_table_chunking: Optional[bool] = None,
+    isolate_tables: Optional[bool] = None,
 ) -> list[Element]:
     """Uses title elements to identify sections within the document for chunking.
 
@@ -91,6 +92,11 @@ def chunk_by_title(
     skip_table_chunking
         Default: `False`. When `True`, `Table` elements are passed through unchanged without
         being split into `TableChunk` elements, regardless of their size.
+    isolate_tables
+        Default: `True`. When `True`, `Table` and `TableChunk` elements are always staged in
+        their own pre-chunk and never combined with adjacent non-table elements. Specify
+        `False` to allow tables to share pre-chunks with adjacent elements (the pre-#4307
+        behavior).
     """
     opts = _ByTitleChunkingOptions.new(
         combine_text_under_n_chars=combine_text_under_n_chars,
@@ -105,6 +111,7 @@ def chunk_by_title(
         tokenizer=tokenizer,
         repeat_table_headers=repeat_table_headers,
         skip_table_chunking=skip_table_chunking,
+        isolate_tables=isolate_tables,
     )
     return _chunk_by_title(elements, opts)