diff --git a/CHANGELOG.md b/CHANGELOG.md index 249b64f1a6..8746c1a289 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +## 0.22.35 + +### Fixes + +- **Fix crash on processing instructions in HTML**: `partition_html()` no longer raises `AttributeError: 'lxml.etree._ProcessingInstruction' object has no attribute 'is_phrasing'` when the document contains a processing instruction such as ``. Processing instructions are now dropped during parsing, consistent with how comments are already handled. + ## 0.22.34 ### Fixes diff --git a/test_unstructured/partition/html/test_partition.py b/test_unstructured/partition/html/test_partition.py index 7018b4b4ef..9dba3761fc 100644 --- a/test_unstructured/partition/html/test_partition.py +++ b/test_unstructured/partition/html/test_partition.py @@ -203,6 +203,15 @@ def test_partition_html_processes_chinese_chracters(): assert elements[0].text == "每日新闻" +def test_partition_html_ignores_processing_instructions(): + """A processing-instruction like `` must not crash the parser (see #4358).""" + html_text = ( + '
\n\n' + "

hello world

" + ) + assert partition_html(text=html_text) == [Text("hello world")] + + def test_emoji_appears_with_emoji_utf8_code(): assert partition_html(text='

Hello 😀

') == [ Text("Hello 😀") diff --git a/test_unstructured/partition/test_md.py b/test_unstructured/partition/test_md.py index 6ae8b98872..4cbbb30fcd 100644 --- a/test_unstructured/partition/test_md.py +++ b/test_unstructured/partition/test_md.py @@ -43,6 +43,15 @@ def test_partition_md_from_text(): assert all(e.metadata.filename is None for e in elements) +def test_partition_md_ignores_processing_instructions(): + """A processing-instruction like `` must not crash partition_md (see #4358).""" + elements = partition_md(text='Before\n\n\n\nAfter') + + texts = [e.text for e in elements] + assert "Before" in texts + assert "After" in texts + + class MockResponse: def __init__(self, text: str, status_code: int, headers: dict[str, Any] = {}): self.text = text diff --git a/unstructured/__version__.py b/unstructured/__version__.py index 59c38a47b1..4781090584 100644 --- a/unstructured/__version__.py +++ b/unstructured/__version__.py @@ -1 +1 @@ -__version__ = "0.22.34" # pragma: no cover +__version__ = "0.22.35" # pragma: no cover diff --git a/unstructured/partition/html/parser.py b/unstructured/partition/html/parser.py index b0ffa54973..72349cff48 100644 --- a/unstructured/partition/html/parser.py +++ b/unstructured/partition/html/parser.py @@ -945,7 +945,10 @@ def derive_element_type_from_text(text: str) -> type[Text] | None: # ------------------------------------------------------------------------------------------------ -html_parser = etree.HTMLParser(remove_comments=True) +# -- `remove_pis` drops processing-instructions (e.g. ``); like comments they +# -- carry no rendered text and otherwise reach the parser as `_ProcessingInstruction` nodes +# -- that lack the custom-element interface (`.is_phrasing`), crashing iteration. +html_parser = etree.HTMLParser(remove_comments=True, remove_pis=True) # -- elements that don't have a registered class get DefaultElement -- fallback = etree.ElementDefaultClassLookup(element=DefaultElement) # -- elements that do have a registered class are assigned that class via lookup --