Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,3 +1,9 @@
## 0.22.35

### Fixes

- **Fix crash on processing instructions in HTML**: `partition_html()` no longer raises `AttributeError: 'lxml.etree._ProcessingInstruction' object has no attribute 'is_phrasing'` when the document contains a processing instruction such as `<?xml ...?>`. Processing instructions are now dropped during parsing, consistent with how comments are already handled.

## 0.22.34

### Fixes
Expand Down
9 changes: 9 additions & 0 deletions test_unstructured/partition/html/test_partition.py
Original file line number Diff line number Diff line change
Expand Up @@ -203,6 +203,15 @@ def test_partition_html_processes_chinese_chracters():
assert elements[0].text == "每日新闻"


def test_partition_html_ignores_processing_instructions():
"""A processing-instruction like `<?xml ...?>` must not crash the parser (see #4358)."""
html_text = (
'<html><body><div>\n<?xml version="1.0" encoding="UTF-8"?>\n'
"<p>hello world</p></div></body></html>"
)
assert partition_html(text=html_text) == [Text("hello world")]


def test_emoji_appears_with_emoji_utf8_code():
assert partition_html(text='<html charset="utf-8"><p>Hello &#128512;</p></html>') == [
Text("Hello 😀")
Expand Down
9 changes: 9 additions & 0 deletions test_unstructured/partition/test_md.py
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,15 @@ def test_partition_md_from_text():
assert all(e.metadata.filename is None for e in elements)


def test_partition_md_ignores_processing_instructions():
"""A processing-instruction like `<?xml ...?>` must not crash partition_md (see #4358)."""
elements = partition_md(text='Before\n\n<?xml version="1.0"?>\n\nAfter')

texts = [e.text for e in elements]
assert "Before" in texts
assert "After" in texts


class MockResponse:
def __init__(self, text: str, status_code: int, headers: dict[str, Any] = {}):
self.text = text
Expand Down
2 changes: 1 addition & 1 deletion unstructured/__version__.py
Original file line number Diff line number Diff line change
@@ -1 +1 @@
__version__ = "0.22.34" # pragma: no cover
__version__ = "0.22.35" # pragma: no cover
5 changes: 4 additions & 1 deletion unstructured/partition/html/parser.py
Original file line number Diff line number Diff line change
Expand Up @@ -945,7 +945,10 @@ def derive_element_type_from_text(text: str) -> type[Text] | None:
# ------------------------------------------------------------------------------------------------


html_parser = etree.HTMLParser(remove_comments=True)
# -- `remove_pis` drops processing-instructions (e.g. `<?xml ...?>`); like comments they
# -- carry no rendered text and otherwise reach the parser as `_ProcessingInstruction` nodes
# -- that lack the custom-element interface (`.is_phrasing`), crashing iteration.
html_parser = etree.HTMLParser(remove_comments=True, remove_pis=True)
# -- elements that don't have a registered class get DefaultElement --
fallback = etree.ElementDefaultClassLookup(element=DefaultElement)
# -- elements that do have a registered class are assigned that class via lookup --
Expand Down