Skip to content

fix: preserve numeric text in table HTML metadata #4373

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
gyx09212214-prog wants to merge 5 commits into
base: main
Choose a base branch
from
+207 −12
Open

fix: preserve numeric text in table HTML metadata #4373

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
34a28a8
test: guard HTML table numeric cell text
gyx09212214-prog Jun 12, 2026
d566717
fix: preserve numeric text in delimited table html
gyx09212214-prog Jun 19, 2026
5474f43
refactor: share dataframe table html rendering
gyx09212214-prog Jun 19, 2026
602be91
fix: preserve empty cells in dataframe html
gyx09212214-prog Jun 19, 2026
5f52121
test: cover leading-zero table cell text
gyx09212214-prog Jun 20, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
46 changes: 46 additions & 0 deletions test_unstructured/common/test_html_table.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,13 +4,15 @@

from __future__ import annotations

import pandas as pd
import pytest
from lxml.html import fragment_fromstring

from unstructured.common.html_table import (
HtmlCell,
HtmlRow,
HtmlTable,
htmlify_dataframe,
htmlify_matrix_of_cell_texts,
)

Expand Down Expand Up @@ -43,6 +45,50 @@ def test_htmlify_matrix_handles_empty_matrix(self):
assert htmlify_matrix_of_cell_texts([]) == ""


class Describe_htmlify_dataframe:
"""Unit-test suite for `unstructured.common.html_table.htmlify_dataframe()`."""

def it_preserves_numeric_text_when_rendering_dataframe_cell_values(self):
dataframe = pd.DataFrame(
[
["revenue", "478923"],
["ticker", "000001"],
["account", "000123"],
["clause", "007"],
["ratio", "0.000123"],
],
columns=["metric", "value"],
)

assert htmlify_dataframe(dataframe, include_header=True) == (
"<table>"
"<tr><td>metric</td><td>value</td></tr>"
"<tr><td>revenue</td><td>478923</td></tr>"
"<tr><td>ticker</td><td>000001</td></tr>"
"<tr><td>account</td><td>000123</td></tr>"
"<tr><td>clause</td><td>007</td></tr>"
"<tr><td>ratio</td><td>0.000123</td></tr>"
"</table>"
)

def it_renders_dataframe_null_values_as_empty_cells(self):
dataframe = pd.DataFrame(
[
["real nulls", None, float("nan"), pd.NA],
["literal text", "None", "nan", "<NA>"],
],
columns=["kind", "a", "b", "c"],
)

assert htmlify_dataframe(dataframe, include_header=True) == (
"<table>"
"<tr><td>kind</td><td>a</td><td>b</td><td>c</td></tr>"
"<tr><td>real nulls</td><td/><td/><td/></tr>"
"<tr><td>literal text</td><td>None</td><td>nan</td><td>&lt;NA&gt;</td></tr>"
"</table>"
)


class DescribeHtmlTable:
"""Unit-test suite for `unstructured.common.html_table.HtmlTable`."""

Expand Down
41 changes: 41 additions & 0 deletions test_unstructured/partition/html/test_partition.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -203,6 +203,47 @@ def test_partition_html_processes_chinese_chracters():
assert elements[0].text == "每日新闻"


def test_partition_html_preserves_numeric_table_cell_text_in_chunked_html():
html_text = """
<html>
<body>
<h1>Financial metrics</h1>
<table>
<tr><th>metric</th><th>value</th></tr>
<tr><td>revenue</td><td>478923</td></tr>
<tr><td>shares</td><td>1234567</td></tr>
<tr><td>ticker</td><td>000001</td></tr>
<tr><td>account</td><td>000123</td></tr>
<tr><td>clause</td><td>007</td></tr>
<tr><td>ratio</td><td>0.000123</td></tr>
<tr><td>market cap</td><td>999999999999999999</td></tr>
</table>
</body>
</html>
"""

elements = partition_html(
text=html_text,
chunking_strategy="by_title",
max_characters=80,
new_after_n_chars=80,
)

table_html = "".join(
e.metadata.text_as_html or "" for e in elements if isinstance(e, TableChunk)
)

assert "<td>478923</td>" in table_html
assert "<td>1234567</td>" in table_html
assert "<td>000001</td>" in table_html
assert "<td>000123</td>" in table_html
assert "<td>007</td>" in table_html
assert "<td>0.000123</td>" in table_html
assert "<td>999999999999999999</td>" in table_html
assert "e+" not in table_html.lower()
assert "e-" not in table_html.lower()


def test_emoji_appears_with_emoji_utf8_code():
assert partition_html(text='<html charset="utf-8"><p>Hello &#128512;</p></html>') == [
Text("Hello 😀")
Expand Down
30 changes: 30 additions & 0 deletions test_unstructured/partition/test_csv.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -114,6 +114,36 @@ def test_partition_csv_from_file_with_metadata_filename():
assert elements[0].metadata.filename == "test"


def test_partition_csv_preserves_numeric_cell_text_in_table_html(tmp_path):
file_path = tmp_path / "financial-metrics.csv"
file_path.write_text(
"metric,value\n"
"revenue,478923\n"
"shares,1234567\n"
"ticker,000001\n"
"account,000123\n"
"clause,007\n"
"ratio,0.000123\n"
"market cap,999999999999999999\n",
encoding="utf-8",
)

table = partition_csv(filename=str(file_path))[0]

assert table.metadata.text_as_html == (
"<table>"
"<tr><td>metric</td><td>value</td></tr>"
"<tr><td>revenue</td><td>478923</td></tr>"
"<tr><td>shares</td><td>1234567</td></tr>"
"<tr><td>ticker</td><td>000001</td></tr>"
"<tr><td>account</td><td>000123</td></tr>"
"<tr><td>clause</td><td>007</td></tr>"
"<tr><td>ratio</td><td>0.000123</td></tr>"
"<tr><td>market cap</td><td>999999999999999999</td></tr>"
"</table>"
)


# -- .metadata.last_modified ---------------------------------------------------------------------


Expand Down
56 changes: 56 additions & 0 deletions test_unstructured/partition/test_tsv.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@

from __future__ import annotations

import io

import pytest
from pytest_mock import MockFixture

Expand Down Expand Up @@ -73,6 +75,60 @@ def test_partition_tsv_from_file_with_metadata_filename():
assert all(element.metadata.filename == "test" for element in elements)


def test_partition_tsv_preserves_numeric_cell_text_in_table_html(tmp_path):
file_path = tmp_path / "financial-metrics.tsv"
file_path.write_text(
"metric\tvalue\n"
"revenue\t478923\n"
"shares\t1234567\n"
"ticker\t000001\n"
"account\t000123\n"
"clause\t007\n"
"ratio\t0.000123\n"
"market cap\t999999999999999999\n",
encoding="utf-8",
)

table = partition_tsv(filename=str(file_path), include_header=False)[0]

assert table.metadata.text_as_html == (
"<table>"
"<tr><td>metric</td><td>value</td></tr>"
"<tr><td>revenue</td><td>478923</td></tr>"
"<tr><td>shares</td><td>1234567</td></tr>"
"<tr><td>ticker</td><td>000001</td></tr>"
"<tr><td>account</td><td>000123</td></tr>"
"<tr><td>clause</td><td>007</td></tr>"
"<tr><td>ratio</td><td>0.000123</td></tr>"
"<tr><td>market cap</td><td>999999999999999999</td></tr>"
"</table>"
)


def test_partition_tsv_from_file_preserves_numeric_cell_text_in_table_html():
file = io.BytesIO(
b"metric\tvalue\n"
b"revenue\t478923\n"
b"ticker\t000001\n"
b"account\t000123\n"
b"clause\t007\n"
b"ratio\t0.000123\n"
)

table = partition_tsv(file=file, include_header=False)[0]

assert table.metadata.text_as_html == (
"<table>"
"<tr><td>metric</td><td>value</td></tr>"
"<tr><td>revenue</td><td>478923</td></tr>"
"<tr><td>ticker</td><td>000001</td></tr>"
"<tr><td>account</td><td>000123</td></tr>"
"<tr><td>clause</td><td>007</td></tr>"
"<tr><td>ratio</td><td>0.000123</td></tr>"
"</table>"
)


# -- .metadata.last_modified ---------------------------------------------------------------------


Expand Down
16 changes: 15 additions & 1 deletion unstructured/common/html_table.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@

import html
from functools import cached_property
from typing import TYPE_CHECKING, Iterator, Sequence, cast
from typing import TYPE_CHECKING, Any, Iterator, Sequence, cast

from lxml import etree
from lxml.html import fragment_fromstring
Expand Down Expand Up @@ -48,6 +48,20 @@ def iter_tds(row_cell_strs: Sequence[str]) -> Iterator[str]:
return f"<table>{''.join(iter_trs(matrix))}</table>" if matrix else ""


def htmlify_dataframe(dataframe: Any, include_header: bool) -> str:
Comment thread
cubic-dev-ai[bot] marked this conversation as resolved.
"""Render a dataframe as table HTML without pandas numeric formatting."""
# Preserve missing cells as empty cells instead of leaking pandas null tokens like "nan".
rows = [
["" if isna else str(value) for value, isna in zip(value_row, isna_row)]
for value_row, isna_row in zip(
dataframe.to_numpy(dtype=object).tolist(),
dataframe.isna().to_numpy().tolist(),
)
]
matrix = [[str(column) for column in dataframe.columns]] + rows if include_header else rows
return htmlify_matrix_of_cell_texts(matrix)


class HtmlTable:
"""A `<table>` element."""

Expand Down
14 changes: 9 additions & 5 deletions unstructured/partition/csv.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
import pandas as pd

from unstructured.chunking import add_chunking_strategy
from unstructured.common.html_table import HtmlTable
from unstructured.common.html_table import HtmlTable, htmlify_dataframe
from unstructured.documents.elements import Element, ElementMetadata, Table
from unstructured.file_utils.model import FileType
from unstructured.partition.common.metadata import apply_metadata, get_last_modified_date
Expand Down Expand Up @@ -58,15 +58,19 @@ def partition_csv(

csv.field_size_limit(CSV_FIELD_LIMIT)
with ctx.open() as file:
read_kw: dict = {"header": ctx.header, "sep": ctx.delimiter, "encoding": ctx.encoding}
read_kw: dict = {
"header": ctx.header,
"sep": ctx.delimiter,
"encoding": ctx.encoding,
"dtype": str,
"keep_default_na": False,
}
# sep=None is not supported by the C engine; use Python engine to avoid ParserWarning.
if ctx.delimiter is None:
read_kw["engine"] = "python"
dataframe = pd.read_csv(file, **read_kw)

html_table = HtmlTable.from_html_text(
dataframe.to_html(index=False, header=include_header, na_rep="")
)
html_table = HtmlTable.from_html_text(htmlify_dataframe(dataframe, include_header))

metadata = ElementMetadata(
filename=filename,
Expand Down
16 changes: 10 additions & 6 deletions unstructured/partition/tsv.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
import pandas as pd

from unstructured.chunking import add_chunking_strategy
from unstructured.common.html_table import HtmlTable
from unstructured.common.html_table import HtmlTable, htmlify_dataframe
from unstructured.documents.elements import Element, ElementMetadata, Table
from unstructured.file_utils.model import FileType
from unstructured.partition.common.common import (
Expand Down Expand Up @@ -41,18 +41,22 @@ def partition_tsv(

header = 0 if include_header else None

read_kw: dict[str, Any] = {
"sep": "\t",
"header": header,
"dtype": str,
"keep_default_na": False,
}
if filename:
dataframe = pd.read_csv(filename, sep="\t", header=header)
dataframe = pd.read_csv(filename, **read_kw)
else:
assert file is not None
# -- Note(scanny): `SpooledTemporaryFile` on Python<3.11 does not implement `.readable()`
# -- which triggers an exception on `pd.DataFrame.read_csv()` call.
f = spooled_to_bytes_io_if_needed(file)
dataframe = pd.read_csv(f, sep="\t", header=header)
dataframe = pd.read_csv(f, **read_kw)

html_table = HtmlTable.from_html_text(
dataframe.to_html(index=False, header=include_header, na_rep="")
)
html_table = HtmlTable.from_html_text(htmlify_dataframe(dataframe, include_header))

metadata = ElementMetadata(
filename=filename,
Expand Down