feat: add language_fallback for short text and languages param to partition_md

Author: claytonlin1110

test_unstructured/partition/common/test_lang.py

@@ -196,6 +196,44 @@ def test_detect_languages_handles_spelled_out_languages():
     assert languages == ["spa"]
 
 
+def test_detect_languages_short_text_fallback_returns_none():
+    """Short ASCII text with language_fallback returning None leaves language unspecified."""
+    result = detect_languages(
+        text="Hi there.",
+        language_fallback=lambda t: None,
+    )
+    assert result is None
+
+
+def test_detect_languages_short_text_fallback_returns_custom():
+    """Short ASCII text with language_fallback returns custom language."""
+    result = detect_languages(
+        text="Bonjour monde.",
+        language_fallback=lambda t: ["fra"],
+    )
+    assert result == ["fra"]
+
+
+def test_detect_languages_short_text_default_eng_without_fallback():
+    """Short ASCII text without fallback still defaults to English (backward compat)."""
+    result = detect_languages(text="Hi there.")
+    assert result == ["eng"]
+
+
+def test_apply_lang_metadata_with_language_fallback():
+    """apply_lang_metadata passes language_fallback so short text can return None."""
+    elements = [NarrativeText("Hi.")]
+    result = list(
+        apply_lang_metadata(
+            elements=elements,
+            languages=["auto"],
+            language_fallback=lambda t: None,
+        )
+    )
+    assert len(result) == 1
+    assert result[0].metadata.languages is None
+
+
 @pytest.mark.parametrize(
     ("languages", "ocr_languages", "expected_langs"),
     [

test_unstructured/partition/test_md.py

@@ -248,6 +248,15 @@ def test_partition_md_respects_detect_language_per_element():
     assert langs == [["eng"], ["spa", "eng"], ["eng"], ["eng"], ["spa"]]
 
 
+def test_partition_md_languages_empty_disables_detection():
+    """Passing languages=[\"\"] disables language detection (no metadata.languages set)."""
+    filename = "example-docs/README.md"
+    elements = partition_md(filename=filename, languages=[""])
+    # When detection is disabled, metadata.languages should not be set (None)
+    for el in elements:
+        assert el.metadata.languages is None
+
+
 def test_partition_md_parse_table():
     filename = example_doc_path("simple-table.md")
     elements = partition_md(filename=filename)

unstructured/partition/auto.py

@@ -42,6 +42,7 @@ def partition(
     ocr_languages: Optional[str] = None,  # changing to optional for deprecation
     languages: Optional[list[str]] = None,
     detect_language_per_element: bool = False,
+    language_fallback: Optional[Callable[[str], Optional[list[str]]]] = None,
     pdf_infer_table_structure: bool = False,
     extract_images_in_pdf: bool = False,
     extract_image_block_types: Optional[list[str]] = None,
@@ -98,6 +99,10 @@ def partition(
         Additional Parameters:
             detect_language_per_element
                 Detect language per element instead of at the document level.
+            language_fallback
+                Optional callable for short text (e.g. when detection defaults to English).
+                Called with the text; return a list of ISO 639-3 codes or None to leave
+                language unspecified.
     pdf_infer_table_structure
         Deprecated! Use `skip_infer_table_types` to opt out of table extraction for any document
         type.
@@ -280,6 +285,7 @@ def augment_metadata(elements: list[Element]) -> list[Element]:
 
     partitioning_kwargs = copy.deepcopy(kwargs)
     partitioning_kwargs["detect_language_per_element"] = detect_language_per_element
+    partitioning_kwargs["language_fallback"] = language_fallback
     partitioning_kwargs["encoding"] = encoding
     partitioning_kwargs["infer_table_structure"] = infer_table_structure
     partitioning_kwargs["languages"] = languages

unstructured/partition/common/lang.py

@@ -2,7 +2,7 @@
 
 import re
 from functools import lru_cache
-from typing import Iterable, Iterator, Optional
+from typing import Callable, Iterable, Iterator, Optional
 
 import iso639  # pyright: ignore[reportMissingTypeStubs]
 from langdetect import (  # pyright: ignore[reportMissingTypeStubs]
@@ -383,10 +383,16 @@ def _get_all_tesseract_langcodes_with_prefix(prefix: str) -> list[str]:
 def detect_languages(
     text: str,
     languages: Optional[list[str]] = None,
+    language_fallback: Optional[Callable[[str], Optional[list[str]]]] = None,
 ) -> Optional[list[str]]:
     """
     Detects the list of languages present in the text (in the default "auto" mode),
     or formats and passes through the user inputted document languages if provided.
+
+    For short ASCII text (fewer than 5 words), language detection is unreliable. By default
+    such text is assigned English (["eng"]). Use ``language_fallback`` to override:
+    pass a callable that takes the text and returns a list of ISO 639-3 codes or None.
+    Return None to leave language unspecified so the user can handle it.
     """
     if languages is None:
         languages = ["auto"]
@@ -406,6 +412,9 @@ def detect_languages(
     # If text contains special characters (like ñ, å, or Korean/Mandarin/etc.) it will NOT default
     # to English. It will default to English if text is only ascii characters and is short.
     if _ASCII_RE.match(text) and len(text.split()) < 5:
+        if language_fallback is not None:
+            result = language_fallback(text)
+            return result
         logger.debug(f'short text: "{text}". Defaulting to English.')
         return ["eng"]
 
@@ -469,10 +478,12 @@ def apply_lang_metadata(
     elements: Iterable[Element],
     languages: Optional[list[str]],
     detect_language_per_element: bool = False,
+    language_fallback: Optional[Callable[[str], Optional[list[str]]]] = None,
 ) -> Iterator[Element]:
     """Detect language and apply it to metadata.languages for each element in `elements`.
     If languages is None, default to auto detection.
-    If languages is and empty string, skip."""
+    If languages is and empty string, skip.
+    language_fallback is used for short text when detection is unreliable; see detect_languages."""
     # -- Note this function has a stream interface, but reads the full `elements` stream into memory
     # -- before emitting the first updated element as output.
 
@@ -493,8 +504,13 @@ def apply_lang_metadata(
     if not isinstance(elements, list):
         elements = list(elements)
 
+    def detect(text: str) -> Optional[list[str]]:
+        return detect_languages(
+            text=text, languages=languages, language_fallback=language_fallback
+        )
+
     full_text = " ".join(str(e.text) for e in elements if hasattr(e, "text") and e.text)
-    detected_languages = detect_languages(text=full_text, languages=languages)
+    detected_languages = detect(full_text)
     if (
         detected_languages is not None
         and len(detected_languages) == 1
@@ -508,7 +524,7 @@ def apply_lang_metadata(
         for e in elements:
             if hasattr(e, "text"):
                 text_value = str(e.text) if e.text is not None else ""
-                e.metadata.languages = detect_languages(text_value)
+                e.metadata.languages = detect(text_value)
                 yield e
             else:
                 yield e

unstructured/partition/common/metadata.py

@@ -180,11 +180,13 @@ def wrapper(*args: _P.args, **kwargs: _P.kwargs) -> list[Element]:
             # -- `language` - auto-detect language (e.g. eng, spa) --
             languages = call_args.get("languages")
             detect_language_per_element = call_args.get("detect_language_per_element", False)
+            language_fallback = call_args.get("language_fallback")
             elements = list(
                 apply_lang_metadata(
                     elements=elements,
                     languages=languages,
                     detect_language_per_element=detect_language_per_element,
+                    language_fallback=language_fallback,
                 )
             )
 

unstructured/partition/md.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-from typing import IO, Any
+from typing import IO, Any, Optional
 
 import markdown
 import requests
@@ -29,6 +29,7 @@ def partition_md(
     url: str | None = None,
     metadata_filename: str | None = None,
     metadata_last_modified: str | None = None,
+    languages: Optional[list[str]] = None,
     **kwargs: Any,
 ) -> list[Element]:
     """Partitions a markdown file into its constituent elements
@@ -45,6 +46,9 @@ def partition_md(
         The URL of a webpage to parse. Only for URLs that return a markdown document.
     metadata_last_modified
         The last modified date for the document.
+    languages
+        The languages present in the document. Use ``["auto"]`` to detect (default when None).
+        Use ``[""]`` to disable language detection.
     """
     if text is None:
         text = ""
@@ -75,11 +79,14 @@ def partition_md(
 
     html = markdown.markdown(text, extensions=["tables", "fenced_code"])
 
-    return partition_html(
-        text=html,
-        metadata_filename=metadata_filename or filename,
-        metadata_file_type=FileType.MD,
-        metadata_last_modified=metadata_last_modified or last_modified,
-        detection_origin=DETECTION_ORIGIN,
+    html_kwargs: dict[str, Any] = {
+        "text": html,
+        "metadata_filename": metadata_filename or filename,
+        "metadata_file_type": FileType.MD,
+        "metadata_last_modified": metadata_last_modified or last_modified,
+        "detection_origin": DETECTION_ORIGIN,
         **kwargs,
-    )
+    }
+    if languages is not None:
+        html_kwargs["languages"] = languages
+    return partition_html(**html_kwargs)