fix: update

Author: claytonlin1110

CHANGELOG.md

@@ -1,7 +1,7 @@
 ## 0.22.2
 
 ### Enhancements
-- **`partition_md` Markdown `extensions`**: Optional kwarg passed to `markdown.markdown()`; defaults to `["tables", "fenced_code"]`. Override with e.g. `extensions=["tables"]` when needed.
+- **`partition_md` Markdown `extensions` (fixes #4006)**: Optional `extensions` list is passed to `markdown.markdown()`; entries may be registered names (`str`) or `markdown.extensions.Extension` instances. Defaults to `["tables", "fenced_code"]`. Invalid values raise `ValueError`.
 
 ## 0.22.1
 

test_unstructured/partition/test_md.py

@@ -5,6 +5,7 @@
 
 import pytest
 import requests
+from markdown.extensions.fenced_code import FencedCodeExtension
 from pytest_mock import MockFixture
 
 from test_unstructured.unit_utils import assert_round_trips_through_JSON, example_doc_path
@@ -367,15 +368,27 @@ def test_partition_md_custom_extensions_parameter():
     assert elements_fenced[0].category == ElementType.CODE_SNIPPET
     assert elements_fenced[0].text == expected_body
 
+    # Extension instances (normal Python-Markdown API) match string extension names.
+    elements_instance = partition_md(text=text, extensions=[FencedCodeExtension()])
+    assert elements_instance == elements_fenced
 
-def test_partition_md_invalid_extensions_logs_and_falls_back(mocker: MockFixture):
-    """Invalid `extensions` value is ignored with a warning and falls back to the default list."""
-    text = "# Heading"
-    logger = mocker.patch("unstructured.partition.md.logging.warning")
 
-    elements = partition_md(text=text, extensions="not-a-list")  # type: ignore[arg-type]
+def test_partition_md_extensions_not_list_raises():
+    with pytest.raises(ValueError, match="'extensions' must be a list"):
+        partition_md(text="# Hi", extensions=("tables",))  # type: ignore[arg-type]
 
-    # Still parses something
-    assert len(elements) > 0
-    # Warning was logged
-    logger.assert_called_once()
+
+def test_partition_md_extensions_invalid_item_raises():
+    with pytest.raises(ValueError, match="Each entry in 'extensions'"):
+        partition_md(text="# Hi", extensions=[42])  # type: ignore[list-item]
+
+
+def test_partition_md_tables_only_differs_from_default_for_code_fence():
+    """Without ``fenced_code``, ``#`` inside a fence can become a Title (see #4006)."""
+    text = """```bash
+# line
+```"""
+    default_el = partition_md(text=text)[0]
+    tables_only_els = partition_md(text=text, extensions=["tables"])
+    assert default_el.category == ElementType.CODE_SNIPPET
+    assert any(e.category == ElementType.TITLE for e in tables_only_els)

unstructured/partition/md.py

@@ -1,10 +1,10 @@
 from __future__ import annotations
 
-import logging
 from typing import IO, Any, Optional
 
 import markdown
 import requests
+from markdown.extensions import Extension
 
 from unstructured.documents.elements import Element
 from unstructured.file_utils.encoding import read_txt_file
@@ -22,6 +22,28 @@ def optional_decode(contents: str | bytes) -> str:
 
 DETECTION_ORIGIN: str = "md"
 
+_DEFAULT_MARKDOWN_EXTENSIONS: list[str] = ["tables", "fenced_code"]
+
+
+def _validate_markdown_extensions(extensions: Any) -> list[Any]:
+    """Return ``extensions`` if it is a list of strings and/or ``Extension`` instances.
+
+    Python-Markdown accepts extension entry points as registered names (``str``) or configured
+    ``Extension`` instances; both are supported here. Any other shape raises ``ValueError``.
+    """
+    if not isinstance(extensions, list):
+        raise ValueError(
+            "'extensions' must be a list of extension names (str) and/or "
+            f"markdown.extensions.Extension instances, got {type(extensions).__name__!r}"
+        )
+    for item in extensions:
+        if not isinstance(item, (str, Extension)):
+            raise ValueError(
+                "Each entry in 'extensions' must be a str or markdown.extensions.Extension "
+                f"instance, got {type(item).__name__}: {item!r}"
+            )
+    return extensions
+
 
 def partition_md(
     filename: str | None = None,
@@ -52,9 +74,10 @@ def partition_md(
         Use ``[""]`` to disable language detection.
 
     Other keyword arguments are forwarded to ``partition_html``. In addition, ``extensions`` may be
-    passed to select Python-Markdown extensions. The default is ``["tables", "fenced_code"]``.
-    Pass e.g. ``extensions=["tables"]`` if you need the legacy behavior where ``#`` inside unfenced
-    content is parsed as a heading (see #4006).
+    passed to ``markdown.markdown()`` as a list of registered extension names (``str``) and/or
+    configured ``markdown.extensions.Extension`` instances. The default is
+    ``["tables", "fenced_code"]``. Pass e.g. ``extensions=["tables"]`` if you need the legacy
+    behavior where ``#`` inside unfenced content is parsed as a heading (see #4006).
     """
     if text is None:
         text = ""
@@ -84,13 +107,9 @@ def partition_md(
         text = response.text
 
     # -- optional markdown extensions; default matches historical partition_md behavior --
-    _default_extensions = ["tables", "fenced_code"]
-    extensions = kwargs.pop("extensions", _default_extensions)
-    if not (isinstance(extensions, list) and all(isinstance(ext, str) for ext in extensions)):
-        logging.warning(
-            "Ignoring invalid 'extensions' argument (expected list of strings): %r", extensions
-        )
-        extensions = _default_extensions
+    extensions = _validate_markdown_extensions(
+        kwargs.pop("extensions", _DEFAULT_MARKDOWN_EXTENSIONS)
+    )
 
     html = markdown.markdown(text, extensions=extensions)