diff --git a/packages/markitdown/README.md b/packages/markitdown/README.md index bedcba183..520ef7376 100644 --- a/packages/markitdown/README.md +++ b/packages/markitdown/README.md @@ -16,6 +16,12 @@ From PyPI: pip install markitdown[all] ``` +If you need the optional PyMuPDF fallback for some PDFs: + +```bash +pip install markitdown[pymupdf] +``` + From source: ```bash diff --git a/packages/markitdown/pyproject.toml b/packages/markitdown/pyproject.toml index d4c20a402..e2749b559 100644 --- a/packages/markitdown/pyproject.toml +++ b/packages/markitdown/pyproject.toml @@ -42,6 +42,7 @@ all = [ "lxml", "pdfminer.six>=20251230", "pdfplumber>=0.11.9", + "pymupdf", "olefile", "pydub", "SpeechRecognition", @@ -55,6 +56,7 @@ docx = ["mammoth~=1.11.0", "lxml"] xlsx = ["pandas", "openpyxl"] xls = ["pandas", "xlrd"] pdf = ["pdfminer.six>=20251230", "pdfplumber>=0.11.9"] +pymupdf = ["pymupdf"] outlook = ["olefile"] audio-transcription = ["pydub", "SpeechRecognition"] youtube-transcription = ["youtube-transcript-api"] diff --git a/packages/markitdown/src/markitdown/converters/_csv_converter.py b/packages/markitdown/src/markitdown/converters/_csv_converter.py index 7e9631e1b..02bafd8de 100644 --- a/packages/markitdown/src/markitdown/converters/_csv_converter.py +++ b/packages/markitdown/src/markitdown/converters/_csv_converter.py @@ -51,6 +51,12 @@ def convert( reader = csv.reader(io.StringIO(content)) rows = list(reader) + if not rows: + return DocumentConverterResult(markdown="") + + while rows and not any(cell.strip() for cell in rows[0]): + rows.pop(0) + if not rows: return DocumentConverterResult(markdown="") diff --git a/packages/markitdown/src/markitdown/converters/_pdf_converter.py b/packages/markitdown/src/markitdown/converters/_pdf_converter.py index ffbcbd990..2b8fb30e2 100644 --- a/packages/markitdown/src/markitdown/converters/_pdf_converter.py +++ b/packages/markitdown/src/markitdown/converters/_pdf_converter.py @@ -10,6 +10,17 @@ # Pattern for MasterFormat-style partial numbering (e.g., ".1", ".2", ".10") PARTIAL_NUMBERING_PATTERN = re.compile(r"^\.\d+$") +# Thresholds for the PyMuPDF inline-image-truncation recovery pass below. +# Only attempt the (relatively expensive) PyMuPDF re-extraction when the +# primary parsers' output is short enough to plausibly be truncated. +PYMUPDF_FALLBACK_MAX_PRIMARY_LENGTH = 2048 +# Prefer the PyMuPDF result when it recovers meaningfully more text than the +# primary parsers, either proportionally (50% more) or in absolute terms +# (200+ extra characters), to avoid swapping in a result that isn't a real +# improvement. +PYMUPDF_FALLBACK_MIN_LENGTH_RATIO = 1.5 +PYMUPDF_FALLBACK_MIN_LENGTH_DELTA = 200 + def _merge_partial_numbering_lines(text: str) -> str: """ @@ -57,6 +68,29 @@ def _merge_partial_numbering_lines(text: str) -> str: return "\n".join(result_lines) +def _extract_with_pymupdf(pdf_bytes: io.BytesIO) -> str | None: + """Extract text with PyMuPDF when the primary PDF parsers look truncated.""" + if fitz is None: + return None + + pdf_bytes.seek(0) + # Use a zero-copy view into the already-buffered bytes instead of + # `pdf_bytes.read()`, which would duplicate the whole PDF in memory. + view = pdf_bytes.getbuffer() + try: + chunks: list[str] = [] + with fitz.open(stream=view, filetype="pdf") as doc: + for page in doc: + text = page.get_text("text") + if text and text.strip(): + chunks.append(text.strip()) + finally: + view.release() + + markdown = "\n\n".join(chunks).strip() + return markdown or None + + # Load dependencies _dependency_exc_info = None try: @@ -66,6 +100,11 @@ def _merge_partial_numbering_lines(text: str) -> str: except ImportError: _dependency_exc_info = sys.exc_info() +try: + import fitz +except ImportError: + fitz = None + ACCEPTED_MIME_TYPE_PREFIXES = [ "application/pdf", @@ -536,6 +575,9 @@ def convert( assert isinstance(file_stream, io.IOBase) + markdown_chunks: list[str] = [] + has_images = False + # Read file stream into BytesIO for compatibility with pdfplumber pdf_bytes = io.BytesIO(file_stream.read()) @@ -545,12 +587,14 @@ def convert( # pages are collected separately. page.close() is called # after each page to free pdfplumber's cached objects and # keep memory usage constant regardless of page count. - markdown_chunks: list[str] = [] form_page_count = 0 plain_page_indices: list[int] = [] with pdfplumber.open(pdf_bytes) as pdf: for page_idx, page in enumerate(pdf.pages): + has_images = has_images or bool(getattr(page, "images", None)) + + # Try form-style word position extraction page_content = _extract_form_content_from_words(page) if page_content is not None: @@ -583,6 +627,31 @@ def convert( pdf_bytes.seek(0) markdown = pdfminer.high_level.extract_text(pdf_bytes) + # Recover from inline-image truncation cases where the primary parsers + # return a much shorter body than an optional PyMuPDF pass. + if ( + fitz is not None + and has_images + and markdown + and len(markdown) < PYMUPDF_FALLBACK_MAX_PRIMARY_LENGTH + ): + try: + pymupdf_markdown = _extract_with_pymupdf(pdf_bytes) + except Exception: + pymupdf_markdown = None + else: + if pymupdf_markdown is not None: + primary_length = len(markdown.strip()) + pymupdf_length = len(pymupdf_markdown.strip()) + if pymupdf_length > primary_length and ( + primary_length == 0 + or pymupdf_length + >= primary_length * PYMUPDF_FALLBACK_MIN_LENGTH_RATIO + or pymupdf_length - primary_length + >= PYMUPDF_FALLBACK_MIN_LENGTH_DELTA + ): + markdown = pymupdf_markdown + # Post-process to merge MasterFormat-style partial numbering with following text markdown = _merge_partial_numbering_lines(markdown) diff --git a/packages/markitdown/tests/test_module_misc.py b/packages/markitdown/tests/test_module_misc.py index 4d62e4919..c28ff3592 100644 --- a/packages/markitdown/tests/test_module_misc.py +++ b/packages/markitdown/tests/test_module_misc.py @@ -6,6 +6,9 @@ import pytest from unittest.mock import MagicMock +from markitdown.converters._csv_converter import CsvConverter +import markitdown.converters._pdf_converter as pdf_converter_module + from markitdown._uri_utils import parse_data_uri, file_uri_to_path from markitdown import ( @@ -432,6 +435,87 @@ def test_exceptions() -> None: assert type(exc_info.value.attempts[0].converter).__name__ == "PptxConverter" +def test_pdf_converter_prefers_pymupdf_when_primary_extraction_is_truncated( + monkeypatch: pytest.MonkeyPatch, +) -> None: + class _FakePage: + images = [object()] + + def extract_words(self, keep_blank_chars=True, x_tolerance=3, y_tolerance=3): + return [] + + def extract_text(self): + return "BEFORE_IMAGE: this text should be extracted" + + class _FakePdf: + pages = [_FakePage()] + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, tb): + return False + + class _FakePyMuPdfPage: + def get_text(self, kind): + assert kind == "text" + return ( + "BEFORE_IMAGE: this text should be extracted\n" + "AFTER_IMAGE: this text should also be extracted" + ) + + class _FakePyMuPdfDoc: + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, tb): + return False + + def __iter__(self): + return iter([_FakePyMuPdfPage()]) + + class _FakeFitz: + @staticmethod + def open(*args, **kwargs): + return _FakePyMuPdfDoc() + + monkeypatch.setattr(pdf_converter_module.pdfplumber, "open", lambda _: _FakePdf()) + monkeypatch.setattr( + pdf_converter_module.pdfminer.high_level, + "extract_text", + lambda _: "BEFORE_IMAGE: this text should be extracted", + ) + # Patch the module-level `fitz` name itself (not an attribute on it), since + # `fitz` is `None` when PyMuPDF isn't installed and `setattr(None, ...)` fails. + monkeypatch.setattr(pdf_converter_module, "fitz", _FakeFitz) + + converter = pdf_converter_module.PdfConverter() + result = converter.convert( + io.BytesIO(b"%PDF-1.4 fake"), + StreamInfo(mimetype="application/pdf", extension=".pdf"), + ) + + assert "BEFORE_IMAGE: this text should be extracted" in result.markdown + assert "AFTER_IMAGE: this text should also be extracted" in result.markdown + + +def test_csv_converter_skips_leading_blank_rows() -> None: + converter = CsvConverter() + result = converter.convert( + io.BytesIO(b"\nname,age\nbob,3\nalice,7\n"), + StreamInfo(mimetype="text/csv", extension=".csv", charset="utf-8"), + ) + + assert result.markdown == "\n".join( + [ + "| name | age |", + "| --- | --- |", + "| bob | 3 |", + "| alice | 7 |", + ] + ) + + @pytest.mark.skipif( skip_exiftool, reason="do not run if exiftool is not installed",