From 07d779c560fe056c0f6e3a628502140b3eea1e02 Mon Sep 17 00:00:00 2001 From: Muhtasim-Munif-Fahim Date: Thu, 11 Jun 2026 14:16:52 +0600 Subject: [PATCH 1/3] fix pdf truncation after inline images --- packages/markitdown/README.md | 6 ++ packages/markitdown/pyproject.toml | 2 + .../markitdown/converters/_pdf_converter.py | 47 +++++++++++++- packages/markitdown/tests/test_module_misc.py | 63 +++++++++++++++++++ 4 files changed, 117 insertions(+), 1 deletion(-) diff --git a/packages/markitdown/README.md b/packages/markitdown/README.md index bedcba183..520ef7376 100644 --- a/packages/markitdown/README.md +++ b/packages/markitdown/README.md @@ -16,6 +16,12 @@ From PyPI: pip install markitdown[all] ``` +If you need the optional PyMuPDF fallback for some PDFs: + +```bash +pip install markitdown[pymupdf] +``` + From source: ```bash diff --git a/packages/markitdown/pyproject.toml b/packages/markitdown/pyproject.toml index d4c20a402..e2749b559 100644 --- a/packages/markitdown/pyproject.toml +++ b/packages/markitdown/pyproject.toml @@ -42,6 +42,7 @@ all = [ "lxml", "pdfminer.six>=20251230", "pdfplumber>=0.11.9", + "pymupdf", "olefile", "pydub", "SpeechRecognition", @@ -55,6 +56,7 @@ docx = ["mammoth~=1.11.0", "lxml"] xlsx = ["pandas", "openpyxl"] xls = ["pandas", "xlrd"] pdf = ["pdfminer.six>=20251230", "pdfplumber>=0.11.9"] +pymupdf = ["pymupdf"] outlook = ["olefile"] audio-transcription = ["pydub", "SpeechRecognition"] youtube-transcription = ["youtube-transcript-api"] diff --git a/packages/markitdown/src/markitdown/converters/_pdf_converter.py b/packages/markitdown/src/markitdown/converters/_pdf_converter.py index ffbcbd990..3bce7c1cf 100644 --- a/packages/markitdown/src/markitdown/converters/_pdf_converter.py +++ b/packages/markitdown/src/markitdown/converters/_pdf_converter.py @@ -57,6 +57,23 @@ def _merge_partial_numbering_lines(text: str) -> str: return "\n".join(result_lines) +def _extract_with_pymupdf(pdf_bytes: io.BytesIO) -> str | None: + """Extract text with PyMuPDF when the primary PDF parsers look truncated.""" + if fitz is None: + return None + + pdf_bytes.seek(0) + chunks: list[str] = [] + with fitz.open(stream=pdf_bytes.read(), filetype="pdf") as doc: + for page in doc: + text = page.get_text("text") + if text and text.strip(): + chunks.append(text.strip()) + + markdown = "\n\n".join(chunks).strip() + return markdown or None + + # Load dependencies _dependency_exc_info = None try: @@ -66,6 +83,11 @@ def _merge_partial_numbering_lines(text: str) -> str: except ImportError: _dependency_exc_info = sys.exc_info() +try: + import fitz +except ImportError: + fitz = None + ACCEPTED_MIME_TYPE_PREFIXES = [ "application/pdf", @@ -536,6 +558,9 @@ def convert( assert isinstance(file_stream, io.IOBase) + markdown_chunks: list[str] = [] + has_images = False + # Read file stream into BytesIO for compatibility with pdfplumber pdf_bytes = io.BytesIO(file_stream.read()) @@ -545,12 +570,14 @@ def convert( # pages are collected separately. page.close() is called # after each page to free pdfplumber's cached objects and # keep memory usage constant regardless of page count. - markdown_chunks: list[str] = [] form_page_count = 0 plain_page_indices: list[int] = [] with pdfplumber.open(pdf_bytes) as pdf: for page_idx, page in enumerate(pdf.pages): + has_images = has_images or bool(getattr(page, "images", None)) + + # Try form-style word position extraction page_content = _extract_form_content_from_words(page) if page_content is not None: @@ -583,6 +610,24 @@ def convert( pdf_bytes.seek(0) markdown = pdfminer.high_level.extract_text(pdf_bytes) + # Recover from inline-image truncation cases where the primary parsers + # return a much shorter body than an optional PyMuPDF pass. + if fitz is not None and has_images and markdown and len(markdown) < 2048: + try: + pymupdf_markdown = _extract_with_pymupdf(pdf_bytes) + except Exception: + pymupdf_markdown = None + else: + if pymupdf_markdown is not None: + primary_length = len(markdown.strip()) + pymupdf_length = len(pymupdf_markdown.strip()) + if pymupdf_length > primary_length and ( + primary_length == 0 + or pymupdf_length >= primary_length * 1.5 + or pymupdf_length - primary_length >= 200 + ): + markdown = pymupdf_markdown + # Post-process to merge MasterFormat-style partial numbering with following text markdown = _merge_partial_numbering_lines(markdown) diff --git a/packages/markitdown/tests/test_module_misc.py b/packages/markitdown/tests/test_module_misc.py index 4d62e4919..a40d1480d 100644 --- a/packages/markitdown/tests/test_module_misc.py +++ b/packages/markitdown/tests/test_module_misc.py @@ -6,6 +6,8 @@ import pytest from unittest.mock import MagicMock +import markitdown.converters._pdf_converter as pdf_converter_module + from markitdown._uri_utils import parse_data_uri, file_uri_to_path from markitdown import ( @@ -432,6 +434,67 @@ def test_exceptions() -> None: assert type(exc_info.value.attempts[0].converter).__name__ == "PptxConverter" +def test_pdf_converter_prefers_pymupdf_when_primary_extraction_is_truncated( + monkeypatch: pytest.MonkeyPatch, +) -> None: + class _FakePage: + images = [object()] + + def extract_words(self, keep_blank_chars=True, x_tolerance=3, y_tolerance=3): + return [] + + def extract_text(self): + return "BEFORE_IMAGE: this text should be extracted" + + class _FakePdf: + pages = [_FakePage()] + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, tb): + return False + + class _FakePyMuPdfPage: + def get_text(self, kind): + assert kind == "text" + return ( + "BEFORE_IMAGE: this text should be extracted\n" + "AFTER_IMAGE: this text should also be extracted" + ) + + class _FakePyMuPdfDoc: + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, tb): + return False + + def __iter__(self): + return iter([_FakePyMuPdfPage()]) + + monkeypatch.setattr(pdf_converter_module.pdfplumber, "open", lambda _: _FakePdf()) + monkeypatch.setattr( + pdf_converter_module.pdfminer.high_level, + "extract_text", + lambda _: "BEFORE_IMAGE: this text should be extracted", + ) + monkeypatch.setattr( + pdf_converter_module.fitz, + "open", + lambda *args, **kwargs: _FakePyMuPdfDoc(), + ) + + converter = pdf_converter_module.PdfConverter() + result = converter.convert( + io.BytesIO(b"%PDF-1.4 fake"), + StreamInfo(mimetype="application/pdf", extension=".pdf"), + ) + + assert "BEFORE_IMAGE: this text should be extracted" in result.markdown + assert "AFTER_IMAGE: this text should also be extracted" in result.markdown + + @pytest.mark.skipif( skip_exiftool, reason="do not run if exiftool is not installed", From bb17389b05b50b7a1d07c55330ce97d64e879257 Mon Sep 17 00:00:00 2001 From: Muhtasim-Munif-Fahim Date: Sat, 20 Jun 2026 04:50:52 +0600 Subject: [PATCH 2/3] fix CSV blank leading row handling --- .../markitdown/converters/_csv_converter.py | 6 ++++++ packages/markitdown/tests/test_module_misc.py | 18 ++++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/packages/markitdown/src/markitdown/converters/_csv_converter.py b/packages/markitdown/src/markitdown/converters/_csv_converter.py index 7e9631e1b..02bafd8de 100644 --- a/packages/markitdown/src/markitdown/converters/_csv_converter.py +++ b/packages/markitdown/src/markitdown/converters/_csv_converter.py @@ -51,6 +51,12 @@ def convert( reader = csv.reader(io.StringIO(content)) rows = list(reader) + if not rows: + return DocumentConverterResult(markdown="") + + while rows and not any(cell.strip() for cell in rows[0]): + rows.pop(0) + if not rows: return DocumentConverterResult(markdown="") diff --git a/packages/markitdown/tests/test_module_misc.py b/packages/markitdown/tests/test_module_misc.py index a40d1480d..3bf0aef80 100644 --- a/packages/markitdown/tests/test_module_misc.py +++ b/packages/markitdown/tests/test_module_misc.py @@ -6,6 +6,7 @@ import pytest from unittest.mock import MagicMock +from markitdown.converters._csv_converter import CsvConverter import markitdown.converters._pdf_converter as pdf_converter_module from markitdown._uri_utils import parse_data_uri, file_uri_to_path @@ -495,6 +496,23 @@ def __iter__(self): assert "AFTER_IMAGE: this text should also be extracted" in result.markdown +def test_csv_converter_skips_leading_blank_rows() -> None: + converter = CsvConverter() + result = converter.convert( + io.BytesIO(b"\nname,age\nbob,3\nalice,7\n"), + StreamInfo(mimetype="text/csv", extension=".csv", charset="utf-8"), + ) + + assert result.markdown == "\n".join( + [ + "| name | age |", + "| --- | --- |", + "| bob | 3 |", + "| alice | 7 |", + ] + ) + + @pytest.mark.skipif( skip_exiftool, reason="do not run if exiftool is not installed", From c4a4d59ffaf11adb9290c75b812dcf48532088dc Mon Sep 17 00:00:00 2001 From: Muhtasim-Munif-Fahim Date: Sun, 12 Jul 2026 04:17:45 +0600 Subject: [PATCH 3/3] fix: address review feedback on PyMuPDF fallback - Patch the module-level `fitz` name in the test instead of an attribute on it, since `fitz` is `None` when PyMuPDF isn't installed and `monkeypatch.setattr(None, "open", ...)` raises AttributeError. - Avoid buffering the PDF twice: pass a zero-copy `getbuffer()` view to fitz.open() instead of `pdf_bytes.read()`, which duplicated the whole file in memory. - Extract the magic thresholds (2048, 1.5, 200) into named module-level constants. --- .../markitdown/converters/_pdf_converter.py | 42 +++++++++++++++---- packages/markitdown/tests/test_module_misc.py | 13 +++--- 2 files changed, 41 insertions(+), 14 deletions(-) diff --git a/packages/markitdown/src/markitdown/converters/_pdf_converter.py b/packages/markitdown/src/markitdown/converters/_pdf_converter.py index 3bce7c1cf..2b8fb30e2 100644 --- a/packages/markitdown/src/markitdown/converters/_pdf_converter.py +++ b/packages/markitdown/src/markitdown/converters/_pdf_converter.py @@ -10,6 +10,17 @@ # Pattern for MasterFormat-style partial numbering (e.g., ".1", ".2", ".10") PARTIAL_NUMBERING_PATTERN = re.compile(r"^\.\d+$") +# Thresholds for the PyMuPDF inline-image-truncation recovery pass below. +# Only attempt the (relatively expensive) PyMuPDF re-extraction when the +# primary parsers' output is short enough to plausibly be truncated. +PYMUPDF_FALLBACK_MAX_PRIMARY_LENGTH = 2048 +# Prefer the PyMuPDF result when it recovers meaningfully more text than the +# primary parsers, either proportionally (50% more) or in absolute terms +# (200+ extra characters), to avoid swapping in a result that isn't a real +# improvement. +PYMUPDF_FALLBACK_MIN_LENGTH_RATIO = 1.5 +PYMUPDF_FALLBACK_MIN_LENGTH_DELTA = 200 + def _merge_partial_numbering_lines(text: str) -> str: """ @@ -63,12 +74,18 @@ def _extract_with_pymupdf(pdf_bytes: io.BytesIO) -> str | None: return None pdf_bytes.seek(0) - chunks: list[str] = [] - with fitz.open(stream=pdf_bytes.read(), filetype="pdf") as doc: - for page in doc: - text = page.get_text("text") - if text and text.strip(): - chunks.append(text.strip()) + # Use a zero-copy view into the already-buffered bytes instead of + # `pdf_bytes.read()`, which would duplicate the whole PDF in memory. + view = pdf_bytes.getbuffer() + try: + chunks: list[str] = [] + with fitz.open(stream=view, filetype="pdf") as doc: + for page in doc: + text = page.get_text("text") + if text and text.strip(): + chunks.append(text.strip()) + finally: + view.release() markdown = "\n\n".join(chunks).strip() return markdown or None @@ -612,7 +629,12 @@ def convert( # Recover from inline-image truncation cases where the primary parsers # return a much shorter body than an optional PyMuPDF pass. - if fitz is not None and has_images and markdown and len(markdown) < 2048: + if ( + fitz is not None + and has_images + and markdown + and len(markdown) < PYMUPDF_FALLBACK_MAX_PRIMARY_LENGTH + ): try: pymupdf_markdown = _extract_with_pymupdf(pdf_bytes) except Exception: @@ -623,8 +645,10 @@ def convert( pymupdf_length = len(pymupdf_markdown.strip()) if pymupdf_length > primary_length and ( primary_length == 0 - or pymupdf_length >= primary_length * 1.5 - or pymupdf_length - primary_length >= 200 + or pymupdf_length + >= primary_length * PYMUPDF_FALLBACK_MIN_LENGTH_RATIO + or pymupdf_length - primary_length + >= PYMUPDF_FALLBACK_MIN_LENGTH_DELTA ): markdown = pymupdf_markdown diff --git a/packages/markitdown/tests/test_module_misc.py b/packages/markitdown/tests/test_module_misc.py index 3bf0aef80..c28ff3592 100644 --- a/packages/markitdown/tests/test_module_misc.py +++ b/packages/markitdown/tests/test_module_misc.py @@ -474,17 +474,20 @@ def __exit__(self, exc_type, exc, tb): def __iter__(self): return iter([_FakePyMuPdfPage()]) + class _FakeFitz: + @staticmethod + def open(*args, **kwargs): + return _FakePyMuPdfDoc() + monkeypatch.setattr(pdf_converter_module.pdfplumber, "open", lambda _: _FakePdf()) monkeypatch.setattr( pdf_converter_module.pdfminer.high_level, "extract_text", lambda _: "BEFORE_IMAGE: this text should be extracted", ) - monkeypatch.setattr( - pdf_converter_module.fitz, - "open", - lambda *args, **kwargs: _FakePyMuPdfDoc(), - ) + # Patch the module-level `fitz` name itself (not an attribute on it), since + # `fitz` is `None` when PyMuPDF isn't installed and `setattr(None, ...)` fails. + monkeypatch.setattr(pdf_converter_module, "fitz", _FakeFitz) converter = pdf_converter_module.PdfConverter() result = converter.convert(