From dbb811ca91f194d178e91ce0d9d6e36bea16ebbb Mon Sep 17 00:00:00 2001 From: jeonsworld Date: Fri, 23 May 2025 16:24:30 +0900 Subject: [PATCH 1/5] feat: add page-level text extraction for PDF documents - Add PageInfo class to store page number and content - Enhance DocumentConverterResult with optional pages attribute - Extend PdfConverter with extract_pages parameter for page-by-page processing - Add CLI support with --extract-pages and --pages-json flags - Implement robust error handling with fallback to full document extraction - Maintain 100% backward compatibility with existing API - Add comprehensive test suite with 8 test cases covering all scenarios --- .../markitdown/src/markitdown/__init__.py | 3 +- .../markitdown/src/markitdown/__main__.py | 64 ++++++- .../src/markitdown/_base_converter.py | 22 ++- .../markitdown/converters/_pdf_converter.py | 69 ++++++- .../tests/test_pdf_page_extraction.py | 171 ++++++++++++++++++ 5 files changed, 321 insertions(+), 8 deletions(-) create mode 100644 packages/markitdown/tests/test_pdf_page_extraction.py diff --git a/packages/markitdown/src/markitdown/__init__.py b/packages/markitdown/src/markitdown/__init__.py index af356dd63..980d8f2a2 100644 --- a/packages/markitdown/src/markitdown/__init__.py +++ b/packages/markitdown/src/markitdown/__init__.py @@ -8,7 +8,7 @@ PRIORITY_SPECIFIC_FILE_FORMAT, PRIORITY_GENERIC_FILE_FORMAT, ) -from ._base_converter import DocumentConverterResult, DocumentConverter +from ._base_converter import DocumentConverterResult, DocumentConverter, PageInfo from ._stream_info import StreamInfo from ._exceptions import ( MarkItDownException, @@ -23,6 +23,7 @@ "MarkItDown", "DocumentConverter", "DocumentConverterResult", + "PageInfo", "MarkItDownException", "MissingDependencyException", "FailedConversionAttempt", diff --git a/packages/markitdown/src/markitdown/__main__.py b/packages/markitdown/src/markitdown/__main__.py index 6085ad6bb..885f39288 100644 --- a/packages/markitdown/src/markitdown/__main__.py +++ b/packages/markitdown/src/markitdown/__main__.py @@ -8,6 +8,8 @@ from importlib.metadata import entry_points from .__about__ import __version__ from ._markitdown import MarkItDown, StreamInfo, DocumentConverterResult +from ._base_converter import PageInfo +import json def main(): @@ -41,6 +43,14 @@ def main(): OR markitdown example.pdf > example.md + + OR to extract pages separately from PDF + + markitdown example.pdf --extract-pages + + OR to get page information as JSON + + markitdown example.pdf --extract-pages --pages-json """ ).strip(), ) @@ -110,6 +120,18 @@ def main(): help="Keep data URIs (like base64-encoded images) in the output. By default, data URIs are truncated.", ) + parser.add_argument( + "--extract-pages", + action="store_true", + help="Extract pages separately for PDF files. Returns page-by-page information.", + ) + + parser.add_argument( + "--pages-json", + action="store_true", + help="Output page information as JSON when using --extract-pages.", + ) + parser.add_argument("filename", nargs="?") args = parser.parse_args() @@ -191,10 +213,14 @@ def main(): sys.stdin.buffer, stream_info=stream_info, keep_data_uris=args.keep_data_uris, + extract_pages=args.extract_pages, ) else: result = markitdown.convert( - args.filename, stream_info=stream_info, keep_data_uris=args.keep_data_uris + args.filename, + stream_info=stream_info, + keep_data_uris=args.keep_data_uris, + extract_pages=args.extract_pages, ) _handle_output(args, result) @@ -202,13 +228,45 @@ def main(): def _handle_output(args, result: DocumentConverterResult): """Handle output to stdout or file""" + output_content = "" + + if args.extract_pages and result.pages and args.pages_json: + # Output as JSON with page information + pages_data = [ + { + "page_number": page.page_number, + "content": page.content + } + for page in result.pages + ] + output_data = { + "markdown": result.markdown, + "title": result.title, + "pages": pages_data + } + output_content = json.dumps(output_data, ensure_ascii=False, indent=2) + elif args.extract_pages and result.pages: + # Output with page separators + output_content = result.markdown + output_content += "\n\n" + "=" * 50 + "\n" + output_content += f"EXTRACTED {len(result.pages)} PAGES:\n" + output_content += "=" * 50 + "\n\n" + + for page in result.pages: + output_content += f"--- PAGE {page.page_number} ---\n" + output_content += page.content + output_content += "\n\n" + else: + # Standard output + output_content = result.markdown + if args.output: with open(args.output, "w", encoding="utf-8") as f: - f.write(result.markdown) + f.write(output_content) else: # Handle stdout encoding errors more gracefully print( - result.markdown.encode(sys.stdout.encoding, errors="replace").decode( + output_content.encode(sys.stdout.encoding, errors="replace").decode( sys.stdout.encoding ) ) diff --git a/packages/markitdown/src/markitdown/_base_converter.py b/packages/markitdown/src/markitdown/_base_converter.py index fa2b11145..bbb197bdc 100644 --- a/packages/markitdown/src/markitdown/_base_converter.py +++ b/packages/markitdown/src/markitdown/_base_converter.py @@ -1,7 +1,22 @@ -from typing import Any, BinaryIO, Optional +from typing import Any, BinaryIO, Optional, List from ._stream_info import StreamInfo +class PageInfo: + """Information about a specific page in a document.""" + + def __init__(self, page_number: int, content: str): + """ + Initialize page information. + + Parameters: + - page_number: The page number (1-indexed) + - content: The markdown content of the page + """ + self.page_number = page_number + self.content = content + + class DocumentConverterResult: """The result of converting a document to Markdown.""" @@ -10,19 +25,22 @@ def __init__( markdown: str, *, title: Optional[str] = None, + pages: Optional[List[PageInfo]] = None, ): """ Initialize the DocumentConverterResult. The only required parameter is the converted Markdown text. - The title, and any other metadata that may be added in the future, are optional. + The title, pages, and any other metadata that may be added in the future, are optional. Parameters: - markdown: The converted Markdown text. - title: Optional title of the document. + - pages: Optional list of page information for documents with page structure. """ self.markdown = markdown self.title = title + self.pages = pages @property def text_content(self) -> str: diff --git a/packages/markitdown/src/markitdown/converters/_pdf_converter.py b/packages/markitdown/src/markitdown/converters/_pdf_converter.py index ffbcbd990..24d03b19f 100644 --- a/packages/markitdown/src/markitdown/converters/_pdf_converter.py +++ b/packages/markitdown/src/markitdown/converters/_pdf_converter.py @@ -1,9 +1,9 @@ import sys import io import re -from typing import BinaryIO, Any +from typing import BinaryIO, Any, List -from .._base_converter import DocumentConverter, DocumentConverterResult +from .._base_converter import DocumentConverter, DocumentConverterResult, PageInfo from .._stream_info import StreamInfo from .._exceptions import MissingDependencyException, MISSING_DEPENDENCY_MESSAGE @@ -62,6 +62,10 @@ def _merge_partial_numbering_lines(text: str) -> str: try: import pdfminer import pdfminer.high_level + from pdfminer.pdfpage import PDFPage + from pdfminer.pdfinterp import PDFResourceManager, PDFPageInterpreter + from pdfminer.converter import TextConverter + from pdfminer.layout import LAParams import pdfplumber except ImportError: _dependency_exc_info = sys.exc_info() @@ -536,6 +540,19 @@ def convert( assert isinstance(file_stream, io.IOBase) + # Check if page-level extraction is requested + extract_pages = kwargs.get("extract_pages", False) + + if extract_pages: + # Extract text page by page + pages = self._extract_pages(file_stream) + # Combine all pages for the main markdown content + markdown = "\n\n".join([page.content for page in pages]) + return DocumentConverterResult( + markdown=markdown, + pages=pages, + ) + # Read file stream into BytesIO for compatibility with pdfplumber pdf_bytes = io.BytesIO(file_stream.read()) @@ -587,3 +604,51 @@ def convert( markdown = _merge_partial_numbering_lines(markdown) return DocumentConverterResult(markdown=markdown) + + def _extract_pages(self, file_stream: BinaryIO) -> List[PageInfo]: + """Extract text from each page separately.""" + pages = [] + + try: + # Reset stream position + file_stream.seek(0) + + # Create resource manager + rsrcmgr = PDFResourceManager() + laparams = LAParams() + + # Get all pages + pdf_pages = list(PDFPage.get_pages(file_stream)) + + for page_num, page in enumerate(pdf_pages, 1): + output_string = None + device = None + try: + # Create a new StringIO for each page + output_string = io.StringIO() + device = TextConverter(rsrcmgr, output_string, laparams=laparams) + interpreter = PDFPageInterpreter(rsrcmgr, device) + + # Process the page + interpreter.process_page(page) + + # Get the text content + text = output_string.getvalue() + + # Add page info + pages.append(PageInfo(page_number=page_num, content=text.strip())) + + finally: + # Clean up resources + if device: + device.close() + if output_string: + output_string.close() + + except Exception: + # If page-by-page extraction fails, fall back to full document extraction + file_stream.seek(0) + full_text = pdfminer.high_level.extract_text(file_stream) + pages = [PageInfo(page_number=1, content=full_text.strip())] + + return pages diff --git a/packages/markitdown/tests/test_pdf_page_extraction.py b/packages/markitdown/tests/test_pdf_page_extraction.py new file mode 100644 index 000000000..7b5d95b36 --- /dev/null +++ b/packages/markitdown/tests/test_pdf_page_extraction.py @@ -0,0 +1,171 @@ +#!/usr/bin/env python3 -m pytest +""" +Unit tests for PDF page extraction functionality. +""" + +import os +import tempfile +import pytest +from typing import Optional + +from markitdown import MarkItDown, PageInfo, DocumentConverterResult + + +class TestPdfPageExtraction: + """Test cases for PDF page extraction functionality.""" + + @pytest.fixture(autouse=True) + def setup(self): + """Set up test fixtures.""" + self.markitdown = MarkItDown() + self.test_pdf_path = os.path.join( + os.path.dirname(__file__), + 'test_files', + 'test.pdf' + ) + + def test_traditional_pdf_conversion(self): + """Test that traditional PDF conversion works unchanged.""" + if not os.path.exists(self.test_pdf_path): + pytest.skip("Test PDF file not found") + + result = self.markitdown.convert(self.test_pdf_path) + + # Verify result structure + assert isinstance(result, DocumentConverterResult) + assert isinstance(result.markdown, str) + assert len(result.markdown) > 0 + assert result.pages is None # Should be None by default + + # Verify backward compatibility + assert hasattr(result, 'text_content') + assert result.text_content == result.markdown + assert str(result) == result.markdown + + def test_pdf_page_extraction_enabled(self): + """Test PDF conversion with page extraction enabled.""" + if not os.path.exists(self.test_pdf_path): + pytest.skip("Test PDF file not found") + + result = self.markitdown.convert(self.test_pdf_path, extract_pages=True) + + # Verify result structure + assert isinstance(result, DocumentConverterResult) + assert isinstance(result.markdown, str) + assert len(result.markdown) > 0 + assert result.pages is not None + assert isinstance(result.pages, list) + assert len(result.pages) > 0 + + # Verify page structure + for page in result.pages: + assert isinstance(page, PageInfo) + assert isinstance(page.page_number, int) + assert page.page_number > 0 + assert isinstance(page.content, str) + assert len(page.content) > 0 + + # Verify page numbers are sequential + page_numbers = [page.page_number for page in result.pages] + assert page_numbers == list(range(1, len(result.pages) + 1)) + + def test_pdf_page_extraction_disabled(self): + """Test PDF conversion with page extraction explicitly disabled.""" + if not os.path.exists(self.test_pdf_path): + pytest.skip("Test PDF file not found") + + result = self.markitdown.convert(self.test_pdf_path, extract_pages=False) + + # Should behave the same as default + assert isinstance(result, DocumentConverterResult) + assert isinstance(result.markdown, str) + assert len(result.markdown) > 0 + assert result.pages is None + + def test_page_info_class(self): + """Test PageInfo class functionality.""" + page = PageInfo(page_number=1, content="Test content") + + assert page.page_number == 1 + assert page.content == "Test content" + assert isinstance(page.page_number, int) + assert isinstance(page.content, str) + + def test_document_converter_result_with_pages(self): + """Test DocumentConverterResult with pages parameter.""" + pages = [ + PageInfo(page_number=1, content="Page 1 content"), + PageInfo(page_number=2, content="Page 2 content") + ] + + result = DocumentConverterResult( + markdown="Combined content", + title="Test Document", + pages=pages + ) + + assert result.markdown == "Combined content" + assert result.title == "Test Document" + assert len(result.pages) == 2 + assert result.pages[0].page_number == 1 + assert result.pages[1].page_number == 2 + + def test_backward_compatibility(self): + """Test that all existing functionality remains intact.""" + if not os.path.exists(self.test_pdf_path): + pytest.skip("Test PDF file not found") + + # Test different ways of calling convert + result1 = self.markitdown.convert(self.test_pdf_path) + result2 = self.markitdown.convert(self.test_pdf_path, extract_pages=False) + + # Results should be equivalent + assert result1.markdown == result2.markdown + assert result1.title == result2.title + assert result1.pages == result2.pages + + # Both should work with string conversion + assert str(result1) == str(result2) + + # Both should work with text_content property + assert result1.text_content == result2.text_content + + def test_non_pdf_file_with_extract_pages(self): + """Test that extract_pages parameter doesn't affect non-PDF files.""" + # Create a temporary markdown file + with tempfile.NamedTemporaryFile(mode='w', suffix='.md', delete=False) as f: + f.write("# Test\n\nThis is a test.") + temp_path = f.name + + try: + result1 = self.markitdown.convert(temp_path) + result2 = self.markitdown.convert(temp_path, extract_pages=True) + + # Both should behave the same for non-PDF files + assert result1.markdown == result2.markdown + assert result1.pages is None + assert result2.pages is None + + finally: + os.unlink(temp_path) + + def test_extract_pages_parameter_types(self): + """Test different types for extract_pages parameter.""" + if not os.path.exists(self.test_pdf_path): + pytest.skip("Test PDF file not found") + + # Test with different truthy/falsy values + result_false = self.markitdown.convert(self.test_pdf_path, extract_pages=False) + result_true = self.markitdown.convert(self.test_pdf_path, extract_pages=True) + result_none = self.markitdown.convert(self.test_pdf_path, extract_pages=None) + result_zero = self.markitdown.convert(self.test_pdf_path, extract_pages=0) + result_one = self.markitdown.convert(self.test_pdf_path, extract_pages=1) + + # False, None, 0 should not extract pages + assert result_false.pages is None + assert result_none.pages is None + assert result_zero.pages is None + + # True, 1 should extract pages + assert result_true.pages is not None + assert result_one.pages is not None \ No newline at end of file From fd6aa685064abfc4d9affbe3eb5d30bbe4dc6436 Mon Sep 17 00:00:00 2001 From: jeonsworld Date: Fri, 23 May 2025 17:34:48 +0900 Subject: [PATCH 2/5] feat: extend page extraction to PPTX and DOCX converters - Add slide-level extraction for PPTX files with extract_pages parameter - Each slide is treated as a PageInfo object with sequential numbering - Add extract_pages parameter to DOCX for API consistency (returns None due to dynamic pagination) - Import PageInfo class in both converters to support the new functionality - Add comprehensive test suites for both formats ensuring backward compatibility - Maintain 100% backward compatibility with existing API --- .../markitdown/converters/_docx_converter.py | 23 +++- .../markitdown/converters/_pptx_converter.py | 41 ++++-- .../tests/test_docx_page_extraction.py | 124 +++++++++++++++++ .../tests/test_pptx_page_extraction.py | 128 ++++++++++++++++++ 4 files changed, 296 insertions(+), 20 deletions(-) create mode 100644 packages/markitdown/tests/test_docx_page_extraction.py create mode 100644 packages/markitdown/tests/test_pptx_page_extraction.py diff --git a/packages/markitdown/src/markitdown/converters/_docx_converter.py b/packages/markitdown/src/markitdown/converters/_docx_converter.py index 3975107b1..1ddd04fa4 100644 --- a/packages/markitdown/src/markitdown/converters/_docx_converter.py +++ b/packages/markitdown/src/markitdown/converters/_docx_converter.py @@ -6,7 +6,7 @@ from ._html_converter import HtmlConverter from ..converter_utils.docx.pre_process import pre_process_docx -from .._base_converter import DocumentConverterResult +from .._base_converter import DocumentConverterResult, PageInfo from .._stream_info import StreamInfo from .._exceptions import MissingDependencyException, MISSING_DEPENDENCY_MESSAGE @@ -75,9 +75,22 @@ def convert( _dependency_exc_info[2] ) + # Check if page extraction is requested + extract_pages = kwargs.get("extract_pages", False) + style_map = kwargs.get("style_map", None) pre_process_stream = pre_process_docx(file_stream) - return self._html_converter.convert_string( - mammoth.convert_to_html(pre_process_stream, style_map=style_map).value, - **kwargs, - ) + + # Convert to HTML + html_result = mammoth.convert_to_html(pre_process_stream, style_map=style_map) + + # Convert HTML to markdown + result = self._html_converter.convert_string(html_result.value, **kwargs) + + # Note: DOCX files don't have fixed pages like PDFs. + # Page breaks depend on rendering settings (margins, font size, etc.) + # For now, we'll return None for pages even if extract_pages is True + # This maintains API compatibility while acknowledging the limitation + pages = None + + return DocumentConverterResult(markdown=result.markdown, title=result.title, pages=pages) diff --git a/packages/markitdown/src/markitdown/converters/_pptx_converter.py b/packages/markitdown/src/markitdown/converters/_pptx_converter.py index 360f17706..c39fc9d2d 100644 --- a/packages/markitdown/src/markitdown/converters/_pptx_converter.py +++ b/packages/markitdown/src/markitdown/converters/_pptx_converter.py @@ -10,7 +10,7 @@ from ._html_converter import HtmlConverter from ._llm_caption import llm_caption -from .._base_converter import DocumentConverter, DocumentConverterResult +from .._base_converter import DocumentConverter, DocumentConverterResult, PageInfo from .._stream_info import StreamInfo from .._exceptions import MissingDependencyException, MISSING_DEPENDENCY_MESSAGE @@ -78,19 +78,23 @@ def convert( _dependency_exc_info[2] ) + # Check if page extraction is requested + extract_pages = kwargs.get("extract_pages", False) + # Perform the conversion presentation = pptx.Presentation(file_stream) md_content = "" + pages = [] if extract_pages else None slide_num = 0 for slide in presentation.slides: slide_num += 1 - md_content += f"\n\n\n" + slide_content = f"\n\n\n" title = slide.shapes.title def get_shape_content(shape, **kwargs): - nonlocal md_content + nonlocal slide_content # Pictures if self._is_picture(shape): # https://github.com/scanny/python-pptx/pull/512#issuecomment-1713100069 @@ -145,26 +149,26 @@ def get_shape_content(shape, **kwargs): blob = shape.image.blob content_type = shape.image.content_type or "image/png" b64_string = base64.b64encode(blob).decode("utf-8") - md_content += f"\n![{alt_text}](data:{content_type};base64,{b64_string})\n" + slide_content += f"\n![{alt_text}](data:{content_type};base64,{b64_string})\n" else: # A placeholder name filename = re.sub(r"\W", "", shape.name) + ".jpg" - md_content += "\n![" + alt_text + "](" + filename + ")\n" + slide_content += "\n![" + alt_text + "](" + filename + ")\n" # Tables if self._is_table(shape): - md_content += self._convert_table_to_markdown(shape.table, **kwargs) + slide_content += self._convert_table_to_markdown(shape.table, **kwargs) # Charts if shape.has_chart: - md_content += self._convert_chart_to_markdown(shape.chart) + slide_content += self._convert_chart_to_markdown(shape.chart) # Text areas elif shape.has_text_frame: if shape == title: - md_content += "# " + shape.text.lstrip() + "\n" + slide_content += "# " + shape.text.lstrip() + "\n" else: - md_content += shape.text + "\n" + slide_content += shape.text + "\n" # Group Shapes if shape.shape_type == pptx.enum.shapes.MSO_SHAPE_TYPE.GROUP: @@ -188,16 +192,23 @@ def get_shape_content(shape, **kwargs): for shape in sorted_shapes: get_shape_content(shape, **kwargs) - md_content = md_content.strip() + slide_content = slide_content.strip() if slide.has_notes_slide: - md_content += "\n\n### Notes:\n" + slide_content += "\n\n### Notes:\n" notes_frame = slide.notes_slide.notes_text_frame if notes_frame is not None: - md_content += notes_frame.text - md_content = md_content.strip() - - return DocumentConverterResult(markdown=md_content.strip()) + slide_content += notes_frame.text + slide_content = slide_content.strip() + + # Add to overall content + md_content += slide_content + + # If extracting pages, add to pages list + if extract_pages: + pages.append(PageInfo(page_number=slide_num, content=slide_content.strip())) + + return DocumentConverterResult(markdown=md_content.strip(), pages=pages) def _is_picture(self, shape): if shape.shape_type == pptx.enum.shapes.MSO_SHAPE_TYPE.PICTURE: diff --git a/packages/markitdown/tests/test_docx_page_extraction.py b/packages/markitdown/tests/test_docx_page_extraction.py new file mode 100644 index 000000000..69ae2bb5d --- /dev/null +++ b/packages/markitdown/tests/test_docx_page_extraction.py @@ -0,0 +1,124 @@ +#!/usr/bin/env python3 -m pytest +""" +Unit tests for DOCX page extraction functionality. +""" + +import os +import tempfile +import pytest +from typing import Optional + +from markitdown import MarkItDown, PageInfo, DocumentConverterResult + + +class TestDocxPageExtraction: + """Test cases for DOCX page extraction functionality.""" + + @pytest.fixture(autouse=True) + def setup(self): + """Set up test fixtures.""" + self.markitdown = MarkItDown() + self.test_docx_path = os.path.join( + os.path.dirname(__file__), + 'test_files', + 'test.docx' + ) + + def test_traditional_docx_conversion(self): + """Test that traditional DOCX conversion works unchanged.""" + if not os.path.exists(self.test_docx_path): + pytest.skip("Test DOCX file not found") + + result = self.markitdown.convert(self.test_docx_path) + + # Verify result structure + assert isinstance(result, DocumentConverterResult) + assert isinstance(result.markdown, str) + assert len(result.markdown) > 0 + assert result.pages is None # Should be None by default + + # Verify backward compatibility + assert hasattr(result, 'text_content') + assert result.text_content == result.markdown + assert str(result) == result.markdown + + def test_docx_page_extraction_enabled(self): + """Test DOCX conversion with page extraction enabled. + + Note: DOCX files don't have fixed pages like PDFs. + Page breaks depend on rendering settings. Currently, + this returns None for pages even with extract_pages=True. + """ + if not os.path.exists(self.test_docx_path): + pytest.skip("Test DOCX file not found") + + result = self.markitdown.convert(self.test_docx_path, extract_pages=True) + + # Verify result structure + assert isinstance(result, DocumentConverterResult) + assert isinstance(result.markdown, str) + assert len(result.markdown) > 0 + + # Currently, DOCX doesn't support page extraction due to dynamic pagination + assert result.pages is None + + def test_docx_page_extraction_disabled(self): + """Test DOCX conversion with page extraction explicitly disabled.""" + if not os.path.exists(self.test_docx_path): + pytest.skip("Test DOCX file not found") + + result = self.markitdown.convert(self.test_docx_path, extract_pages=False) + + # Should behave the same as default + assert isinstance(result, DocumentConverterResult) + assert isinstance(result.markdown, str) + assert len(result.markdown) > 0 + assert result.pages is None + + def test_backward_compatibility(self): + """Test that all existing functionality remains intact.""" + if not os.path.exists(self.test_docx_path): + pytest.skip("Test DOCX file not found") + + # Test different ways of calling convert + result1 = self.markitdown.convert(self.test_docx_path) + result2 = self.markitdown.convert(self.test_docx_path, extract_pages=False) + result3 = self.markitdown.convert(self.test_docx_path, extract_pages=True) + + # Results should be equivalent for markdown content + assert result1.markdown == result2.markdown + assert result1.markdown == result3.markdown + assert result1.title == result2.title + assert result1.title == result3.title + + # All should have None for pages (DOCX limitation) + assert result1.pages is None + assert result2.pages is None + assert result3.pages is None + + # All should work with string conversion + assert str(result1) == str(result2) + assert str(result1) == str(result3) + + # All should work with text_content property + assert result1.text_content == result2.text_content + assert result1.text_content == result3.text_content + + def test_extract_pages_parameter_types(self): + """Test different types for extract_pages parameter.""" + if not os.path.exists(self.test_docx_path): + pytest.skip("Test DOCX file not found") + + # Test with different truthy/falsy values + result_false = self.markitdown.convert(self.test_docx_path, extract_pages=False) + result_true = self.markitdown.convert(self.test_docx_path, extract_pages=True) + result_none = self.markitdown.convert(self.test_docx_path, extract_pages=None) + result_zero = self.markitdown.convert(self.test_docx_path, extract_pages=0) + result_one = self.markitdown.convert(self.test_docx_path, extract_pages=1) + + # All should return None for pages (DOCX limitation) + assert result_false.pages is None + assert result_true.pages is None + assert result_none.pages is None + assert result_zero.pages is None + assert result_one.pages is None \ No newline at end of file diff --git a/packages/markitdown/tests/test_pptx_page_extraction.py b/packages/markitdown/tests/test_pptx_page_extraction.py new file mode 100644 index 000000000..e3c6ea52d --- /dev/null +++ b/packages/markitdown/tests/test_pptx_page_extraction.py @@ -0,0 +1,128 @@ +#!/usr/bin/env python3 -m pytest +""" +Unit tests for PPTX slide extraction functionality. +""" + +import os +import tempfile +import pytest +from typing import Optional + +from markitdown import MarkItDown, PageInfo, DocumentConverterResult + + +class TestPptxPageExtraction: + """Test cases for PPTX slide extraction functionality.""" + + @pytest.fixture(autouse=True) + def setup(self): + """Set up test fixtures.""" + self.markitdown = MarkItDown() + self.test_pptx_path = os.path.join( + os.path.dirname(__file__), + 'test_files', + 'test.pptx' + ) + + def test_traditional_pptx_conversion(self): + """Test that traditional PPTX conversion works unchanged.""" + if not os.path.exists(self.test_pptx_path): + pytest.skip("Test PPTX file not found") + + result = self.markitdown.convert(self.test_pptx_path) + + # Verify result structure + assert isinstance(result, DocumentConverterResult) + assert isinstance(result.markdown, str) + assert len(result.markdown) > 0 + assert result.pages is None # Should be None by default + + # Verify backward compatibility + assert hasattr(result, 'text_content') + assert result.text_content == result.markdown + assert str(result) == result.markdown + + def test_pptx_page_extraction_enabled(self): + """Test PPTX conversion with slide extraction enabled.""" + if not os.path.exists(self.test_pptx_path): + pytest.skip("Test PPTX file not found") + + result = self.markitdown.convert(self.test_pptx_path, extract_pages=True) + + # Verify result structure + assert isinstance(result, DocumentConverterResult) + assert isinstance(result.markdown, str) + assert len(result.markdown) > 0 + assert result.pages is not None + assert isinstance(result.pages, list) + assert len(result.pages) > 0 + + # Verify page structure + for page in result.pages: + assert isinstance(page, PageInfo) + assert isinstance(page.page_number, int) + assert page.page_number > 0 + assert isinstance(page.content, str) + # Slides can be empty, so we don't check content length + + # Verify page numbers are sequential + page_numbers = [page.page_number for page in result.pages] + assert page_numbers == list(range(1, len(result.pages) + 1)) + + # Verify each slide has slide number comment + for page in result.pages: + assert f"" in page.content + + def test_pptx_page_extraction_disabled(self): + """Test PPTX conversion with slide extraction explicitly disabled.""" + if not os.path.exists(self.test_pptx_path): + pytest.skip("Test PPTX file not found") + + result = self.markitdown.convert(self.test_pptx_path, extract_pages=False) + + # Should behave the same as default + assert isinstance(result, DocumentConverterResult) + assert isinstance(result.markdown, str) + assert len(result.markdown) > 0 + assert result.pages is None + + def test_backward_compatibility(self): + """Test that all existing functionality remains intact.""" + if not os.path.exists(self.test_pptx_path): + pytest.skip("Test PPTX file not found") + + # Test different ways of calling convert + result1 = self.markitdown.convert(self.test_pptx_path) + result2 = self.markitdown.convert(self.test_pptx_path, extract_pages=False) + + # Results should be equivalent + assert result1.markdown == result2.markdown + assert result1.title == result2.title + assert result1.pages == result2.pages + + # Both should work with string conversion + assert str(result1) == str(result2) + + # Both should work with text_content property + assert result1.text_content == result2.text_content + + def test_extract_pages_parameter_types(self): + """Test different types for extract_pages parameter.""" + if not os.path.exists(self.test_pptx_path): + pytest.skip("Test PPTX file not found") + + # Test with different truthy/falsy values + result_false = self.markitdown.convert(self.test_pptx_path, extract_pages=False) + result_true = self.markitdown.convert(self.test_pptx_path, extract_pages=True) + result_none = self.markitdown.convert(self.test_pptx_path, extract_pages=None) + result_zero = self.markitdown.convert(self.test_pptx_path, extract_pages=0) + result_one = self.markitdown.convert(self.test_pptx_path, extract_pages=1) + + # False, None, 0 should not extract pages + assert result_false.pages is None + assert result_none.pages is None + assert result_zero.pages is None + + # True, 1 should extract pages + assert result_true.pages is not None + assert result_one.pages is not None \ No newline at end of file From 41addf9a595f7f3f97143ccd454c04a49901647b Mon Sep 17 00:00:00 2001 From: jeonsworld Date: Sat, 24 May 2025 13:34:36 +0900 Subject: [PATCH 3/5] style: apply Black formatting to fix pre-commit checks - Format all Python files with Black (v23.7.0) - Fix line length and formatting issues in page extraction feature files - Ensure consistent code style across the codebase --- .../markitdown/src/markitdown/__main__.py | 17 ++-- .../src/markitdown/_base_converter.py | 4 +- .../markitdown/src/markitdown/_stream_info.py | 6 +- .../markitdown/converters/_docx_converter.py | 14 ++-- .../markitdown/converters/_pptx_converter.py | 14 ++-- .../tests/test_docx_page_extraction.py | 54 +++++++------ .../tests/test_pdf_page_extraction.py | 78 +++++++++---------- .../tests/test_pptx_page_extraction.py | 54 +++++++------ 8 files changed, 118 insertions(+), 123 deletions(-) diff --git a/packages/markitdown/src/markitdown/__main__.py b/packages/markitdown/src/markitdown/__main__.py index 885f39288..ef50bd48b 100644 --- a/packages/markitdown/src/markitdown/__main__.py +++ b/packages/markitdown/src/markitdown/__main__.py @@ -217,8 +217,8 @@ def main(): ) else: result = markitdown.convert( - args.filename, - stream_info=stream_info, + args.filename, + stream_info=stream_info, keep_data_uris=args.keep_data_uris, extract_pages=args.extract_pages, ) @@ -229,20 +229,17 @@ def main(): def _handle_output(args, result: DocumentConverterResult): """Handle output to stdout or file""" output_content = "" - + if args.extract_pages and result.pages and args.pages_json: # Output as JSON with page information pages_data = [ - { - "page_number": page.page_number, - "content": page.content - } + {"page_number": page.page_number, "content": page.content} for page in result.pages ] output_data = { "markdown": result.markdown, "title": result.title, - "pages": pages_data + "pages": pages_data, } output_content = json.dumps(output_data, ensure_ascii=False, indent=2) elif args.extract_pages and result.pages: @@ -251,7 +248,7 @@ def _handle_output(args, result: DocumentConverterResult): output_content += "\n\n" + "=" * 50 + "\n" output_content += f"EXTRACTED {len(result.pages)} PAGES:\n" output_content += "=" * 50 + "\n\n" - + for page in result.pages: output_content += f"--- PAGE {page.page_number} ---\n" output_content += page.content @@ -259,7 +256,7 @@ def _handle_output(args, result: DocumentConverterResult): else: # Standard output output_content = result.markdown - + if args.output: with open(args.output, "w", encoding="utf-8") as f: f.write(output_content) diff --git a/packages/markitdown/src/markitdown/_base_converter.py b/packages/markitdown/src/markitdown/_base_converter.py index bbb197bdc..15b068423 100644 --- a/packages/markitdown/src/markitdown/_base_converter.py +++ b/packages/markitdown/src/markitdown/_base_converter.py @@ -4,11 +4,11 @@ class PageInfo: """Information about a specific page in a document.""" - + def __init__(self, page_number: int, content: str): """ Initialize page information. - + Parameters: - page_number: The page number (1-indexed) - content: The markdown content of the page diff --git a/packages/markitdown/src/markitdown/_stream_info.py b/packages/markitdown/src/markitdown/_stream_info.py index 84a1f6403..9c38c81e0 100644 --- a/packages/markitdown/src/markitdown/_stream_info.py +++ b/packages/markitdown/src/markitdown/_stream_info.py @@ -11,9 +11,9 @@ class StreamInfo: mimetype: Optional[str] = None extension: Optional[str] = None charset: Optional[str] = None - filename: Optional[ - str - ] = None # From local path, url, or Content-Disposition header + filename: Optional[str] = ( + None # From local path, url, or Content-Disposition header + ) local_path: Optional[str] = None # If read from disk url: Optional[str] = None # If read from url diff --git a/packages/markitdown/src/markitdown/converters/_docx_converter.py b/packages/markitdown/src/markitdown/converters/_docx_converter.py index 1ddd04fa4..9e45e1550 100644 --- a/packages/markitdown/src/markitdown/converters/_docx_converter.py +++ b/packages/markitdown/src/markitdown/converters/_docx_converter.py @@ -77,20 +77,22 @@ def convert( # Check if page extraction is requested extract_pages = kwargs.get("extract_pages", False) - + style_map = kwargs.get("style_map", None) pre_process_stream = pre_process_docx(file_stream) - + # Convert to HTML html_result = mammoth.convert_to_html(pre_process_stream, style_map=style_map) - + # Convert HTML to markdown result = self._html_converter.convert_string(html_result.value, **kwargs) - + # Note: DOCX files don't have fixed pages like PDFs. # Page breaks depend on rendering settings (margins, font size, etc.) # For now, we'll return None for pages even if extract_pages is True # This maintains API compatibility while acknowledging the limitation pages = None - - return DocumentConverterResult(markdown=result.markdown, title=result.title, pages=pages) + + return DocumentConverterResult( + markdown=result.markdown, title=result.title, pages=pages + ) diff --git a/packages/markitdown/src/markitdown/converters/_pptx_converter.py b/packages/markitdown/src/markitdown/converters/_pptx_converter.py index c39fc9d2d..2909f1ffb 100644 --- a/packages/markitdown/src/markitdown/converters/_pptx_converter.py +++ b/packages/markitdown/src/markitdown/converters/_pptx_converter.py @@ -80,7 +80,7 @@ def convert( # Check if page extraction is requested extract_pages = kwargs.get("extract_pages", False) - + # Perform the conversion presentation = pptx.Presentation(file_stream) md_content = "" @@ -157,7 +157,9 @@ def get_shape_content(shape, **kwargs): # Tables if self._is_table(shape): - slide_content += self._convert_table_to_markdown(shape.table, **kwargs) + slide_content += self._convert_table_to_markdown( + shape.table, **kwargs + ) # Charts if shape.has_chart: @@ -200,13 +202,15 @@ def get_shape_content(shape, **kwargs): if notes_frame is not None: slide_content += notes_frame.text slide_content = slide_content.strip() - + # Add to overall content md_content += slide_content - + # If extracting pages, add to pages list if extract_pages: - pages.append(PageInfo(page_number=slide_num, content=slide_content.strip())) + pages.append( + PageInfo(page_number=slide_num, content=slide_content.strip()) + ) return DocumentConverterResult(markdown=md_content.strip(), pages=pages) diff --git a/packages/markitdown/tests/test_docx_page_extraction.py b/packages/markitdown/tests/test_docx_page_extraction.py index 69ae2bb5d..c728f4b04 100644 --- a/packages/markitdown/tests/test_docx_page_extraction.py +++ b/packages/markitdown/tests/test_docx_page_extraction.py @@ -13,112 +13,110 @@ class TestDocxPageExtraction: """Test cases for DOCX page extraction functionality.""" - + @pytest.fixture(autouse=True) def setup(self): """Set up test fixtures.""" self.markitdown = MarkItDown() self.test_docx_path = os.path.join( - os.path.dirname(__file__), - 'test_files', - 'test.docx' + os.path.dirname(__file__), "test_files", "test.docx" ) - + def test_traditional_docx_conversion(self): """Test that traditional DOCX conversion works unchanged.""" if not os.path.exists(self.test_docx_path): pytest.skip("Test DOCX file not found") - + result = self.markitdown.convert(self.test_docx_path) - + # Verify result structure assert isinstance(result, DocumentConverterResult) assert isinstance(result.markdown, str) assert len(result.markdown) > 0 assert result.pages is None # Should be None by default - + # Verify backward compatibility - assert hasattr(result, 'text_content') + assert hasattr(result, "text_content") assert result.text_content == result.markdown assert str(result) == result.markdown - + def test_docx_page_extraction_enabled(self): """Test DOCX conversion with page extraction enabled. - - Note: DOCX files don't have fixed pages like PDFs. + + Note: DOCX files don't have fixed pages like PDFs. Page breaks depend on rendering settings. Currently, this returns None for pages even with extract_pages=True. """ if not os.path.exists(self.test_docx_path): pytest.skip("Test DOCX file not found") - + result = self.markitdown.convert(self.test_docx_path, extract_pages=True) - + # Verify result structure assert isinstance(result, DocumentConverterResult) assert isinstance(result.markdown, str) assert len(result.markdown) > 0 - + # Currently, DOCX doesn't support page extraction due to dynamic pagination assert result.pages is None - + def test_docx_page_extraction_disabled(self): """Test DOCX conversion with page extraction explicitly disabled.""" if not os.path.exists(self.test_docx_path): pytest.skip("Test DOCX file not found") - + result = self.markitdown.convert(self.test_docx_path, extract_pages=False) - + # Should behave the same as default assert isinstance(result, DocumentConverterResult) assert isinstance(result.markdown, str) assert len(result.markdown) > 0 assert result.pages is None - + def test_backward_compatibility(self): """Test that all existing functionality remains intact.""" if not os.path.exists(self.test_docx_path): pytest.skip("Test DOCX file not found") - + # Test different ways of calling convert result1 = self.markitdown.convert(self.test_docx_path) result2 = self.markitdown.convert(self.test_docx_path, extract_pages=False) result3 = self.markitdown.convert(self.test_docx_path, extract_pages=True) - + # Results should be equivalent for markdown content assert result1.markdown == result2.markdown assert result1.markdown == result3.markdown assert result1.title == result2.title assert result1.title == result3.title - + # All should have None for pages (DOCX limitation) assert result1.pages is None assert result2.pages is None assert result3.pages is None - + # All should work with string conversion assert str(result1) == str(result2) assert str(result1) == str(result3) - + # All should work with text_content property assert result1.text_content == result2.text_content assert result1.text_content == result3.text_content - + def test_extract_pages_parameter_types(self): """Test different types for extract_pages parameter.""" if not os.path.exists(self.test_docx_path): pytest.skip("Test DOCX file not found") - + # Test with different truthy/falsy values result_false = self.markitdown.convert(self.test_docx_path, extract_pages=False) result_true = self.markitdown.convert(self.test_docx_path, extract_pages=True) result_none = self.markitdown.convert(self.test_docx_path, extract_pages=None) result_zero = self.markitdown.convert(self.test_docx_path, extract_pages=0) result_one = self.markitdown.convert(self.test_docx_path, extract_pages=1) - + # All should return None for pages (DOCX limitation) assert result_false.pages is None assert result_true.pages is None assert result_none.pages is None assert result_zero.pages is None - assert result_one.pages is None \ No newline at end of file + assert result_one.pages is None diff --git a/packages/markitdown/tests/test_pdf_page_extraction.py b/packages/markitdown/tests/test_pdf_page_extraction.py index 7b5d95b36..f93d1b6b5 100644 --- a/packages/markitdown/tests/test_pdf_page_extraction.py +++ b/packages/markitdown/tests/test_pdf_page_extraction.py @@ -13,42 +13,40 @@ class TestPdfPageExtraction: """Test cases for PDF page extraction functionality.""" - + @pytest.fixture(autouse=True) def setup(self): """Set up test fixtures.""" self.markitdown = MarkItDown() self.test_pdf_path = os.path.join( - os.path.dirname(__file__), - 'test_files', - 'test.pdf' + os.path.dirname(__file__), "test_files", "test.pdf" ) - + def test_traditional_pdf_conversion(self): """Test that traditional PDF conversion works unchanged.""" if not os.path.exists(self.test_pdf_path): pytest.skip("Test PDF file not found") - + result = self.markitdown.convert(self.test_pdf_path) - + # Verify result structure assert isinstance(result, DocumentConverterResult) assert isinstance(result.markdown, str) assert len(result.markdown) > 0 assert result.pages is None # Should be None by default - + # Verify backward compatibility - assert hasattr(result, 'text_content') + assert hasattr(result, "text_content") assert result.text_content == result.markdown assert str(result) == result.markdown - + def test_pdf_page_extraction_enabled(self): """Test PDF conversion with page extraction enabled.""" if not os.path.exists(self.test_pdf_path): pytest.skip("Test PDF file not found") - + result = self.markitdown.convert(self.test_pdf_path, extract_pages=True) - + # Verify result structure assert isinstance(result, DocumentConverterResult) assert isinstance(result.markdown, str) @@ -56,7 +54,7 @@ def test_pdf_page_extraction_enabled(self): assert result.pages is not None assert isinstance(result.pages, list) assert len(result.pages) > 0 - + # Verify page structure for page in result.pages: assert isinstance(page, PageInfo) @@ -64,108 +62,106 @@ def test_pdf_page_extraction_enabled(self): assert page.page_number > 0 assert isinstance(page.content, str) assert len(page.content) > 0 - + # Verify page numbers are sequential page_numbers = [page.page_number for page in result.pages] assert page_numbers == list(range(1, len(result.pages) + 1)) - + def test_pdf_page_extraction_disabled(self): """Test PDF conversion with page extraction explicitly disabled.""" if not os.path.exists(self.test_pdf_path): pytest.skip("Test PDF file not found") - + result = self.markitdown.convert(self.test_pdf_path, extract_pages=False) - + # Should behave the same as default assert isinstance(result, DocumentConverterResult) assert isinstance(result.markdown, str) assert len(result.markdown) > 0 assert result.pages is None - + def test_page_info_class(self): """Test PageInfo class functionality.""" page = PageInfo(page_number=1, content="Test content") - + assert page.page_number == 1 assert page.content == "Test content" assert isinstance(page.page_number, int) assert isinstance(page.content, str) - + def test_document_converter_result_with_pages(self): """Test DocumentConverterResult with pages parameter.""" pages = [ PageInfo(page_number=1, content="Page 1 content"), - PageInfo(page_number=2, content="Page 2 content") + PageInfo(page_number=2, content="Page 2 content"), ] - + result = DocumentConverterResult( - markdown="Combined content", - title="Test Document", - pages=pages + markdown="Combined content", title="Test Document", pages=pages ) - + assert result.markdown == "Combined content" assert result.title == "Test Document" assert len(result.pages) == 2 assert result.pages[0].page_number == 1 assert result.pages[1].page_number == 2 - + def test_backward_compatibility(self): """Test that all existing functionality remains intact.""" if not os.path.exists(self.test_pdf_path): pytest.skip("Test PDF file not found") - + # Test different ways of calling convert result1 = self.markitdown.convert(self.test_pdf_path) result2 = self.markitdown.convert(self.test_pdf_path, extract_pages=False) - + # Results should be equivalent assert result1.markdown == result2.markdown assert result1.title == result2.title assert result1.pages == result2.pages - + # Both should work with string conversion assert str(result1) == str(result2) - + # Both should work with text_content property assert result1.text_content == result2.text_content - + def test_non_pdf_file_with_extract_pages(self): """Test that extract_pages parameter doesn't affect non-PDF files.""" # Create a temporary markdown file - with tempfile.NamedTemporaryFile(mode='w', suffix='.md', delete=False) as f: + with tempfile.NamedTemporaryFile(mode="w", suffix=".md", delete=False) as f: f.write("# Test\n\nThis is a test.") temp_path = f.name - + try: result1 = self.markitdown.convert(temp_path) result2 = self.markitdown.convert(temp_path, extract_pages=True) - + # Both should behave the same for non-PDF files assert result1.markdown == result2.markdown assert result1.pages is None assert result2.pages is None - + finally: os.unlink(temp_path) - + def test_extract_pages_parameter_types(self): """Test different types for extract_pages parameter.""" if not os.path.exists(self.test_pdf_path): pytest.skip("Test PDF file not found") - + # Test with different truthy/falsy values result_false = self.markitdown.convert(self.test_pdf_path, extract_pages=False) result_true = self.markitdown.convert(self.test_pdf_path, extract_pages=True) result_none = self.markitdown.convert(self.test_pdf_path, extract_pages=None) result_zero = self.markitdown.convert(self.test_pdf_path, extract_pages=0) result_one = self.markitdown.convert(self.test_pdf_path, extract_pages=1) - + # False, None, 0 should not extract pages assert result_false.pages is None assert result_none.pages is None assert result_zero.pages is None - + # True, 1 should extract pages assert result_true.pages is not None - assert result_one.pages is not None \ No newline at end of file + assert result_one.pages is not None diff --git a/packages/markitdown/tests/test_pptx_page_extraction.py b/packages/markitdown/tests/test_pptx_page_extraction.py index e3c6ea52d..82c6f4d93 100644 --- a/packages/markitdown/tests/test_pptx_page_extraction.py +++ b/packages/markitdown/tests/test_pptx_page_extraction.py @@ -13,42 +13,40 @@ class TestPptxPageExtraction: """Test cases for PPTX slide extraction functionality.""" - + @pytest.fixture(autouse=True) def setup(self): """Set up test fixtures.""" self.markitdown = MarkItDown() self.test_pptx_path = os.path.join( - os.path.dirname(__file__), - 'test_files', - 'test.pptx' + os.path.dirname(__file__), "test_files", "test.pptx" ) - + def test_traditional_pptx_conversion(self): """Test that traditional PPTX conversion works unchanged.""" if not os.path.exists(self.test_pptx_path): pytest.skip("Test PPTX file not found") - + result = self.markitdown.convert(self.test_pptx_path) - + # Verify result structure assert isinstance(result, DocumentConverterResult) assert isinstance(result.markdown, str) assert len(result.markdown) > 0 assert result.pages is None # Should be None by default - + # Verify backward compatibility - assert hasattr(result, 'text_content') + assert hasattr(result, "text_content") assert result.text_content == result.markdown assert str(result) == result.markdown - + def test_pptx_page_extraction_enabled(self): """Test PPTX conversion with slide extraction enabled.""" if not os.path.exists(self.test_pptx_path): pytest.skip("Test PPTX file not found") - + result = self.markitdown.convert(self.test_pptx_path, extract_pages=True) - + # Verify result structure assert isinstance(result, DocumentConverterResult) assert isinstance(result.markdown, str) @@ -56,7 +54,7 @@ def test_pptx_page_extraction_enabled(self): assert result.pages is not None assert isinstance(result.pages, list) assert len(result.pages) > 0 - + # Verify page structure for page in result.pages: assert isinstance(page, PageInfo) @@ -64,65 +62,65 @@ def test_pptx_page_extraction_enabled(self): assert page.page_number > 0 assert isinstance(page.content, str) # Slides can be empty, so we don't check content length - + # Verify page numbers are sequential page_numbers = [page.page_number for page in result.pages] assert page_numbers == list(range(1, len(result.pages) + 1)) - + # Verify each slide has slide number comment for page in result.pages: assert f"" in page.content - + def test_pptx_page_extraction_disabled(self): """Test PPTX conversion with slide extraction explicitly disabled.""" if not os.path.exists(self.test_pptx_path): pytest.skip("Test PPTX file not found") - + result = self.markitdown.convert(self.test_pptx_path, extract_pages=False) - + # Should behave the same as default assert isinstance(result, DocumentConverterResult) assert isinstance(result.markdown, str) assert len(result.markdown) > 0 assert result.pages is None - + def test_backward_compatibility(self): """Test that all existing functionality remains intact.""" if not os.path.exists(self.test_pptx_path): pytest.skip("Test PPTX file not found") - + # Test different ways of calling convert result1 = self.markitdown.convert(self.test_pptx_path) result2 = self.markitdown.convert(self.test_pptx_path, extract_pages=False) - + # Results should be equivalent assert result1.markdown == result2.markdown assert result1.title == result2.title assert result1.pages == result2.pages - + # Both should work with string conversion assert str(result1) == str(result2) - + # Both should work with text_content property assert result1.text_content == result2.text_content - + def test_extract_pages_parameter_types(self): """Test different types for extract_pages parameter.""" if not os.path.exists(self.test_pptx_path): pytest.skip("Test PPTX file not found") - + # Test with different truthy/falsy values result_false = self.markitdown.convert(self.test_pptx_path, extract_pages=False) result_true = self.markitdown.convert(self.test_pptx_path, extract_pages=True) result_none = self.markitdown.convert(self.test_pptx_path, extract_pages=None) result_zero = self.markitdown.convert(self.test_pptx_path, extract_pages=0) result_one = self.markitdown.convert(self.test_pptx_path, extract_pages=1) - + # False, None, 0 should not extract pages assert result_false.pages is None assert result_none.pages is None assert result_zero.pages is None - + # True, 1 should extract pages assert result_true.pages is not None - assert result_one.pages is not None \ No newline at end of file + assert result_one.pages is not None From d9df5d03c8c902a8196c6c5d144dc67690b3026b Mon Sep 17 00:00:00 2001 From: Adam Fourney Date: Tue, 27 May 2025 15:27:02 -0700 Subject: [PATCH 4/5] Fixed formatting. --- packages/markitdown/src/markitdown/_stream_info.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/markitdown/src/markitdown/_stream_info.py b/packages/markitdown/src/markitdown/_stream_info.py index 9c38c81e0..84a1f6403 100644 --- a/packages/markitdown/src/markitdown/_stream_info.py +++ b/packages/markitdown/src/markitdown/_stream_info.py @@ -11,9 +11,9 @@ class StreamInfo: mimetype: Optional[str] = None extension: Optional[str] = None charset: Optional[str] = None - filename: Optional[str] = ( - None # From local path, url, or Content-Disposition header - ) + filename: Optional[ + str + ] = None # From local path, url, or Content-Disposition header local_path: Optional[str] = None # If read from disk url: Optional[str] = None # If read from url From 494619363a6156f09714d746c75726a01b458e93 Mon Sep 17 00:00:00 2001 From: jeonsworld Date: Mon, 23 Mar 2026 13:26:45 +0900 Subject: [PATCH 5/5] feat: add extract_pages support to OCR-enhanced PDF and PPTX converters --- .../markitdown_ocr/_pdf_converter_with_ocr.py | 26 +++++++++++++++++-- .../_pptx_converter_with_ocr.py | 15 +++++++++-- 2 files changed, 37 insertions(+), 4 deletions(-) diff --git a/packages/markitdown-ocr/src/markitdown_ocr/_pdf_converter_with_ocr.py b/packages/markitdown-ocr/src/markitdown_ocr/_pdf_converter_with_ocr.py index c1dc0f613..42386552e 100644 --- a/packages/markitdown-ocr/src/markitdown_ocr/_pdf_converter_with_ocr.py +++ b/packages/markitdown-ocr/src/markitdown_ocr/_pdf_converter_with_ocr.py @@ -7,7 +7,7 @@ import sys from typing import Any, BinaryIO, Optional -from markitdown import DocumentConverter, DocumentConverterResult, StreamInfo +from markitdown import DocumentConverter, DocumentConverterResult, PageInfo, StreamInfo from markitdown._exceptions import ( MissingDependencyException, MISSING_DEPENDENCY_MESSAGE, @@ -177,15 +177,20 @@ def convert( kwargs.get("ocr_service") or self.ocr_service ) + # Check if page-level extraction is requested + extract_pages = kwargs.get("extract_pages", False) + # Read PDF into BytesIO file_stream.seek(0) pdf_bytes = io.BytesIO(file_stream.read()) markdown_content = [] + page_contents: list[str] = [] if extract_pages else [] try: with pdfplumber.open(pdf_bytes) as pdf: for page_num, page in enumerate(pdf.pages, 1): + page_parts: list[str] = [] markdown_content.append(f"\n## Page {page_num}\n") # If OCR is enabled, interleave text and images by position @@ -269,22 +274,31 @@ def convert( for item in content_items: if item["type"] == "text": markdown_content.append(item["text"]) + page_parts.append(item["text"]) else: # image ocr_text = item["ocr_text"] img_marker = ( f"\n\n*[Image OCR]\n{ocr_text}\n[End OCR]*\n" ) markdown_content.append(img_marker) + page_parts.append(img_marker) else: # No images detected - just extract regular text text_content = page.extract_text() or "" if text_content.strip(): markdown_content.append(text_content.strip()) + page_parts.append(text_content.strip()) else: # No OCR, just extract text text_content = page.extract_text() or "" if text_content.strip(): markdown_content.append(text_content.strip()) + page_parts.append(text_content.strip()) + + if extract_pages: + page_contents.append( + "\n\n".join(page_parts).strip() + ) # Build final markdown markdown = "\n\n".join(markdown_content).strip() @@ -308,7 +322,15 @@ def convert( pdf_bytes.seek(0) markdown = self._ocr_full_pages(pdf_bytes, ocr_service) - return DocumentConverterResult(markdown=markdown) + # Build pages list if requested + pages = None + if extract_pages and page_contents: + pages = [ + PageInfo(page_number=i + 1, content=content) + for i, content in enumerate(page_contents) + ] + + return DocumentConverterResult(markdown=markdown, pages=pages) def _extract_page_images(self, pdf_bytes: io.BytesIO, page_num: int) -> list[dict]: """ diff --git a/packages/markitdown-ocr/src/markitdown_ocr/_pptx_converter_with_ocr.py b/packages/markitdown-ocr/src/markitdown_ocr/_pptx_converter_with_ocr.py index 7e91ed6b4..1732167af 100644 --- a/packages/markitdown-ocr/src/markitdown_ocr/_pptx_converter_with_ocr.py +++ b/packages/markitdown-ocr/src/markitdown_ocr/_pptx_converter_with_ocr.py @@ -10,7 +10,7 @@ from typing import BinaryIO, Any, Optional from markitdown.converters import HtmlConverter -from markitdown import DocumentConverter, DocumentConverterResult, StreamInfo +from markitdown import DocumentConverter, DocumentConverterResult, PageInfo, StreamInfo from markitdown._exceptions import ( MissingDependencyException, MISSING_DEPENDENCY_MESSAGE, @@ -74,13 +74,18 @@ def convert( ) llm_client = kwargs.get("llm_client") + # Check if page-level extraction is requested + extract_pages = kwargs.get("extract_pages", False) + presentation = pptx.Presentation(file_stream) md_content = "" + pages = [] if extract_pages else None slide_num = 0 for slide in presentation.slides: slide_num += 1 md_content += f"\\n\\n\\n" + slide_content_start = len(md_content) title = slide.shapes.title @@ -183,7 +188,13 @@ def get_shape_content(shape, **kwargs): md_content += notes_frame.text md_content = md_content.strip() - return DocumentConverterResult(markdown=md_content.strip()) + if extract_pages: + slide_content = md_content[slide_content_start:] + pages.append( + PageInfo(page_number=slide_num, content=slide_content.strip()) + ) + + return DocumentConverterResult(markdown=md_content.strip(), pages=pages) def _is_picture(self, shape): if shape.shape_type == pptx.enum.shapes.MSO_SHAPE_TYPE.PICTURE: