Add page-level text extraction for PDF/PPTX/DOCX documents

packages/markitdown-ocr/src/markitdown_ocr/_pdf_converter_with_ocr.py

@@ -7,7 +7,7 @@
 import sys
 from typing import Any, BinaryIO, Optional
 
-from markitdown import DocumentConverter, DocumentConverterResult, StreamInfo
+from markitdown import DocumentConverter, DocumentConverterResult, PageInfo, StreamInfo
 from markitdown._exceptions import (
     MissingDependencyException,
     MISSING_DEPENDENCY_MESSAGE,
@@ -177,15 +177,20 @@ def convert(
             kwargs.get("ocr_service") or self.ocr_service
         )
 
+        # Check if page-level extraction is requested
+        extract_pages = kwargs.get("extract_pages", False)
+
         # Read PDF into BytesIO
         file_stream.seek(0)
         pdf_bytes = io.BytesIO(file_stream.read())
 
         markdown_content = []
+        page_contents: list[str] = [] if extract_pages else []
 
         try:
             with pdfplumber.open(pdf_bytes) as pdf:
                 for page_num, page in enumerate(pdf.pages, 1):
+                    page_parts: list[str] = []
                     markdown_content.append(f"\n## Page {page_num}\n")
 
                     # If OCR is enabled, interleave text and images by position
@@ -269,22 +274,31 @@ def convert(
                             for item in content_items:
                                 if item["type"] == "text":
                                     markdown_content.append(item["text"])
+                                    page_parts.append(item["text"])
                                 else:  # image
                                     ocr_text = item["ocr_text"]
                                     img_marker = (
                                         f"\n\n*[Image OCR]\n{ocr_text}\n[End OCR]*\n"
                                     )
                                     markdown_content.append(img_marker)
+                                    page_parts.append(img_marker)
                         else:
                             # No images detected - just extract regular text
                             text_content = page.extract_text() or ""
                             if text_content.strip():
                                 markdown_content.append(text_content.strip())
+                                page_parts.append(text_content.strip())
                     else:
                         # No OCR, just extract text
                         text_content = page.extract_text() or ""
                         if text_content.strip():
                             markdown_content.append(text_content.strip())
+                            page_parts.append(text_content.strip())
+
+                    if extract_pages:
+                        page_contents.append(
+                            "\n\n".join(page_parts).strip()
+                        )
 
                 # Build final markdown
                 markdown = "\n\n".join(markdown_content).strip()
@@ -308,7 +322,15 @@ def convert(
             pdf_bytes.seek(0)
             markdown = self._ocr_full_pages(pdf_bytes, ocr_service)
 
-        return DocumentConverterResult(markdown=markdown)
+        # Build pages list if requested
+        pages = None
+        if extract_pages and page_contents:
+            pages = [
+                PageInfo(page_number=i + 1, content=content)
+                for i, content in enumerate(page_contents)
+            ]
+
+        return DocumentConverterResult(markdown=markdown, pages=pages)
 
     def _extract_page_images(self, pdf_bytes: io.BytesIO, page_num: int) -> list[dict]:
         """

packages/markitdown-ocr/src/markitdown_ocr/_pptx_converter_with_ocr.py

@@ -10,7 +10,7 @@
 from typing import BinaryIO, Any, Optional
 
 from markitdown.converters import HtmlConverter
-from markitdown import DocumentConverter, DocumentConverterResult, StreamInfo
+from markitdown import DocumentConverter, DocumentConverterResult, PageInfo, StreamInfo
 from markitdown._exceptions import (
     MissingDependencyException,
     MISSING_DEPENDENCY_MESSAGE,
@@ -74,13 +74,18 @@ def convert(
         )
         llm_client = kwargs.get("llm_client")
 
+        # Check if page-level extraction is requested
+        extract_pages = kwargs.get("extract_pages", False)
+
         presentation = pptx.Presentation(file_stream)
         md_content = ""
+        pages = [] if extract_pages else None
         slide_num = 0
 
         for slide in presentation.slides:
             slide_num += 1
             md_content += f"\\n\\n<!-- Slide number: {slide_num} -->\\n"
+            slide_content_start = len(md_content)
 
             title = slide.shapes.title
 
@@ -183,7 +188,13 @@ def get_shape_content(shape, **kwargs):
                     md_content += notes_frame.text
                 md_content = md_content.strip()
 
-        return DocumentConverterResult(markdown=md_content.strip())
+            if extract_pages:
+                slide_content = md_content[slide_content_start:]
+                pages.append(
+                    PageInfo(page_number=slide_num, content=slide_content.strip())
+                )
+
+        return DocumentConverterResult(markdown=md_content.strip(), pages=pages)
 
     def _is_picture(self, shape):
         if shape.shape_type == pptx.enum.shapes.MSO_SHAPE_TYPE.PICTURE:

packages/markitdown/src/markitdown/__init__.py

@@ -8,7 +8,7 @@
     PRIORITY_SPECIFIC_FILE_FORMAT,
     PRIORITY_GENERIC_FILE_FORMAT,
 )
-from ._base_converter import DocumentConverterResult, DocumentConverter
+from ._base_converter import DocumentConverterResult, DocumentConverter, PageInfo
 from ._stream_info import StreamInfo
 from ._exceptions import (
     MarkItDownException,
@@ -23,6 +23,7 @@
     "MarkItDown",
     "DocumentConverter",
     "DocumentConverterResult",
+    "PageInfo",
     "MarkItDownException",
     "MissingDependencyException",
     "FailedConversionAttempt",

packages/markitdown/src/markitdown/__main__.py

@@ -9,6 +9,8 @@
 from importlib.metadata import entry_points
 from .__about__ import __version__
 from ._markitdown import MarkItDown, StreamInfo, DocumentConverterResult
+from ._base_converter import PageInfo
+import json
 
 
 def main():
@@ -42,6 +44,14 @@ def main():
                 OR
 
                 markitdown example.pdf > example.md
+
+                OR to extract pages separately from PDF
+
+                markitdown example.pdf --extract-pages
+
+                OR to get page information as JSON
+
+                markitdown example.pdf --extract-pages --pages-json
             """
         ).strip(),
     )
@@ -138,6 +148,18 @@ def main():
         help="Keep data URIs (like base64-encoded images) in the output. By default, data URIs are truncated.",
     )
 
+    parser.add_argument(
+        "--extract-pages",
+        action="store_true",
+        help="Extract pages separately for PDF files. Returns page-by-page information.",
+    )
+
+    parser.add_argument(
+        "--pages-json",
+        action="store_true",
+        help="Output page information as JSON when using --extract-pages.",
+    )
+
     parser.add_argument("filename", nargs="?")
     args = parser.parse_args()
 
@@ -249,24 +271,57 @@ def main():
             sys.stdin.buffer,
             stream_info=stream_info,
             keep_data_uris=args.keep_data_uris,
+            extract_pages=args.extract_pages,
         )
     else:
         result = markitdown.convert(
-            args.filename, stream_info=stream_info, keep_data_uris=args.keep_data_uris
+            args.filename,
+            stream_info=stream_info,
+            keep_data_uris=args.keep_data_uris,
+            extract_pages=args.extract_pages,
         )
 
     _handle_output(args, result)
 
 
 def _handle_output(args, result: DocumentConverterResult):
     """Handle output to stdout or file"""
+    output_content = ""
+
+    if args.extract_pages and result.pages and args.pages_json:
+        # Output as JSON with page information
+        pages_data = [
+            {"page_number": page.page_number, "content": page.content}
+            for page in result.pages
+        ]
+        output_data = {
+            "markdown": result.markdown,
+            "title": result.title,
+            "pages": pages_data,
+        }
+        output_content = json.dumps(output_data, ensure_ascii=False, indent=2)
+    elif args.extract_pages and result.pages:
+        # Output with page separators
+        output_content = result.markdown
+        output_content += "\n\n" + "=" * 50 + "\n"
+        output_content += f"EXTRACTED {len(result.pages)} PAGES:\n"
+        output_content += "=" * 50 + "\n\n"
+
+        for page in result.pages:
+            output_content += f"--- PAGE {page.page_number} ---\n"
+            output_content += page.content
+            output_content += "\n\n"
+    else:
+        # Standard output
+        output_content = result.markdown
+
     if args.output:
         with open(args.output, "w", encoding="utf-8") as f:
-            f.write(result.markdown)
+            f.write(output_content)
     else:
         # Handle stdout encoding errors more gracefully
         print(
-            result.markdown.encode(sys.stdout.encoding, errors="replace").decode(
+            output_content.encode(sys.stdout.encoding, errors="replace").decode(
                 sys.stdout.encoding
             )
         )

packages/markitdown/src/markitdown/_base_converter.py

@@ -1,7 +1,22 @@
-from typing import Any, BinaryIO, Optional
+from typing import Any, BinaryIO, Optional, List
 from ._stream_info import StreamInfo
 
 
+class PageInfo:
+    """Information about a specific page in a document."""
+
+    def __init__(self, page_number: int, content: str):
+        """
+        Initialize page information.
+
+        Parameters:
+        - page_number: The page number (1-indexed)
+        - content: The markdown content of the page
+        """
+        self.page_number = page_number
+        self.content = content
+
+
 class DocumentConverterResult:
     """The result of converting a document to Markdown."""
 
@@ -10,19 +25,22 @@ def __init__(
         markdown: str,
         *,
         title: Optional[str] = None,
+        pages: Optional[List[PageInfo]] = None,
     ):
         """
         Initialize the DocumentConverterResult.
 
         The only required parameter is the converted Markdown text.
-        The title, and any other metadata that may be added in the future, are optional.
+        The title, pages, and any other metadata that may be added in the future, are optional.
 
         Parameters:
         - markdown: The converted Markdown text.
         - title: Optional title of the document.
+        - pages: Optional list of page information for documents with page structure.
         """
         self.markdown = markdown
         self.title = title
+        self.pages = pages
 
     @property
     def text_content(self) -> str:

packages/markitdown/src/markitdown/converters/_docx_converter.py

@@ -6,7 +6,7 @@
 
 from ._html_converter import HtmlConverter
 from ..converter_utils.docx.pre_process import pre_process_docx
-from .._base_converter import DocumentConverterResult
+from .._base_converter import DocumentConverterResult, PageInfo
 from .._stream_info import StreamInfo
 from .._exceptions import MissingDependencyException, MISSING_DEPENDENCY_MESSAGE
 
@@ -75,9 +75,24 @@ def convert(
                 _dependency_exc_info[2]
             )
 
+        # Check if page extraction is requested
+        extract_pages = kwargs.get("extract_pages", False)
+
         style_map = kwargs.get("style_map", None)
         pre_process_stream = pre_process_docx(file_stream)
-        return self._html_converter.convert_string(
-            mammoth.convert_to_html(pre_process_stream, style_map=style_map).value,
-            **kwargs,
+
+        # Convert to HTML
+        html_result = mammoth.convert_to_html(pre_process_stream, style_map=style_map)
+
+        # Convert HTML to markdown
+        result = self._html_converter.convert_string(html_result.value, **kwargs)
+
+        # Note: DOCX files don't have fixed pages like PDFs.
+        # Page breaks depend on rendering settings (margins, font size, etc.)
+        # For now, we'll return None for pages even if extract_pages is True
+        # This maintains API compatibility while acknowledging the limitation
+        pages = None
+
+        return DocumentConverterResult(
+            markdown=result.markdown, title=result.title, pages=pages
         )