From 99154b070f0c78d9a912575496e73a59897e1674 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mence=20Lesn=C3=A9?= Date: Mon, 7 Jul 2025 16:11:53 +0200 Subject: [PATCH] perf: Migrate logic to async For backward compatibility, converters are added new "_async" methods, in complement of the original (sync). Sync method is marked deprecated. Youtube converted is executed in a thread context (the lib isn't natively async) and OpenAI integration use the AsyncOpenAI class, also marking sync class as deprecated. --- README.md | 47 ++- .../src/markitdown_mcp/__main__.py | 4 +- packages/markitdown-sample-plugin/README.md | 21 +- .../src/markitdown_sample_plugin/_plugin.py | 4 +- .../tests/test_sample_plugin.py | 7 +- packages/markitdown/README.md | 2 +- packages/markitdown/pyproject.toml | 28 +- .../src/markitdown/_base_converter.py | 102 ++++- .../markitdown/src/markitdown/_markitdown.py | 386 ++++++++++++++---- .../src/markitdown/converter_utils/llm.py | 107 +++++ .../markitdown/converters/_audio_converter.py | 8 +- .../converters/_bing_serp_converter.py | 4 +- .../markitdown/converters/_csv_converter.py | 4 +- .../converters/_doc_intel_converter.py | 30 +- .../markitdown/converters/_docx_converter.py | 6 +- .../markitdown/converters/_epub_converter.py | 6 +- .../markitdown/converters/_html_converter.py | 8 +- .../markitdown/converters/_image_converter.py | 71 +--- .../markitdown/converters/_ipynb_converter.py | 4 +- .../src/markitdown/converters/_llm_caption.py | 50 --- .../converters/_outlook_msg_converter.py | 6 +- .../markitdown/converters/_pdf_converter.py | 4 +- .../converters/_plain_text_converter.py | 4 +- .../markitdown/converters/_pptx_converter.py | 54 +-- .../markitdown/converters/_rss_converter.py | 4 +- .../converters/_transcribe_audio.py | 20 +- .../converters/_wikipedia_converter.py | 4 +- .../markitdown/converters/_xlsx_converter.py | 22 +- .../converters/_youtube_converter.py | 14 +- .../markitdown/converters/_zip_converter.py | 6 +- 30 files changed, 675 insertions(+), 362 deletions(-) create mode 100644 packages/markitdown/src/markitdown/converter_utils/llm.py delete mode 100644 packages/markitdown/src/markitdown/converters/_llm_caption.py diff --git a/README.md b/README.md index e030c408a..a4754e6ef 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,8 @@ > [!IMPORTANT] > Breaking changes between 0.0.1 to 0.1.0: -> * Dependencies are now organized into optional feature-groups (further details below). Use `pip install 'markitdown[all]'` to have backward-compatible behavior. +> +> * Dependencies are now organized into optional feature-groups (further details below). Use `pip install 'markitdown[all]'` to have backward-compatible behavior. > * convert\_stream() now requires a binary file-like object (e.g., a file opened in binary mode, or an io.BytesIO object). This is a breaking change from the previous version, where it previously also accepted text file-like objects, like io.StringIO. > * The DocumentConverter class interface has changed to read from file-like streams rather than file paths. *No temporary files are created anymore*. If you are the maintainer of a plugin, or custom DocumentConverter, you likely need to update your code. Otherwise, if only using the MarkItDown class or CLI (as in these examples), you should not need to change anything. @@ -17,29 +18,30 @@ MarkItDown is a lightweight Python utility for converting various files to Markd MarkItDown currently supports the conversion from: -- PDF -- PowerPoint -- Word -- Excel -- Images (EXIF metadata and OCR) -- Audio (EXIF metadata and speech transcription) -- HTML -- Text-based formats (CSV, JSON, XML) -- ZIP files (iterates over contents) -- Youtube URLs -- EPubs -- ... and more! +* PDF +* PowerPoint +* Word +* Excel +* Images (EXIF metadata and OCR) +* Audio (EXIF metadata and speech transcription) +* HTML +* Text-based formats (CSV, JSON, XML) +* ZIP files (iterates over contents) +* Youtube URLs +* EPubs +* ... and more! ## Why Markdown? Markdown is extremely close to plain text, with minimal markup or formatting, but still provides a way to represent important document structure. Mainstream LLMs, such as -OpenAI's GPT-4o, natively "_speak_" Markdown, and often incorporate Markdown into their +OpenAI's GPT-4o, natively "*speak*" Markdown, and often incorporate Markdown into their responses unprompted. This suggests that they have been trained on vast amounts of Markdown-formatted text, and understand it well. As a side benefit, Markdown conventions are also highly token-efficient. ## Prerequisites + MarkItDown requires Python 3.10 or higher. It is recommended to use a virtual environment to avoid dependency conflicts. With the standard Python installation, you can create and activate a virtual environment using the following commands: @@ -95,6 +97,7 @@ cat path-to-file.pdf | markitdown ``` ### Optional Dependencies + MarkItDown has optional dependencies for activating various file formats. Earlier in this document, we installed all optional dependencies with the `[all]` option. However, you can also install them individually for more control. For example: ```bash @@ -144,7 +147,7 @@ More information about how to set up an Azure Document Intelligence Resource can ### Python API -Basic usage in Python: +Basic usage in Python. All functions are available in async and sync versions: ```python from markitdown import MarkItDown @@ -164,13 +167,13 @@ result = md.convert("test.pdf") print(result.text_content) ``` -To use Large Language Models for image descriptions, provide `llm_client` and `llm_model`: +To use Large Language Models for image descriptions, provide `llm_client` (AsyncOpenAI) and `llm_model` (str), and enable the `[openai]` optional dependency group: ```python from markitdown import MarkItDown -from openai import OpenAI +from openai import AsyncOpenAI -client = OpenAI() +client = AsyncOpenAI() md = MarkItDown(llm_client=client, llm_model="gpt-4o") result = md.convert("example.jpg") print(result.text_content) @@ -187,7 +190,7 @@ docker run --rm -i markitdown:latest < ~/your-file.pdf > output.md This project welcomes contributions and suggestions. Most contributions require you to agree to a Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us -the rights to use your contribution. For details, visit https://cla.opensource.microsoft.com. +the rights to use your contribution. For details, visit . When you submit a pull request, a CLA bot will automatically determine whether you need to provide a CLA and decorate the PR appropriately (e.g., status check, comment). Simply follow the instructions @@ -212,13 +215,13 @@ You can help by looking at issues or helping review PRs. Any issue or PR is welc ### Running Tests and Checks -- Navigate to the MarkItDown package: +* Navigate to the MarkItDown package: ```sh cd packages/markitdown ``` -- Install `hatch` in your environment and run tests: +* Install `hatch` in your environment and run tests: ```sh pip install hatch # Other ways of installing hatch: https://hatch.pypa.io/dev/install/ @@ -233,7 +236,7 @@ You can help by looking at issues or helping review PRs. Any issue or PR is welc hatch test ``` -- Run pre-commit checks before submitting a PR: `pre-commit run --all-files` +* Run pre-commit checks before submitting a PR: `pre-commit run --all-files` ### Contributing 3rd-party Plugins diff --git a/packages/markitdown-mcp/src/markitdown_mcp/__main__.py b/packages/markitdown-mcp/src/markitdown_mcp/__main__.py index 77ade9974..4e0a4beca 100644 --- a/packages/markitdown-mcp/src/markitdown_mcp/__main__.py +++ b/packages/markitdown-mcp/src/markitdown_mcp/__main__.py @@ -20,7 +20,9 @@ @mcp.tool() async def convert_to_markdown(uri: str) -> str: """Convert a resource described by an http:, https:, file: or data: URI to markdown""" - return MarkItDown(enable_plugins=check_plugins_enabled()).convert_uri(uri).markdown + return ( + await MarkItDown(enable_plugins=check_plugins_enabled()).convert_uri_async(uri) + ).markdown def check_plugins_enabled() -> bool: diff --git a/packages/markitdown-sample-plugin/README.md b/packages/markitdown-sample-plugin/README.md index adf1d9e7c..006851701 100644 --- a/packages/markitdown-sample-plugin/README.md +++ b/packages/markitdown-sample-plugin/README.md @@ -20,28 +20,25 @@ class RtfConverter(DocumentConverter): ): super().__init__(priority=priority) - def accepts( + async def accepts_async( self, file_stream: BinaryIO, stream_info: StreamInfo, **kwargs: Any, ) -> bool: - - # Implement logic to check if the file stream is an RTF file - # ... - raise NotImplementedError() + # Implement logic to check if the file stream is an RTF file + # ... + raise NotImplementedError() - - def convert( + async def convert_async( self, file_stream: BinaryIO, stream_info: StreamInfo, **kwargs: Any, ) -> DocumentConverterResult: - - # Implement logic to convert the file stream to Markdown - # ... - raise NotImplementedError() + # Implement logic to convert the file stream to Markdown + # ... + raise NotImplementedError() ``` Next, make sure your package implements and exports the following: @@ -98,7 +95,7 @@ In Python, plugins can be enabled as follows: from markitdown import MarkItDown md = MarkItDown(enable_plugins=True) -result = md.convert("path-to-file.rtf") +result = await md.convert_async("path-to-file.rtf") print(result.text_content) ``` diff --git a/packages/markitdown-sample-plugin/src/markitdown_sample_plugin/_plugin.py b/packages/markitdown-sample-plugin/src/markitdown_sample_plugin/_plugin.py index 1ca00ccc5..850406bd5 100644 --- a/packages/markitdown-sample-plugin/src/markitdown_sample_plugin/_plugin.py +++ b/packages/markitdown-sample-plugin/src/markitdown_sample_plugin/_plugin.py @@ -36,7 +36,7 @@ class RtfConverter(DocumentConverter): Converts an RTF file to in the simplest possible way. """ - def accepts( + async def accepts_async( self, file_stream: BinaryIO, stream_info: StreamInfo, @@ -54,7 +54,7 @@ def accepts( return False - def convert( + async def convert_async( self, file_stream: BinaryIO, stream_info: StreamInfo, diff --git a/packages/markitdown-sample-plugin/tests/test_sample_plugin.py b/packages/markitdown-sample-plugin/tests/test_sample_plugin.py index 696824742..67b8772a0 100644 --- a/packages/markitdown-sample-plugin/tests/test_sample_plugin.py +++ b/packages/markitdown-sample-plugin/tests/test_sample_plugin.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 -m pytest +import asyncio import os from markitdown import MarkItDown, StreamInfo @@ -12,11 +13,11 @@ } -def test_converter() -> None: +async def test_converter() -> None: """Tests the RTF converter dirctly.""" with open(os.path.join(TEST_FILES_DIR, "test.rtf"), "rb") as file_stream: converter = RtfConverter() - result = converter.convert( + result = await converter.convert_async( file_stream=file_stream, stream_info=StreamInfo( mimetype="text/rtf", extension=".rtf", filename="test.rtf" @@ -38,6 +39,6 @@ def test_markitdown() -> None: if __name__ == "__main__": """Runs this file's tests from the command line.""" - test_converter() + asyncio.run(test_converter()) test_markitdown() print("All tests passed.") diff --git a/packages/markitdown/README.md b/packages/markitdown/README.md index edd270166..04cbf8cc5 100644 --- a/packages/markitdown/README.md +++ b/packages/markitdown/README.md @@ -1,7 +1,7 @@ # MarkItDown > [!IMPORTANT] -> MarkItDown is a Python package and command-line utility for converting various files to Markdown (e.g., for indexing, text analysis, etc). +> MarkItDown is a Python package and command-line utility for converting various files to Markdown (e.g., for indexing, text analysis, etc). > > For more information, and full documentation, see the project [README.md](https://github.com/microsoft/markitdown) on GitHub. diff --git a/packages/markitdown/pyproject.toml b/packages/markitdown/pyproject.toml index afb5d3197..8dff9ed9b 100644 --- a/packages/markitdown/pyproject.toml +++ b/packages/markitdown/pyproject.toml @@ -10,9 +10,7 @@ readme = "README.md" requires-python = ">=3.10" license = "MIT" keywords = [] -authors = [ - { name = "Adam Fourney", email = "adamfo@microsoft.com" }, -] +authors = [{ name = "Adam Fourney", email = "adamfo@microsoft.com" }] classifiers = [ "Development Status :: 4 - Beta", "Programming Language :: Python", @@ -25,12 +23,13 @@ classifiers = [ ] dependencies = [ "beautifulsoup4", - "requests", "markdownify", "magika~=0.6.1", "charset-normalizer", "defusedxml", "onnxruntime<=1.20.1; sys_platform == 'win32'", + "aiohttp", + "openai", ] [project.optional-dependencies] @@ -47,7 +46,7 @@ all = [ "SpeechRecognition", "youtube-transcript-api~=1.0.0", "azure-ai-documentintelligence", - "azure-identity" + "azure-identity", ] pptx = ["python-pptx"] docx = ["mammoth", "lxml"] @@ -75,16 +74,11 @@ features = ["all"] [tool.hatch.envs.hatch-test] features = ["all"] -extra-dependencies = [ - "openai", -] +parallel = true [tool.hatch.envs.types] features = ["all"] -extra-dependencies = [ - "openai", - "mypy>=1.0.0", -] +extra-dependencies = ["mypy>=1.0.0"] [tool.hatch.envs.types.scripts] check = "mypy --install-types --non-interactive --ignore-missing-imports {args:src/markitdown tests}" @@ -93,20 +87,14 @@ check = "mypy --install-types --non-interactive --ignore-missing-imports {args:s source_pkgs = ["markitdown", "tests"] branch = true parallel = true -omit = [ - "src/markitdown/__about__.py", -] +omit = ["src/markitdown/__about__.py"] [tool.coverage.paths] markitdown = ["src/markitdown", "*/markitdown/src/markitdown"] tests = ["tests", "*/markitdown/tests"] [tool.coverage.report] -exclude_lines = [ - "no cov", - "if __name__ == .__main__.:", - "if TYPE_CHECKING:", -] +exclude_lines = ["no cov", "if __name__ == .__main__.:", "if TYPE_CHECKING:"] [tool.hatch.build.targets.sdist] only-include = ["src/markitdown"] diff --git a/packages/markitdown/src/markitdown/_base_converter.py b/packages/markitdown/src/markitdown/_base_converter.py index a6f2a2d91..d7a843c31 100644 --- a/packages/markitdown/src/markitdown/_base_converter.py +++ b/packages/markitdown/src/markitdown/_base_converter.py @@ -1,7 +1,30 @@ +import functools +import warnings from typing import Any, BinaryIO, Optional + from ._stream_info import StreamInfo +def deprecated(reason): + """ + Indicate that a class, function or overload is deprecated. + """ + + def decorator(func): + @functools.wraps(func) + def wrapper(*args, **kwargs): + warnings.warn( + f"{func.__name__} is deprecated: {reason}", + DeprecationWarning, + stacklevel=2, + ) + return func(*args, **kwargs) + + return wrapper + + return decorator + + class DocumentConverterResult: """The result of converting a document to Markdown.""" @@ -40,8 +63,11 @@ def __str__(self) -> str: class DocumentConverter: - """Abstract superclass of all DocumentConverters.""" + """ + Abstract superclass of all DocumentConverters. + """ + @deprecated("Use accepts_async() instead.") def accepts( self, file_stream: BinaryIO, @@ -56,6 +82,8 @@ def accepts( Finally, it is conceivable that the `stream_info.filename` might be used to in cases where the filename is well-known (e.g., `Dockerfile`, `Makefile`, etc) + PERFORMANCE: This method is kept for backward compatibility with existing code that uses the synchronous interfaces. Prefer to use the async version. Async code will be used in priorities over synchronous code. + NOTE: The method signature is designed to match that of the convert() method. This provides some assurance that, if accepts() returns True, the convert() method will also be able to handle the document. @@ -77,10 +105,51 @@ def accepts( Returns: - bool: True if the converter can handle the document, False otherwise. """ + raise NotImplementedError( f"The subclass, {type(self).__name__}, must implement the accepts() method to determine if they can handle the document." ) + async def accepts_async( + self, + file_stream: BinaryIO, + stream_info: StreamInfo, + **kwargs: Any, # Options to pass to the converter + ) -> bool: + """ + Return a quick determination on if the converter should attempt converting the document. + This is primarily based `stream_info` (typically, `stream_info.mimetype`, `stream_info.extension`). + In cases where the data is retrieved via HTTP, the `steam_info.url` might also be referenced to + make a determination (e.g., special converters for Wikipedia, YouTube etc). + Finally, it is conceivable that the `stream_info.filename` might be used to in cases + where the filename is well-known (e.g., `Dockerfile`, `Makefile`, etc) + + NOTE: The method signature is designed to match that of the convert() method. This provides some + assurance that, if accepts() returns True, the convert() method will also be able to handle the document. + + IMPORTANT: In rare cases, (e.g., OutlookMsgConverter) we need to read more from the stream to make a final + determination. Read operations inevitably advances the position in file_stream. In these case, the position + MUST be reset it MUST be reset before returning. This is because the convert() method may be called immediately + after accepts(), and will expect the file_stream to be at the original position. + + E.g., + cur_pos = file_stream.tell() # Save the current position + data = file_stream.read(100) # ... peek at the first 100 bytes, etc. + file_stream.seek(cur_pos) # Reset the position to the original position + + Prameters: + - file_stream: The file-like object to convert. Must support seek(), tell(), and read() methods. + - stream_info: The StreamInfo object containing metadata about the file (mimetype, extension, charset, set) + - kwargs: Additional keyword arguments for the converter. + + Returns: + - bool: True if the converter can handle the document, False otherwise. + """ + raise NotImplementedError( + f"The subclass, {type(self).__name__}, must implement the accepts_async() method to determine if they can handle the document." + ) + + @deprecated("Use convert_async() instead.") def convert( self, file_stream: BinaryIO, @@ -90,6 +159,8 @@ def convert( """ Convert a document to Markdown text. + PERFORMANCE: This method is kept for backward compatibility with existing code that uses the synchronous interfaces. Prefer to use the async version. Async code will be used in priorities over synchronous code. + Prameters: - file_stream: The file-like object to convert. Must support seek(), tell(), and read() methods. - stream_info: The StreamInfo object containing metadata about the file (mimetype, extension, charset, set) @@ -102,4 +173,31 @@ def convert( - FileConversionException: If the mimetype is recognized, but the conversion fails for some other reason. - MissingDependencyException: If the converter requires a dependency that is not installed. """ - raise NotImplementedError("Subclasses must implement this method") + raise NotImplementedError( + f"The subclass, {type(self).__name__}, must implement the convert() method to convert the document to Markdown." + ) + + async def convert_async( + self, + file_stream: BinaryIO, + stream_info: StreamInfo, + **kwargs: Any, # Options to pass to the converter + ) -> DocumentConverterResult: + """ + Convert a document to Markdown text. + + Prameters: + - file_stream: The file-like object to convert. Must support seek(), tell(), and read() methods. + - stream_info: The StreamInfo object containing metadata about the file (mimetype, extension, charset, set) + - kwargs: Additional keyword arguments for the converter. + + Returns: + - DocumentConverterResult: The result of the conversion, which includes the title and markdown content. + + Raises: + - FileConversionException: If the mimetype is recognized, but the conversion fails for some other reason. + - MissingDependencyException: If the converter requires a dependency that is not installed. + """ + raise NotImplementedError( + f"The subclass, {type(self).__name__}, must implement the convert_async() method to convert the document to Markdown." + ) diff --git a/packages/markitdown/src/markitdown/_markitdown.py b/packages/markitdown/src/markitdown/_markitdown.py index 3027efc65..e4fe34e94 100644 --- a/packages/markitdown/src/markitdown/_markitdown.py +++ b/packages/markitdown/src/markitdown/_markitdown.py @@ -1,55 +1,63 @@ +import asyncio +import codecs +import concurrent.futures +import io import mimetypes import os import re -import sys import shutil +import sys import traceback -import io from dataclasses import dataclass from importlib.metadata import entry_points -from typing import Any, List, Dict, Optional, Union, BinaryIO from pathlib import Path -from urllib.parse import urlparse +from typing import ( + Any, + Awaitable, + BinaryIO, + Callable, + Dict, + List, + Optional, + TypeVar, + Union, +) from warnings import warn -import requests -import magika + +import aiohttp import charset_normalizer -import codecs +import magika +from ._base_converter import DocumentConverter, DocumentConverterResult +from ._exceptions import ( + FailedConversionAttempt, + FileConversionException, + UnsupportedFormatException, +) from ._stream_info import StreamInfo -from ._uri_utils import parse_data_uri, file_uri_to_path - +from ._uri_utils import file_uri_to_path, parse_data_uri from .converters import ( - PlainTextConverter, - HtmlConverter, - RssConverter, - WikipediaConverter, - YouTubeConverter, - IpynbConverter, + AudioConverter, BingSerpConverter, - PdfConverter, + CsvConverter, + DocumentIntelligenceConverter, DocxConverter, - XlsxConverter, - XlsConverter, - PptxConverter, + EpubConverter, + HtmlConverter, ImageConverter, - AudioConverter, + IpynbConverter, OutlookMsgConverter, + PdfConverter, + PlainTextConverter, + PptxConverter, + RssConverter, + WikipediaConverter, + XlsConverter, + XlsxConverter, + YouTubeConverter, ZipConverter, - EpubConverter, - DocumentIntelligenceConverter, - CsvConverter, -) - -from ._base_converter import DocumentConverter, DocumentConverterResult - -from ._exceptions import ( - FileConversionException, - UnsupportedFormatException, - FailedConversionAttempt, ) - # Lower priority values are tried first. PRIORITY_SPECIFIC_FILE_FORMAT = ( 0.0 # e.g., .docx, .pdf, .xlsx, Or specific pages, e.g., wikipedia @@ -58,7 +66,7 @@ 10.0 # Near catch-all converters for mimetypes like text/*, etc. ) - +T = TypeVar("T") _plugins: Union[None, List[Any]] = None # If None, plugins have not been loaded yet. @@ -90,6 +98,37 @@ class ConverterRegistration: priority: float +def _wrap_async( + func: Callable[..., Awaitable[T]], + *args: Any, + **kwargs: Any, +) -> T: + """ + Wrap an async function to run it in a synchronous context. + + This ensures backwards compatibility with existing synchronous code that calls async functions, without duplicating the codebase. + """ + try: + # Check if we're already in an event loop + asyncio.get_running_loop() + # If we are, we need to run in a thread to avoid blocking + + warn( + f"Running async function '{func.__name__}' in a thread pool because we're already in an event loop. " + f"Consider using the async version directly (e.g., '{func.__name__.replace('_async', '')}_async()') instead for better performance.", + RuntimeWarning, + stacklevel=3, + ) + + with concurrent.futures.ThreadPoolExecutor() as executor: + future = executor.submit(asyncio.run, func(*args, **kwargs)) + return future.result() + + except RuntimeError: + # No running loop, safe to use asyncio.run() + return asyncio.run(func(*args, **kwargs)) + + class MarkItDown: """(In preview) An extremely simple text-based document reader, suitable for LLM use. This reader will convert common file-types or webpages to Markdown.""" @@ -103,13 +142,6 @@ def __init__( ): self._builtins_enabled = False self._plugins_enabled = False - - requests_session = kwargs.get("requests_session") - if requests_session is None: - self._requests_session = requests.Session() - else: - self._requests_session = requests_session - self._magika = magika.Magika() # TODO - remove these (see enable_builtins) @@ -242,14 +274,44 @@ def enable_plugins(self, **kwargs) -> None: def convert( self, - source: Union[str, requests.Response, Path, BinaryIO], + source: Union[ + str, + aiohttp.ClientResponse, + Path, + BinaryIO, + ], *, stream_info: Optional[StreamInfo] = None, **kwargs: Any, ) -> DocumentConverterResult: # TODO: deal with kwargs """ Args: - - source: can be a path (str or Path), url, or a requests.response object + - source: can be a path (str or Path), url, or a aiohttp.ClientResponse object + - stream_info: optional stream info to use for the conversion. If None, infer from source + - kwargs: additional arguments to pass to the converter + """ + return _wrap_async( + self.convert_async, + source, + stream_info=stream_info, + **kwargs, + ) + + async def convert_async( + self, + source: Union[ + str, + aiohttp.ClientResponse, + Path, + BinaryIO, + ], + *, + stream_info: Optional[StreamInfo] = None, + **kwargs: Any, + ) -> DocumentConverterResult: # TODO: deal with kwargs + """ + Args: + - source: can be a path (str or Path), url, or a aiohttp.ClientResponse object - stream_info: optional stream info to use for the conversion. If None, infer from source - kwargs: additional arguments to pass to the converter """ @@ -269,22 +331,38 @@ def convert( _kwargs["mock_url"] = _kwargs["url"] del _kwargs["url"] - return self.convert_uri(source, stream_info=stream_info, **_kwargs) + return await self.convert_uri_async( + source, + stream_info=stream_info, + **_kwargs, + ) else: - return self.convert_local(source, stream_info=stream_info, **kwargs) + return await self.convert_local_async( + source, + stream_info=stream_info, + **kwargs, + ) # Path object elif isinstance(source, Path): - return self.convert_local(source, stream_info=stream_info, **kwargs) + return await self.convert_local_async( + source, stream_info=stream_info, **kwargs + ) # Request response - elif isinstance(source, requests.Response): - return self.convert_response(source, stream_info=stream_info, **kwargs) + elif isinstance(source, aiohttp.ClientResponse): + return await self.convert_response_async( + source, stream_info=stream_info, **kwargs + ) # Binary stream elif ( hasattr(source, "read") and callable(source.read) and not isinstance(source, io.TextIOBase) ): - return self.convert_stream(source, stream_info=stream_info, **kwargs) + return await self.convert_stream_async( + source, + stream_info=stream_info, + **kwargs, + ) else: raise TypeError( f"Invalid source type: {type(source)}. Expected str, requests.Response, BinaryIO." @@ -298,15 +376,33 @@ def convert_local( file_extension: Optional[str] = None, # Deprecated -- use stream_info url: Optional[str] = None, # Deprecated -- use stream_info **kwargs: Any, + ) -> DocumentConverterResult: + return _wrap_async( + self.convert_local_async, + path, + stream_info=stream_info, + file_extension=file_extension, + url=url, + **kwargs, + ) + + async def convert_local_async( + self, + path: Union[str, Path], + *, + stream_info: Optional[StreamInfo] = None, + file_extension: Optional[str] = None, # Deprecated -- use stream_info + url: Optional[str] = None, # Deprecated -- use stream_info + **kwargs: Any, ) -> DocumentConverterResult: if isinstance(path, Path): path = str(path) # Build a base StreamInfo object from which to start guesses base_guess = StreamInfo( - local_path=path, extension=os.path.splitext(path)[1], filename=os.path.basename(path), + local_path=path, ) # Extend the base_guess with any additional info from the arguments @@ -323,9 +419,14 @@ def convert_local( with open(path, "rb") as fh: guesses = self._get_stream_info_guesses( - file_stream=fh, base_guess=base_guess + base_guess=base_guess, + file_stream=fh, + ) + return await self._convert( + file_stream=fh, + stream_info_guesses=guesses, + **kwargs, ) - return self._convert(file_stream=fh, stream_info_guesses=guesses, **kwargs) def convert_stream( self, @@ -335,6 +436,24 @@ def convert_stream( file_extension: Optional[str] = None, # Deprecated -- use stream_info url: Optional[str] = None, # Deprecated -- use stream_info **kwargs: Any, + ) -> DocumentConverterResult: + return _wrap_async( + self.convert_stream_async, + stream, + stream_info=stream_info, + file_extension=file_extension, + url=url, + **kwargs, + ) + + async def convert_stream_async( + self, + stream: BinaryIO, + *, + stream_info: Optional[StreamInfo] = None, + file_extension: Optional[str] = None, # Deprecated -- use stream_info + url: Optional[str] = None, # Deprecated -- use stream_info + **kwargs: Any, ) -> DocumentConverterResult: guesses: List[StreamInfo] = [] @@ -370,9 +489,12 @@ def convert_stream( # Add guesses based on stream content guesses = self._get_stream_info_guesses( - file_stream=stream, base_guess=base_guess or StreamInfo() + base_guess=base_guess or StreamInfo(), + file_stream=stream, + ) + return await self._convert( + file_stream=stream, stream_info_guesses=guesses, **kwargs ) - return self._convert(file_stream=stream, stream_info_guesses=guesses, **kwargs) def convert_url( self, @@ -383,9 +505,32 @@ def convert_url( mock_url: Optional[str] = None, **kwargs: Any, ) -> DocumentConverterResult: - """Alias for convert_uri()""" + """ + Alias for convert_uri(). + """ + return _wrap_async( + self.convert_uri_async, + url, + stream_info=stream_info, + file_extension=file_extension, + mock_url=mock_url, + **kwargs, + ) + + async def convert_url_async( + self, + url: str, + *, + stream_info: Optional[StreamInfo] = None, + file_extension: Optional[str] = None, + mock_url: Optional[str] = None, + **kwargs: Any, + ) -> DocumentConverterResult: + """ + Alias for convert_uri(). + """ # convert_url will likely be deprecated in the future in favor of convert_uri - return self.convert_uri( + return await self.convert_uri_async( url, stream_info=stream_info, file_extension=file_extension, @@ -403,6 +548,26 @@ def convert_uri( str ] = None, # Mock the request as if it came from a different URL **kwargs: Any, + ) -> DocumentConverterResult: + return _wrap_async( + self.convert_uri_async, + uri, + stream_info=stream_info, + file_extension=file_extension, + mock_url=mock_url, + **kwargs, + ) + + async def convert_uri_async( + self, + uri: str, + *, + stream_info: Optional[StreamInfo] = None, + file_extension: Optional[str] = None, # Deprecated -- use stream_info + mock_url: Optional[ + str + ] = None, # Mock the request as if it came from a different URL + **kwargs: Any, ) -> DocumentConverterResult: uri = uri.strip() @@ -413,7 +578,7 @@ def convert_uri( raise ValueError( f"Unsupported file URI: {uri}. Netloc must be empty or localhost." ) - return self.convert_local( + return await self.convert_local_async( path, stream_info=stream_info, file_extension=file_extension, @@ -425,13 +590,13 @@ def convert_uri( mimetype, attributes, data = parse_data_uri(uri) base_guess = StreamInfo( - mimetype=mimetype, charset=attributes.get("charset"), + mimetype=mimetype, ) if stream_info is not None: base_guess = base_guess.copy_and_update(stream_info) - return self.convert_stream( + return await self.convert_stream_async( io.BytesIO(data), stream_info=base_guess, file_extension=file_extension, @@ -440,15 +605,16 @@ def convert_uri( ) # HTTP/HTTPS URIs elif uri.startswith("http:") or uri.startswith("https:"): - response = self._requests_session.get(uri, stream=True) - response.raise_for_status() - return self.convert_response( - response, - stream_info=stream_info, - file_extension=file_extension, - url=mock_url, - **kwargs, - ) + async with aiohttp.ClientSession() as session: + async with session.get(uri) as response: + response.raise_for_status() + return await self.convert_response_async( + response, + stream_info=stream_info, + file_extension=file_extension, + url=mock_url, + **kwargs, + ) else: raise ValueError( f"Unsupported URI scheme: {uri.split(':')[0]}. Supported schemes are: file:, data:, http:, https:" @@ -456,7 +622,25 @@ def convert_uri( def convert_response( self, - response: requests.Response, + response: aiohttp.ClientResponse, + *, + stream_info: Optional[StreamInfo] = None, + file_extension: Optional[str] = None, # Deprecated -- use stream_info + url: Optional[str] = None, # Deprecated -- use stream_info + **kwargs: Any, + ) -> DocumentConverterResult: + return _wrap_async( + self.convert_response_async, + response, + stream_info=stream_info, + file_extension=file_extension, + url=url, + **kwargs, + ) + + async def convert_response_async( + self, + response: aiohttp.ClientResponse, *, stream_info: Optional[StreamInfo] = None, file_extension: Optional[str] = None, # Deprecated -- use stream_info @@ -489,19 +673,18 @@ def convert_response( # If there is still no filename, try to read it from the url if filename is None: - parsed_url = urlparse(response.url) - _, _extension = os.path.splitext(parsed_url.path) + _, _extension = os.path.splitext(response.url.path) if len(_extension) > 0: # Looks like this might be a file! - filename = os.path.basename(parsed_url.path) + filename = os.path.basename(response.url.path) extension = _extension # Create an initial guess from all this information base_guess = StreamInfo( - mimetype=mimetype, charset=charset, - filename=filename, extension=extension, - url=response.url, + filename=filename, + mimetype=mimetype, + url=str(response.url), ) # Update with any additional info from the arguments @@ -514,9 +697,10 @@ def convert_response( # Deprecated -- use stream_info base_guess = base_guess.copy_and_update(url=url) - # Read into BytesIO + # Read into BytesIO, 8k chunks should be fine for most use cases + # TODO: Stream the response instead of consuming the memory (temp file?), can't use async iterator because lib handles both async and sync converters buffer = io.BytesIO() - for chunk in response.iter_content(chunk_size=512): + async for chunk in response.content.iter_chunked(8192): buffer.write(chunk) buffer.seek(0) @@ -524,10 +708,18 @@ def convert_response( guesses = self._get_stream_info_guesses( file_stream=buffer, base_guess=base_guess ) - return self._convert(file_stream=buffer, stream_info_guesses=guesses, **kwargs) + return await self._convert( + file_stream=buffer, + stream_info_guesses=guesses, + **kwargs, + ) - def _convert( - self, *, file_stream: BinaryIO, stream_info_guesses: List[StreamInfo], **kwargs + async def _convert( + self, + *, + file_stream: BinaryIO, + stream_info_guesses: List[StreamInfo], + **kwargs, ) -> DocumentConverterResult: res: Union[None, DocumentConverterResult] = None @@ -579,9 +771,22 @@ def _convert( # Check if the converter will accept the file, and if so, try to convert it _accepts = False try: - _accepts = converter.accepts(file_stream, stream_info, **_kwargs) + # First, try the async version + _accepts = await converter.accepts_async( + file_stream=file_stream, + stream_info=stream_info, + **_kwargs, + ) except NotImplementedError: - pass + try: + # In backwards compatibility mode, use the sync version + _accepts = converter.accepts( + file_stream=file_stream, + stream_info=stream_info, + **_kwargs, + ) + except NotImplementedError: + pass # accept() should not have changed the file stream position assert ( @@ -591,7 +796,20 @@ def _convert( # Attempt the conversion if _accepts: try: - res = converter.convert(file_stream, stream_info, **_kwargs) + try: + # First, try the async version + res = await converter.convert_async( + file_stream=file_stream, + stream_info=stream_info, + **_kwargs, + ) + except NotImplementedError: + # In backwards compatibility mode, use the sync version + res = converter.convert( + file_stream=file_stream, + stream_info=stream_info, + **_kwargs, + ) except Exception: failed_attempts.append( FailedConversionAttempt( @@ -659,7 +877,9 @@ def register_converter( ) def _get_stream_info_guesses( - self, file_stream: BinaryIO, base_guess: StreamInfo + self, + file_stream: BinaryIO, + base_guess: StreamInfo, ) -> List[StreamInfo]: """ Given a base guess, attempt to guess or expand on the stream info using the stream content (via magika). diff --git a/packages/markitdown/src/markitdown/converter_utils/llm.py b/packages/markitdown/src/markitdown/converter_utils/llm.py new file mode 100644 index 000000000..f05758f96 --- /dev/null +++ b/packages/markitdown/src/markitdown/converter_utils/llm.py @@ -0,0 +1,107 @@ +import base64 +import mimetypes +from typing import BinaryIO, Union + +from openai import AsyncOpenAI, OpenAI +from openai.types.chat import ChatCompletionMessageParam, ChatCompletionUserMessageParam +from openai.types.chat.chat_completion_content_part_image_param import ( + ChatCompletionContentPartImageParam, + ImageURL, +) +from openai.types.chat.chat_completion_content_part_text_param import ( + ChatCompletionContentPartTextParam, +) + +from .._stream_info import StreamInfo + + +# TODO: Add Content Safety test to ensure there is no prompt injection +async def _llm_completion( + client: Union[AsyncOpenAI, OpenAI], + messages: list[ChatCompletionMessageParam], + model: str, +) -> Union[str, None]: + """ + Perform a completion request using the OpenAI API. + + Either an asynchronous or synchronous client can be used. If an sync client is used, user will be warned. + + Return the raw response content. + """ + # Use either async or sync client based on the type of client provided + if isinstance(client, AsyncOpenAI): + response = await client.chat.completions.create( + messages=messages, + model=model, + ) + else: + print("Warning: Using synchronous OpenAI is blocking the event loop") + response = client.chat.completions.create( + messages=messages, + model=model, + ) + + return response.choices[0].message.content if response.choices else None + + +async def llm_image_caption( + file_stream: BinaryIO, + stream_info: StreamInfo, + *, + client: Union[AsyncOpenAI, OpenAI], + model: str, + prompt: str = None, +) -> str | None: + """ + Generate a caption for an image using the OpenAI API. + + Image is converted to a base64 then sent in the request. + """ + if prompt is None or prompt.strip() == "": + prompt = "Write a detailed caption for this image." + + # Get the content type + content_type = stream_info.mimetype + if not content_type: + content_type, _ = mimetypes.guess_type("_dummy" + (stream_info.extension or "")) + if not content_type: + content_type = "application/octet-stream" + + # Convert to base64 + # TODO: Stream the request to avoid buffering the base64 image in memory + cur_pos = file_stream.tell() + try: + base64_image = base64.b64encode(file_stream.read()).decode("utf-8") + except Exception: + return None + finally: + file_stream.seek(cur_pos) + + # Prepare the data-uri + data_uri = f"data:{content_type};base64,{base64_image}" + + # Prepare the OpenAI API request + messages = [ + ChatCompletionUserMessageParam( + role="user", + content=[ + ChatCompletionContentPartTextParam( + type="text", + text=prompt, + ), + ChatCompletionContentPartImageParam( + type="image_url", + image_url=ImageURL( + detail="auto", + url=data_uri, + ), + ), + ], + ), + ] + + return await _llm_completion( + client=client, + messages=messages, + model=model, + ) diff --git a/packages/markitdown/src/markitdown/converters/_audio_converter.py b/packages/markitdown/src/markitdown/converters/_audio_converter.py index 3d96b53c8..05948fda7 100644 --- a/packages/markitdown/src/markitdown/converters/_audio_converter.py +++ b/packages/markitdown/src/markitdown/converters/_audio_converter.py @@ -25,7 +25,7 @@ class AudioConverter(DocumentConverter): Converts audio files to markdown via extraction of metadata (if `exiftool` is installed), and speech transcription (if `speech_recognition` is installed). """ - def accepts( + async def accepts_async( self, file_stream: BinaryIO, stream_info: StreamInfo, @@ -43,7 +43,7 @@ def accepts( return False - def convert( + async def convert_async( self, file_stream: BinaryIO, stream_info: StreamInfo, @@ -91,7 +91,9 @@ def convert( # Transcribe if audio_format: try: - transcript = transcribe_audio(file_stream, audio_format=audio_format) + transcript = await transcribe_audio( + file_stream, audio_format=audio_format + ) if transcript: md_content += "\n\n### Audio Transcript:\n" + transcript except MissingDependencyException: diff --git a/packages/markitdown/src/markitdown/converters/_bing_serp_converter.py b/packages/markitdown/src/markitdown/converters/_bing_serp_converter.py index 6a3834872..d89dab48e 100644 --- a/packages/markitdown/src/markitdown/converters/_bing_serp_converter.py +++ b/packages/markitdown/src/markitdown/converters/_bing_serp_converter.py @@ -26,7 +26,7 @@ class BingSerpConverter(DocumentConverter): NOTE: It is better to use the Bing API """ - def accepts( + async def accepts_async( self, file_stream: BinaryIO, stream_info: StreamInfo, @@ -54,7 +54,7 @@ def accepts( # Not HTML content return False - def convert( + async def convert_async( self, file_stream: BinaryIO, stream_info: StreamInfo, diff --git a/packages/markitdown/src/markitdown/converters/_csv_converter.py b/packages/markitdown/src/markitdown/converters/_csv_converter.py index 7e9631e1b..10c20c0df 100644 --- a/packages/markitdown/src/markitdown/converters/_csv_converter.py +++ b/packages/markitdown/src/markitdown/converters/_csv_converter.py @@ -20,7 +20,7 @@ class CsvConverter(DocumentConverter): def __init__(self): super().__init__() - def accepts( + async def accepts_async( self, file_stream: BinaryIO, stream_info: StreamInfo, @@ -35,7 +35,7 @@ def accepts( return True return False - def convert( + async def convert_async( self, file_stream: BinaryIO, stream_info: StreamInfo, diff --git a/packages/markitdown/src/markitdown/converters/_doc_intel_converter.py b/packages/markitdown/src/markitdown/converters/_doc_intel_converter.py index ba66b5b5a..38989dba3 100644 --- a/packages/markitdown/src/markitdown/converters/_doc_intel_converter.py +++ b/packages/markitdown/src/markitdown/converters/_doc_intel_converter.py @@ -12,14 +12,10 @@ # Save reporting of any exceptions for later _dependency_exc_info = None try: - from azure.ai.documentintelligence import DocumentIntelligenceClient - from azure.ai.documentintelligence.models import ( - AnalyzeDocumentRequest, - AnalyzeResult, - DocumentAnalysisFeature, - ) + from azure.ai.documentintelligence.aio import DocumentIntelligenceClient + from azure.ai.documentintelligence.models import DocumentAnalysisFeature from azure.core.credentials import AzureKeyCredential, TokenCredential - from azure.identity import DefaultAzureCredential + from azure.identity.aio import DefaultAzureCredential except ImportError: # Preserve the error and stack trace for later _dependency_exc_info = sys.exc_info() @@ -34,12 +30,6 @@ class TokenCredential: class DocumentIntelligenceClient: pass - class AnalyzeDocumentRequest: - pass - - class AnalyzeResult: - pass - class DocumentAnalysisFeature: pass @@ -181,7 +171,7 @@ def __init__( credential=credential, ) - def accepts( + async def accepts_async( self, file_stream: BinaryIO, stream_info: StreamInfo, @@ -229,21 +219,21 @@ def _analysis_features(self, stream_info: StreamInfo) -> List[str]: DocumentAnalysisFeature.STYLE_FONT, # enable font style extraction ] - def convert( + async def convert_async( self, file_stream: BinaryIO, stream_info: StreamInfo, **kwargs: Any, # Options to pass to the converter ) -> DocumentConverterResult: # Extract the text using Azure Document Intelligence - poller = self.doc_intel_client.begin_analyze_document( - model_id="prebuilt-layout", - body=AnalyzeDocumentRequest(bytes_source=file_stream.read()), + poller = await self.doc_intel_client.begin_analyze_document( + body=file_stream, features=self._analysis_features(stream_info), + model_id="prebuilt-layout", output_content_format=CONTENT_FORMAT, # TODO: replace with "ContentFormat.MARKDOWN" when the bug is fixed ) - result: AnalyzeResult = poller.result() + result = (await poller.result()).content # remove comments from the markdown content generated by Doc Intelligence and append to markdown string - markdown_text = re.sub(r"", "", result.content, flags=re.DOTALL) + markdown_text = re.sub(r"", "", result, flags=re.DOTALL) return DocumentConverterResult(markdown=markdown_text) diff --git a/packages/markitdown/src/markitdown/converters/_docx_converter.py b/packages/markitdown/src/markitdown/converters/_docx_converter.py index 69c1ea833..0e4ab953e 100644 --- a/packages/markitdown/src/markitdown/converters/_docx_converter.py +++ b/packages/markitdown/src/markitdown/converters/_docx_converter.py @@ -34,7 +34,7 @@ def __init__(self): super().__init__() self._html_converter = HtmlConverter() - def accepts( + async def accepts_async( self, file_stream: BinaryIO, stream_info: StreamInfo, @@ -52,7 +52,7 @@ def accepts( return False - def convert( + async def convert_async( self, file_stream: BinaryIO, stream_info: StreamInfo, @@ -74,7 +74,7 @@ def convert( style_map = kwargs.get("style_map", None) pre_process_stream = pre_process_docx(file_stream) - return self._html_converter.convert_string( + return await self._html_converter.convert_string( mammoth.convert_to_html(pre_process_stream, style_map=style_map).value, **kwargs, ) diff --git a/packages/markitdown/src/markitdown/converters/_epub_converter.py b/packages/markitdown/src/markitdown/converters/_epub_converter.py index 3be65b016..9733f5eab 100644 --- a/packages/markitdown/src/markitdown/converters/_epub_converter.py +++ b/packages/markitdown/src/markitdown/converters/_epub_converter.py @@ -32,7 +32,7 @@ def __init__(self): super().__init__() self._html_converter = HtmlConverter() - def accepts( + async def accepts_async( self, file_stream: BinaryIO, stream_info: StreamInfo, @@ -50,7 +50,7 @@ def accepts( return False - def convert( + async def convert_async( self, file_stream: BinaryIO, stream_info: StreamInfo, @@ -105,7 +105,7 @@ def convert( filename = os.path.basename(file) extension = os.path.splitext(filename)[1].lower() mimetype = MIME_TYPE_MAPPING.get(extension) - converted_content = self._html_converter.convert( + converted_content = await self._html_converter.convert_async( f, StreamInfo( mimetype=mimetype, diff --git a/packages/markitdown/src/markitdown/converters/_html_converter.py b/packages/markitdown/src/markitdown/converters/_html_converter.py index dabb0d7d3..dcda93295 100644 --- a/packages/markitdown/src/markitdown/converters/_html_converter.py +++ b/packages/markitdown/src/markitdown/converters/_html_converter.py @@ -20,7 +20,7 @@ class HtmlConverter(DocumentConverter): """Anything with content type text/html""" - def accepts( + async def accepts_async( self, file_stream: BinaryIO, stream_info: StreamInfo, @@ -38,7 +38,7 @@ def accepts( return False - def convert( + async def convert_async( self, file_stream: BinaryIO, stream_info: StreamInfo, @@ -70,7 +70,7 @@ def convert( title=None if soup.title is None else soup.title.string, ) - def convert_string( + async def convert_string( self, html_content: str, *, url: Optional[str] = None, **kwargs ) -> DocumentConverterResult: """ @@ -78,7 +78,7 @@ def convert_string( Given that many converters produce HTML as intermediate output, this allows for easy conversion of HTML to markdown. """ - return self.convert( + return await self.convert_async( file_stream=io.BytesIO(html_content.encode("utf-8")), stream_info=StreamInfo( mimetype="text/html", diff --git a/packages/markitdown/src/markitdown/converters/_image_converter.py b/packages/markitdown/src/markitdown/converters/_image_converter.py index dd8fbac61..38371747f 100644 --- a/packages/markitdown/src/markitdown/converters/_image_converter.py +++ b/packages/markitdown/src/markitdown/converters/_image_converter.py @@ -1,9 +1,9 @@ -from typing import BinaryIO, Any, Union -import base64 -import mimetypes -from ._exiftool import exiftool_metadata +from typing import Any, BinaryIO + from .._base_converter import DocumentConverter, DocumentConverterResult from .._stream_info import StreamInfo +from ..converter_utils.llm import llm_image_caption +from ._exiftool import exiftool_metadata ACCEPTED_MIME_TYPE_PREFIXES = [ "image/jpeg", @@ -18,7 +18,7 @@ class ImageConverter(DocumentConverter): Converts images to markdown via extraction of metadata (if `exiftool` is installed), and description via a multimodal LLM (if an llm_client is configured). """ - def accepts( + async def accepts_async( self, file_stream: BinaryIO, stream_info: StreamInfo, @@ -36,7 +36,7 @@ def accepts( return False - def convert( + async def convert_async( self, file_stream: BinaryIO, stream_info: StreamInfo, @@ -69,12 +69,12 @@ def convert( llm_client = kwargs.get("llm_client") llm_model = kwargs.get("llm_model") if llm_client is not None and llm_model is not None: - llm_description = self._get_llm_description( - file_stream, - stream_info, + llm_description = await llm_image_caption( client=llm_client, + file_stream=file_stream, model=llm_model, prompt=kwargs.get("llm_prompt"), + stream_info=stream_info, ) if llm_description is not None: @@ -83,56 +83,3 @@ def convert( return DocumentConverterResult( markdown=md_content, ) - - def _get_llm_description( - self, - file_stream: BinaryIO, - stream_info: StreamInfo, - *, - client, - model, - prompt=None, - ) -> Union[None, str]: - if prompt is None or prompt.strip() == "": - prompt = "Write a detailed caption for this image." - - # Get the content type - content_type = stream_info.mimetype - if not content_type: - content_type, _ = mimetypes.guess_type( - "_dummy" + (stream_info.extension or "") - ) - if not content_type: - content_type = "application/octet-stream" - - # Convert to base64 - cur_pos = file_stream.tell() - try: - base64_image = base64.b64encode(file_stream.read()).decode("utf-8") - except Exception as e: - return None - finally: - file_stream.seek(cur_pos) - - # Prepare the data-uri - data_uri = f"data:{content_type};base64,{base64_image}" - - # Prepare the OpenAI API request - messages = [ - { - "role": "user", - "content": [ - {"type": "text", "text": prompt}, - { - "type": "image_url", - "image_url": { - "url": data_uri, - }, - }, - ], - } - ] - - # Call the OpenAI API - response = client.chat.completions.create(model=model, messages=messages) - return response.choices[0].message.content diff --git a/packages/markitdown/src/markitdown/converters/_ipynb_converter.py b/packages/markitdown/src/markitdown/converters/_ipynb_converter.py index b15e77aa2..e84b0da90 100644 --- a/packages/markitdown/src/markitdown/converters/_ipynb_converter.py +++ b/packages/markitdown/src/markitdown/converters/_ipynb_converter.py @@ -15,7 +15,7 @@ class IpynbConverter(DocumentConverter): """Converts Jupyter Notebook (.ipynb) files to Markdown.""" - def accepts( + async def accepts_async( self, file_stream: BinaryIO, stream_info: StreamInfo, @@ -43,7 +43,7 @@ def accepts( return False - def convert( + async def convert_async( self, file_stream: BinaryIO, stream_info: StreamInfo, diff --git a/packages/markitdown/src/markitdown/converters/_llm_caption.py b/packages/markitdown/src/markitdown/converters/_llm_caption.py deleted file mode 100644 index 004a47aef..000000000 --- a/packages/markitdown/src/markitdown/converters/_llm_caption.py +++ /dev/null @@ -1,50 +0,0 @@ -from typing import BinaryIO, Union -import base64 -import mimetypes -from .._stream_info import StreamInfo - - -def llm_caption( - file_stream: BinaryIO, stream_info: StreamInfo, *, client, model, prompt=None -) -> Union[None, str]: - if prompt is None or prompt.strip() == "": - prompt = "Write a detailed caption for this image." - - # Get the content type - content_type = stream_info.mimetype - if not content_type: - content_type, _ = mimetypes.guess_type("_dummy" + (stream_info.extension or "")) - if not content_type: - content_type = "application/octet-stream" - - # Convert to base64 - cur_pos = file_stream.tell() - try: - base64_image = base64.b64encode(file_stream.read()).decode("utf-8") - except Exception as e: - return None - finally: - file_stream.seek(cur_pos) - - # Prepare the data-uri - data_uri = f"data:{content_type};base64,{base64_image}" - - # Prepare the OpenAI API request - messages = [ - { - "role": "user", - "content": [ - {"type": "text", "text": prompt}, - { - "type": "image_url", - "image_url": { - "url": data_uri, - }, - }, - ], - } - ] - - # Call the OpenAI API - response = client.chat.completions.create(model=model, messages=messages) - return response.choices[0].message.content diff --git a/packages/markitdown/src/markitdown/converters/_outlook_msg_converter.py b/packages/markitdown/src/markitdown/converters/_outlook_msg_converter.py index d216beaea..995d15fc7 100644 --- a/packages/markitdown/src/markitdown/converters/_outlook_msg_converter.py +++ b/packages/markitdown/src/markitdown/converters/_outlook_msg_converter.py @@ -29,7 +29,7 @@ class OutlookMsgConverter(DocumentConverter): - Email body content """ - def accepts( + async def accepts_async( self, file_stream: BinaryIO, stream_info: StreamInfo, @@ -63,14 +63,14 @@ def accepts( "__properties_version1.0" in toc and "__recip_version1.0_#00000000" in toc ) - except Exception as e: + except Exception: pass finally: file_stream.seek(cur_pos) return False - def convert( + async def convert_async( self, file_stream: BinaryIO, stream_info: StreamInfo, diff --git a/packages/markitdown/src/markitdown/converters/_pdf_converter.py b/packages/markitdown/src/markitdown/converters/_pdf_converter.py index 63162d523..ec9b7b51e 100644 --- a/packages/markitdown/src/markitdown/converters/_pdf_converter.py +++ b/packages/markitdown/src/markitdown/converters/_pdf_converter.py @@ -33,7 +33,7 @@ class PdfConverter(DocumentConverter): Converts PDFs to Markdown. Most style information is ignored, so the results are essentially plain-text. """ - def accepts( + async def accepts_async( self, file_stream: BinaryIO, stream_info: StreamInfo, @@ -51,7 +51,7 @@ def accepts( return False - def convert( + async def convert_async( self, file_stream: BinaryIO, stream_info: StreamInfo, diff --git a/packages/markitdown/src/markitdown/converters/_plain_text_converter.py b/packages/markitdown/src/markitdown/converters/_plain_text_converter.py index 6f1306fe8..611b96dd3 100644 --- a/packages/markitdown/src/markitdown/converters/_plain_text_converter.py +++ b/packages/markitdown/src/markitdown/converters/_plain_text_converter.py @@ -33,7 +33,7 @@ class PlainTextConverter(DocumentConverter): """Anything with content type text/plain""" - def accepts( + async def accepts_async( self, file_stream: BinaryIO, stream_info: StreamInfo, @@ -57,7 +57,7 @@ def accepts( return False - def convert( + async def convert_async( self, file_stream: BinaryIO, stream_info: StreamInfo, diff --git a/packages/markitdown/src/markitdown/converters/_pptx_converter.py b/packages/markitdown/src/markitdown/converters/_pptx_converter.py index 087da32c5..b6cd22677 100644 --- a/packages/markitdown/src/markitdown/converters/_pptx_converter.py +++ b/packages/markitdown/src/markitdown/converters/_pptx_converter.py @@ -1,18 +1,19 @@ -import sys import base64 -import os +import html import io +import os import re -import html - -from typing import BinaryIO, Any +import sys from operator import attrgetter +from typing import Any, BinaryIO, Optional, Union + +from openai import AsyncOpenAI, OpenAI -from ._html_converter import HtmlConverter -from ._llm_caption import llm_caption from .._base_converter import DocumentConverter, DocumentConverterResult +from .._exceptions import MISSING_DEPENDENCY_MESSAGE, MissingDependencyException from .._stream_info import StreamInfo -from .._exceptions import MissingDependencyException, MISSING_DEPENDENCY_MESSAGE +from ..converter_utils.llm import llm_image_caption +from ._html_converter import HtmlConverter # Try loading optional (but in this case, required) dependencies # Save reporting of any exceptions for later @@ -40,7 +41,7 @@ def __init__(self): super().__init__() self._html_converter = HtmlConverter() - def accepts( + async def accepts_async( self, file_stream: BinaryIO, stream_info: StreamInfo, @@ -58,7 +59,7 @@ def accepts( return False - def convert( + async def convert_async( self, file_stream: BinaryIO, stream_info: StreamInfo, @@ -89,7 +90,7 @@ def convert( title = slide.shapes.title - def get_shape_content(shape, **kwargs): + async def get_shape_content(shape, **kwargs): nonlocal md_content # Pictures if self._is_picture(shape): @@ -99,8 +100,10 @@ def get_shape_content(shape, **kwargs): alt_text = "" # Potentially generate a description using an LLM - llm_client = kwargs.get("llm_client") - llm_model = kwargs.get("llm_model") + llm_client: Optional[Union[AsyncOpenAI, OpenAI]] = kwargs.get( + "llm_client" + ) + llm_model: Optional[str] = kwargs.get("llm_model") if llm_client is not None and llm_model is not None: # Prepare a file_stream and stream_info for the image data image_filename = shape.image.filename @@ -117,12 +120,12 @@ def get_shape_content(shape, **kwargs): # Caption the image try: - llm_description = llm_caption( - image_stream, - image_stream_info, + llm_description = await llm_image_caption( client=llm_client, + file_stream=image_stream, model=llm_model, prompt=kwargs.get("llm_prompt"), + stream_info=image_stream_info, ) except Exception: # Unable to generate a description @@ -153,11 +156,13 @@ def get_shape_content(shape, **kwargs): # Tables if self._is_table(shape): - md_content += self._convert_table_to_markdown(shape.table, **kwargs) + md_content += await self._convert_table_to_markdown( + shape.table, **kwargs + ) # Charts if shape.has_chart: - md_content += self._convert_chart_to_markdown(shape.chart) + md_content += await self._convert_chart_to_markdown(shape.chart) # Text areas elif shape.has_text_frame: @@ -170,11 +175,11 @@ def get_shape_content(shape, **kwargs): if shape.shape_type == pptx.enum.shapes.MSO_SHAPE_TYPE.GROUP: sorted_shapes = sorted(shape.shapes, key=attrgetter("top", "left")) for subshape in sorted_shapes: - get_shape_content(subshape, **kwargs) + await get_shape_content(subshape, **kwargs) sorted_shapes = sorted(slide.shapes, key=attrgetter("top", "left")) for shape in sorted_shapes: - get_shape_content(shape, **kwargs) + await get_shape_content(shape, **kwargs) md_content = md_content.strip() @@ -200,7 +205,7 @@ def _is_table(self, shape): return True return False - def _convert_table_to_markdown(self, table, **kwargs): + async def _convert_table_to_markdown(self, table, **kwargs): # Write the table as HTML, then convert it to Markdown html_table = "" first_row = True @@ -216,11 +221,10 @@ def _convert_table_to_markdown(self, table, **kwargs): html_table += "
" return ( - self._html_converter.convert_string(html_table, **kwargs).markdown.strip() - + "\n" - ) + await self._html_converter.convert_string(html_table, **kwargs) + ).markdown.strip() + "\n" - def _convert_chart_to_markdown(self, chart): + async def _convert_chart_to_markdown(self, chart): try: md = "\n\n### Chart" if chart.has_title: diff --git a/packages/markitdown/src/markitdown/converters/_rss_converter.py b/packages/markitdown/src/markitdown/converters/_rss_converter.py index bec42484f..2f0c250a9 100644 --- a/packages/markitdown/src/markitdown/converters/_rss_converter.py +++ b/packages/markitdown/src/markitdown/converters/_rss_converter.py @@ -33,7 +33,7 @@ def __init__(self): super().__init__() self._kwargs = {} - def accepts( + async def accepts_async( self, file_stream: BinaryIO, stream_info: StreamInfo, @@ -81,7 +81,7 @@ def _feed_type(self, doc: Any) -> str | None: return "atom" return None - def convert( + async def convert_async( self, file_stream: BinaryIO, stream_info: StreamInfo, diff --git a/packages/markitdown/src/markitdown/converters/_transcribe_audio.py b/packages/markitdown/src/markitdown/converters/_transcribe_audio.py index d558e4629..b23ff37b3 100644 --- a/packages/markitdown/src/markitdown/converters/_transcribe_audio.py +++ b/packages/markitdown/src/markitdown/converters/_transcribe_audio.py @@ -1,3 +1,4 @@ +import asyncio import io import sys from typing import BinaryIO @@ -13,14 +14,18 @@ with warnings.catch_warnings(): warnings.filterwarnings("ignore", category=DeprecationWarning) warnings.filterwarnings("ignore", category=SyntaxWarning) - import speech_recognition as sr import pydub + import speech_recognition as sr except ImportError: # Preserve the error and stack trace for later _dependency_exc_info = sys.exc_info() -def transcribe_audio(file_stream: BinaryIO, *, audio_format: str = "wav") -> str: +async def transcribe_audio( + file_stream: BinaryIO, + *, + audio_format: str = "wav", +) -> str: # Check for installed dependencies if _dependency_exc_info is not None: raise MissingDependencyException( @@ -34,10 +39,13 @@ def transcribe_audio(file_stream: BinaryIO, *, audio_format: str = "wav") -> str if audio_format in ["wav", "aiff", "flac"]: audio_source = file_stream elif audio_format in ["mp3", "mp4"]: - audio_segment = pydub.AudioSegment.from_file(file_stream, format=audio_format) + # Run the audio conversion in a thread pool to avoid blocking + audio_segment = await asyncio.to_thread( + pydub.AudioSegment.from_file, file_stream, format=audio_format + ) audio_source = io.BytesIO() - audio_segment.export(audio_source, format="wav") + await asyncio.to_thread(audio_segment.export, audio_source, format="wav") audio_source.seek(0) else: raise ValueError(f"Unsupported audio format: {audio_format}") @@ -45,5 +53,7 @@ def transcribe_audio(file_stream: BinaryIO, *, audio_format: str = "wav") -> str recognizer = sr.Recognizer() with sr.AudioFile(audio_source) as source: audio = recognizer.record(source) - transcript = recognizer.recognize_google(audio).strip() + # Run the speech recognition in a thread pool to avoid blocking + transcript = await asyncio.to_thread(recognizer.recognize_google, audio) + transcript = transcript.strip() return "[No speech detected]" if transcript == "" else transcript diff --git a/packages/markitdown/src/markitdown/converters/_wikipedia_converter.py b/packages/markitdown/src/markitdown/converters/_wikipedia_converter.py index c20018659..48175b737 100644 --- a/packages/markitdown/src/markitdown/converters/_wikipedia_converter.py +++ b/packages/markitdown/src/markitdown/converters/_wikipedia_converter.py @@ -20,7 +20,7 @@ class WikipediaConverter(DocumentConverter): """Handle Wikipedia pages separately, focusing only on the main document content.""" - def accepts( + async def accepts_async( self, file_stream: BinaryIO, stream_info: StreamInfo, @@ -48,7 +48,7 @@ def accepts( # Not HTML content return False - def convert( + async def convert_async( self, file_stream: BinaryIO, stream_info: StreamInfo, diff --git a/packages/markitdown/src/markitdown/converters/_xlsx_converter.py b/packages/markitdown/src/markitdown/converters/_xlsx_converter.py index 4186ec773..866f43684 100644 --- a/packages/markitdown/src/markitdown/converters/_xlsx_converter.py +++ b/packages/markitdown/src/markitdown/converters/_xlsx_converter.py @@ -42,7 +42,7 @@ def __init__(self): super().__init__() self._html_converter = HtmlConverter() - def accepts( + async def accepts_async( self, file_stream: BinaryIO, stream_info: StreamInfo, @@ -60,7 +60,7 @@ def accepts( return False - def convert( + async def convert_async( self, file_stream: BinaryIO, stream_info: StreamInfo, @@ -86,11 +86,8 @@ def convert( md_content += f"## {s}\n" html_content = sheets[s].to_html(index=False) md_content += ( - self._html_converter.convert_string( - html_content, **kwargs - ).markdown.strip() - + "\n\n" - ) + await self._html_converter.convert_string(html_content, **kwargs) + ).markdown.strip() + "\n\n" return DocumentConverterResult(markdown=md_content.strip()) @@ -104,7 +101,7 @@ def __init__(self): super().__init__() self._html_converter = HtmlConverter() - def accepts( + async def accepts_async( self, file_stream: BinaryIO, stream_info: StreamInfo, @@ -122,7 +119,7 @@ def accepts( return False - def convert( + async def convert_async( self, file_stream: BinaryIO, stream_info: StreamInfo, @@ -148,10 +145,7 @@ def convert( md_content += f"## {s}\n" html_content = sheets[s].to_html(index=False) md_content += ( - self._html_converter.convert_string( - html_content, **kwargs - ).markdown.strip() - + "\n\n" - ) + await self._html_converter.convert_string(html_content, **kwargs) + ).markdown.strip() + "\n\n" return DocumentConverterResult(markdown=md_content.strip()) diff --git a/packages/markitdown/src/markitdown/converters/_youtube_converter.py b/packages/markitdown/src/markitdown/converters/_youtube_converter.py index c96e8f4f6..e984a0d84 100644 --- a/packages/markitdown/src/markitdown/converters/_youtube_converter.py +++ b/packages/markitdown/src/markitdown/converters/_youtube_converter.py @@ -1,3 +1,4 @@ +import asyncio import json import time import re @@ -37,7 +38,7 @@ class YouTubeConverter(DocumentConverter): """Handle YouTube specially, focusing on the video title, description, and transcript.""" - def accepts( + async def accepts_async( self, file_stream: BinaryIO, stream_info: StreamInfo, @@ -67,7 +68,7 @@ def accepts( # Not HTML content return False - def convert( + async def convert_async( self, file_stream: BinaryIO, stream_info: StreamInfo, @@ -151,7 +152,7 @@ def convert( params = parse_qs(parsed_url.query) # type: ignore if "v" in params and params["v"][0]: video_id = str(params["v"][0]) - transcript_list = ytt_api.list(video_id) + transcript_list = await asyncio.to_thread(ytt_api.list, video_id) languages = ["en"] for transcript in transcript_list: languages.append(transcript.language_code) @@ -161,7 +162,8 @@ def convert( "youtube_transcript_languages", languages ) # Retry the transcript fetching operation - transcript = self._retry_operation( + transcript = await asyncio.to_thread( + self._retry_operation, lambda: ytt_api.fetch( video_id, languages=youtube_transcript_languages ), @@ -170,9 +172,7 @@ def convert( ) if transcript: - transcript_text = " ".join( - [part.text for part in transcript] - ) # type: ignore + transcript_text = " ".join([part.text for part in transcript]) # type: ignore except Exception as e: # No transcript available if len(languages) == 1: diff --git a/packages/markitdown/src/markitdown/converters/_zip_converter.py b/packages/markitdown/src/markitdown/converters/_zip_converter.py index f87e6c890..3eab63b3e 100644 --- a/packages/markitdown/src/markitdown/converters/_zip_converter.py +++ b/packages/markitdown/src/markitdown/converters/_zip_converter.py @@ -66,7 +66,7 @@ def __init__( super().__init__() self._markitdown = markitdown - def accepts( + async def accepts_async( self, file_stream: BinaryIO, stream_info: StreamInfo, @@ -84,7 +84,7 @@ def accepts( return False - def convert( + async def convert_async( self, file_stream: BinaryIO, stream_info: StreamInfo, @@ -101,7 +101,7 @@ def convert( extension=os.path.splitext(name)[1], filename=os.path.basename(name), ) - result = self._markitdown.convert_stream( + result = await self._markitdown.convert_stream_async( stream=z_file_stream, stream_info=z_file_stream_info, )