feat: Add image converters (deepset-ai#9628)

* Add image converters

* Fix tests

* Update haystack/components/converters/image/__init__.py

Co-authored-by: Stefano Fiorucci <stefanofiorucci@gmail.com>

---------

Co-authored-by: Stefano Fiorucci <stefanofiorucci@gmail.com>

Author: 2 people

docs/pydoc/config/image_converters_api.yml

@@ -0,0 +1,33 @@
+loaders:
+  - type: haystack_pydoc_tools.loaders.CustomPythonLoader
+    search_path: [../../../haystack/components/converters/image]
+    modules:
+      [
+        "document_to_image",
+        "file_to_document",
+        "file_to_image",
+        "pdf_to_image",
+      ]
+    ignore_when_discovered: ["__init__"]
+processors:
+  - type: filter
+    expression:
+    documented_only: true
+    do_not_filter_modules: false
+    skip_empty_modules: true
+  - type: smart
+  - type: crossref
+renderer:
+  type: haystack_pydoc_tools.renderers.ReadmeCoreRenderer
+  excerpt: Various converters to transform image data from one format to another.
+  category_slug: haystack-api
+  title: Image Converters
+  slug: image-converters-api
+  order: 21
+  markdown:
+    descriptive_class_title: false
+    classdef_code_block: false
+    descriptive_module_title: true
+    add_method_class_prefix: true
+    add_member_class_prefix: false
+    filename: image_converters_api.md

docs/pydoc/config_docusaurus/image_converters_api.yml

@@ -0,0 +1,31 @@
+loaders:
+  - type: haystack_pydoc_tools.loaders.CustomPythonLoader
+    search_path: [../../../haystack/components/converters/image]
+    modules:
+      [
+        "document_to_image",
+        "file_to_document",
+        "file_to_image",
+        "pdf_to_image",
+      ]
+    ignore_when_discovered: ["__init__"]
+processors:
+  - type: filter
+    expression:
+    documented_only: true
+    do_not_filter_modules: false
+    skip_empty_modules: true
+  - type: smart
+  - type: crossref
+renderer:
+  type: haystack_pydoc_tools.renderers.DocusaurusRenderer
+  description: Various converters to transform image data from one format to another.
+  title: Image Converters
+  id: image-converters-api
+  markdown:
+    descriptive_class_title: false
+    classdef_code_block: false
+    descriptive_module_title: true
+    add_method_class_prefix: true
+    add_member_class_prefix: false
+    filename: image_converters_api.md

haystack/components/converters/image/__init__.py

@@ -0,0 +1,23 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import sys
+from typing import TYPE_CHECKING
+
+from lazy_imports import LazyImporter
+
+_import_structure = {
+    "document_to_image": ["DocumentToImageContent"],
+    "file_to_document": ["ImageFileToDocument"],
+    "file_to_image": ["ImageFileToImageContent"],
+    "pdf_to_image": ["PDFToImageContent"],
+}
+
+if TYPE_CHECKING:
+    from .document_to_image import DocumentToImageContent
+    from .file_to_document import ImageFileToDocument
+    from .file_to_image import ImageFileToImageContent
+    from .pdf_to_image import PDFToImageContent
+else:
+    sys.modules[__name__] = LazyImporter(name=__name__, module_file=__file__, import_structure=_import_structure)

haystack/components/converters/image/document_to_image.py

@@ -0,0 +1,171 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+from typing import Dict, List, Literal, Optional, Tuple
+
+from haystack import Document, component, logging
+from haystack.components.converters.image.image_utils import (
+    _batch_convert_pdf_pages_to_images,
+    _encode_image_to_base64,
+    _extract_image_sources_info,
+    _PDFPageInfo,
+    pillow_import,
+    pypdfium2_import,
+)
+from haystack.dataclasses import ByteStream
+from haystack.dataclasses.image_content import ImageContent
+
+logger = logging.getLogger(__name__)
+
+
+@component
+class DocumentToImageContent:
+    """
+    Converts documents sourced from PDF and image files into ImageContents.
+
+    This component processes a list of documents and extracts visual content from supported file formats, converting
+    them into ImageContents that can be used for multimodal AI tasks. It handles both direct image files and PDF
+    documents by extracting specific pages as images.
+
+    Documents are expected to have metadata containing:
+    - The `file_path_meta_field` key with a valid file path that exists when combined with `root_path`
+    - A supported image format (MIME type must be one of the supported image types)
+    - For PDF files, a `page_number` key specifying which page to extract
+
+    ### Usage example
+        ```python
+        from haystack import Document
+        from haystack.components.image_converters.document_to_image import DocumentToImageContent
+
+        converter = DocumentToImageContent(
+            file_path_meta_field="file_path",
+            root_path="/data/documents",
+            detail="high",
+            size=(800, 600)
+        )
+
+        documents = [
+            Document(content="Optional description of image.jpg", meta={"file_path": "image.jpg"}),
+            Document(content="Text content of page 1 of doc.pdf", meta={"file_path": "doc.pdf", "page_number": 1})
+        ]
+
+        result = converter.run(documents)
+        image_contents = result["image_contents"]
+        # [ImageContent(
+        #    base64_image='/9j/4A...', mime_type='image/jpeg', detail='high', meta={'file_path': 'image.jpg'}
+        #  ),
+        #  ImageContent(
+        #    base64_image='/9j/4A...', mime_type='image/jpeg', detail='high',
+        #    meta={'page_number': 1, 'file_path': 'doc.pdf'}
+        #  )]
+        ```
+    """
+
+    def __init__(
+        self,
+        *,
+        file_path_meta_field: str = "file_path",
+        root_path: Optional[str] = None,
+        detail: Optional[Literal["auto", "high", "low"]] = None,
+        size: Optional[Tuple[int, int]] = None,
+    ):
+        """
+        Initialize the DocumentToImageContent component.
+
+        :param file_path_meta_field: The metadata field in the Document that contains the file path to the image or PDF.
+        :param root_path: The root directory path where document files are located. If provided, file paths in
+            document metadata will be resolved relative to this path. If None, file paths are treated as absolute paths.
+        :param detail: Optional detail level of the image (only supported by OpenAI). Can be "auto", "high", or "low".
+            This will be passed to the created ImageContent objects.
+        :param size: If provided, resizes the image to fit within the specified dimensions (width, height) while
+            maintaining aspect ratio. This reduces file size, memory usage, and processing time, which is beneficial
+            when working with models that have resolution constraints or when transmitting images to remote services.
+        """
+        pillow_import.check()
+        pypdfium2_import.check()
+
+        self.file_path_meta_field = file_path_meta_field
+        self.root_path = root_path or ""
+        self.detail = detail
+        self.size = size
+
+    @component.output_types(image_contents=List[Optional[ImageContent]])
+    def run(self, documents: List[Document]) -> Dict[str, List[Optional[ImageContent]]]:
+        """
+        Convert documents with image or PDF sources into ImageContent objects.
+
+        This method processes the input documents, extracting images from supported file formats and converting them
+        into ImageContent objects.
+
+        :param documents: A list of documents to process. Each document should have metadata containing at minimum
+            a 'file_path_meta_field' key. PDF documents additionally require a 'page_number' key to specify which
+            page to convert.
+
+        :returns:
+            Dictionary containing one key:
+            - "image_contents": ImageContents created from the processed documents. These contain base64-encoded image
+                data and metadata. The order corresponds to order of input documents.
+        :raises ValueError:
+            If any document is missing the required metadata keys, has an invalid file path, or has an unsupported
+            MIME type. The error message will specify which document and what information is missing or incorrect.
+        """
+        if not documents:
+            return {"image_contents": []}
+
+        images_source_info = _extract_image_sources_info(
+            documents=documents, file_path_meta_field=self.file_path_meta_field, root_path=self.root_path
+        )
+
+        image_contents: List[Optional[ImageContent]] = [None] * len(documents)
+
+        pdf_page_infos: List[_PDFPageInfo] = []
+
+        for doc_idx, image_source_info in enumerate(images_source_info):
+            mime_type = image_source_info["mime_type"]
+            path = image_source_info["path"]
+            if mime_type == "application/pdf":
+                # Store PDF documents for later processing
+                page_number = image_source_info.get("page_number")
+                assert page_number is not None  # checked in _extract_image_sources_info but mypy doesn't know that
+                pdf_page_info: _PDFPageInfo = {"doc_idx": doc_idx, "path": path, "page_number": page_number}
+                pdf_page_infos.append(pdf_page_info)
+            else:
+                # Process images directly
+                bytestream = ByteStream.from_file_path(filepath=path, mime_type=mime_type)
+                _, base64_image = _encode_image_to_base64(bytestream=bytestream, size=self.size)
+                image_contents[doc_idx] = ImageContent(
+                    base64_image=base64_image,
+                    mime_type=mime_type,
+                    detail=self.detail,
+                    meta={"file_path": documents[doc_idx].meta[self.file_path_meta_field]},
+                )
+
+        # efficiently convert PDF pages to images: each PDF is opened and processed only once
+        pdf_page_infos_by_doc_idx: Dict[int, _PDFPageInfo] = {
+            pdf_page_info["doc_idx"]: pdf_page_info for pdf_page_info in pdf_page_infos
+        }
+        pdf_images_by_doc_idx = _batch_convert_pdf_pages_to_images(
+            pdf_page_infos=pdf_page_infos, size=self.size, return_base64=True
+        )
+        for doc_idx, base64_pdf_image in pdf_images_by_doc_idx.items():
+            meta = {
+                "file_path": documents[doc_idx].meta[self.file_path_meta_field],
+                "page_number": pdf_page_infos_by_doc_idx[doc_idx]["page_number"],
+            }
+            # we know that base64_pdf_image is a string because we set return_base64=True but mypy doesn't know that
+            assert isinstance(base64_pdf_image, str)
+            image_contents[doc_idx] = ImageContent(
+                base64_image=base64_pdf_image, mime_type="image/jpeg", detail=self.detail, meta=meta
+            )
+
+        none_image_contents_doc_ids = [
+            documents[doc_idx].id for doc_idx, image_content in enumerate(image_contents) if image_content is None
+        ]
+        if none_image_contents_doc_ids:
+            logger.warning(
+                "Conversion failed for some documents. Their output will be None. "
+                f"Document IDs: {none_image_contents_doc_ids}"
+            )
+
+        return {"image_contents": image_contents}

haystack/components/converters/image/file_to_document.py

@@ -0,0 +1,100 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import os
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Union
+
+from haystack import Document, component, logging
+from haystack.components.converters.utils import get_bytestream_from_source, normalize_metadata
+from haystack.dataclasses import ByteStream
+
+logger = logging.getLogger(__name__)
+
+
+@component
+class ImageFileToDocument:
+    """
+    Converts image file references into empty Document objects with associated metadata.
+
+    This component is useful in pipelines where image file paths need to be wrapped in `Document` objects to be
+    processed by downstream components such as the `SentenceTransformersImageDocumentEmbedder`.
+
+    It does **not** extract any content from the image files, instead it creates `Document` objects with `None` as
+    their content and attaches metadata such as file path and any user-provided values.
+
+    ### Usage example
+    ```python
+    from haystack.components.converters.image import ImageFileToDocument
+
+    converter = ImageFileToDocument()
+
+    sources = ["image.jpg", "another_image.png"]
+
+    result = converter.run(sources=sources)
+    documents = result["documents"]
+
+    print(documents)
+
+    # [Document(id=..., meta: {'file_path': 'image.jpg'}),
+    # Document(id=..., meta: {'file_path': 'another_image.png'})]
+    ```
+    """
+
+    def __init__(self, *, store_full_path: bool = False):
+        """
+        Initialize the ImageFileToDocument component.
+
+        :param store_full_path:
+            If True, the full path of the file is stored in the metadata of the document.
+            If False, only the file name is stored.
+        """
+        self.store_full_path = store_full_path
+
+    @component.output_types(documents=List[Document])
+    def run(
+        self,
+        *,
+        sources: List[Union[str, Path, ByteStream]],
+        meta: Optional[Union[Dict[str, Any], List[Dict[str, Any]]]] = None,
+    ) -> Dict[str, List[Document]]:
+        """
+        Convert image files into empty Document objects with metadata.
+
+        This method accepts image file references (as file paths or ByteStreams) and creates `Document` objects
+        without content. These documents are enriched with metadata derived from the input source and optional
+        user-provided metadata.
+
+        :param sources:
+            List of file paths or ByteStream objects to convert.
+        :param meta:
+            Optional metadata to attach to the documents.
+            This value can be a list of dictionaries or a single dictionary.
+            If it's a single dictionary, its content is added to the metadata of all produced documents.
+            If it's a list, its length must match the number of sources, as they are zipped together.
+            For ByteStream objects, their `meta` is added to the output documents.
+
+        :returns:
+            A dictionary containing:
+            - `documents`: A list of `Document` objects with empty content and associated metadata.
+        """
+
+        documents = []
+        meta_list = normalize_metadata(meta, sources_count=len(sources))
+
+        for source, metadata in zip(sources, meta_list):
+            try:
+                bytestream = get_bytestream_from_source(source)
+            except Exception as e:
+                logger.warning("Could not read {source}. Skipping it. Error: {error}", source=source, error=e)
+                continue
+
+            merged_metadata = {**bytestream.meta, **metadata}
+
+            if not self.store_full_path and (file_path := bytestream.meta.get("file_path")):
+                merged_metadata["file_path"] = os.path.basename(file_path)
+            document = Document(content=None, meta=merged_metadata)
+            documents.append(document)
+
+        return {"documents": documents}