feat(api): add optional MarkItDown OCR support (#2145)

MarkItDown advertises image extensions, but without OCR config it can
fail screenshots or scanned images with low-level no-content errors.

Add server-level MarkItDown OCR config that is off by default and
independent from HINDSIGHT_API_LLM_*. When OCR is enabled, the OCR API
key, base URL, and model are required explicitly.

Wire those settings into MarkItDown's llm_client support with a built-in
OCR prompt. Image uploads now fail fast with actionable errors when OCR
is disabled or required settings are missing.

Docs and front-end copy explain that image OCR depends on server config
and requires an OpenAI-compatible OCR/vision endpoint.

Closes #927

Author: Sanderhoff-alt

.env.example

@@ -87,6 +87,18 @@ HINDSIGHT_API_LOG_LEVEL=info
 # korean_lindera/lindera(korean), ngram(min,max), edge_ngram(min,max)
 # HINDSIGHT_API_TEXT_SEARCH_EXTENSION_PG_SEARCH_TOKENIZER=
 
+# File Parser (Optional - uses markitdown by default)
+# HINDSIGHT_API_FILE_PARSER=markitdown
+# Enable image OCR for MarkItDown using an OpenAI-compatible OCR/vision endpoint.
+# These OCR settings are independent from HINDSIGHT_API_LLM_* because MarkItDown
+# uses the OpenAI SDK directly and requires Chat Completions image input support.
+# When OCR is enabled, API_KEY, BASE_URL, and MODEL are required.
+# HINDSIGHT_API_FILE_PARSER_MARKITDOWN_OCR_ENABLED=false
+# HINDSIGHT_API_FILE_PARSER_MARKITDOWN_OCR_API_KEY=
+# HINDSIGHT_API_FILE_PARSER_MARKITDOWN_OCR_BASE_URL=
+# HINDSIGHT_API_FILE_PARSER_MARKITDOWN_OCR_MODEL=
+# HINDSIGHT_API_FILE_PARSER_MARKITDOWN_OCR_PROMPT=
+
 # Embeddings Configuration (Optional - uses local by default)
 # Provider: "local" (default), "onnx", "tei", "openai", "cohere", "google", "openrouter", "zeroentropy", "litellm", or "litellm-sdk"
 # HINDSIGHT_API_EMBEDDINGS_PROVIDER=local

hindsight-api-slim/hindsight_api/api/http.py

@@ -6755,7 +6755,7 @@ async def api_retain(
         description="Upload files (PDF, DOCX, etc.), convert them to markdown, and retain as memories.\n\n"
         "This endpoint handles file upload, conversion, and memory creation in a single operation.\n\n"
         "**Features:**\n"
-        "- Supports PDF, DOCX, PPTX, XLSX, images (with OCR), audio (with transcription)\n"
+        "- Supports PDF, DOCX, PPTX, XLSX, images (parser-dependent OCR), audio (with transcription)\n"
         "- Automatic file-to-markdown conversion using pluggable parsers\n"
         "- Files stored in object storage (PostgreSQL by default, S3 for production)\n"
         "- Each file becomes a separate document with optional metadata/tags\n"

hindsight-api-slim/hindsight_api/config.py

@@ -424,6 +424,11 @@ def normalize_config_dict(config: dict[str, Any]) -> dict[str, Any]:
 ENV_FILE_STORAGE_AZURE_ACCOUNT_KEY = "HINDSIGHT_API_FILE_STORAGE_AZURE_ACCOUNT_KEY"
 ENV_FILE_PARSER = "HINDSIGHT_API_FILE_PARSER"
 ENV_FILE_PARSER_ALLOWLIST = "HINDSIGHT_API_FILE_PARSER_ALLOWLIST"
+ENV_FILE_PARSER_MARKITDOWN_OCR_ENABLED = "HINDSIGHT_API_FILE_PARSER_MARKITDOWN_OCR_ENABLED"
+ENV_FILE_PARSER_MARKITDOWN_OCR_API_KEY = "HINDSIGHT_API_FILE_PARSER_MARKITDOWN_OCR_API_KEY"
+ENV_FILE_PARSER_MARKITDOWN_OCR_BASE_URL = "HINDSIGHT_API_FILE_PARSER_MARKITDOWN_OCR_BASE_URL"
+ENV_FILE_PARSER_MARKITDOWN_OCR_MODEL = "HINDSIGHT_API_FILE_PARSER_MARKITDOWN_OCR_MODEL"
+ENV_FILE_PARSER_MARKITDOWN_OCR_PROMPT = "HINDSIGHT_API_FILE_PARSER_MARKITDOWN_OCR_PROMPT"
 ENV_FILE_PARSER_IRIS_TOKEN = "HINDSIGHT_API_FILE_PARSER_IRIS_TOKEN"
 ENV_FILE_PARSER_IRIS_ORG_ID = "HINDSIGHT_API_FILE_PARSER_IRIS_ORG_ID"
 ENV_FILE_PARSER_LLAMA_PARSE_API_KEY = "HINDSIGHT_API_FILE_PARSER_LLAMA_PARSE_API_KEY"
@@ -847,6 +852,10 @@ def _parse_strategy_boosts(raw: str | None) -> dict[str, str]:
 DEFAULT_FILE_STORAGE_TYPE = "native"  # PostgreSQL BYTEA storage
 DEFAULT_FILE_PARSER = "markitdown"  # Default parser fallback chain (comma-separated, e.g. "iris,markitdown")
 DEFAULT_FILE_PARSER_ALLOWLIST = None  # Allowlist of parsers clients may request (None = all registered parsers)
+DEFAULT_FILE_PARSER_MARKITDOWN_OCR_ENABLED = False
+DEFAULT_FILE_PARSER_MARKITDOWN_OCR_PROMPT = """You are a precise OCR transcription engine.
+
+Transcribe only the visible text in the image. Do not describe the image, summarize it, translate it, infer missing content, or add commentary. Preserve the original language, wording, numbers, punctuation, capitalization, and reading order. Reconstruct headings, lists, key-value fields, stamps, and tables as clean Markdown when the layout is clear. If text is unreadable or uncertain, write [unclear] for that span. Return only the extracted Markdown."""
 DEFAULT_FILE_CONVERSION_MAX_BATCH_SIZE_MB = 100  # Max total batch size in MB (all files combined)
 DEFAULT_FILE_CONVERSION_MAX_BATCH_SIZE = 10  # Max files per batch upload
 DEFAULT_ENABLE_FILE_UPLOAD_API = True  # Enable file upload endpoint
@@ -1676,6 +1685,11 @@ class HindsightConfig:
     embeddings_zeroentropy_encoding_format: str = DEFAULT_EMBEDDINGS_ZEROENTROPY_ENCODING_FORMAT
     embeddings_zeroentropy_batch_size: int = DEFAULT_EMBEDDINGS_ZEROENTROPY_BATCH_SIZE
     embeddings_zeroentropy_latency: str | None = DEFAULT_EMBEDDINGS_ZEROENTROPY_LATENCY
+    file_parser_markitdown_ocr_enabled: bool = DEFAULT_FILE_PARSER_MARKITDOWN_OCR_ENABLED
+    file_parser_markitdown_ocr_api_key: str | None = None
+    file_parser_markitdown_ocr_base_url: str | None = None
+    file_parser_markitdown_ocr_model: str | None = None
+    file_parser_markitdown_ocr_prompt: str = DEFAULT_FILE_PARSER_MARKITDOWN_OCR_PROMPT
 
     # Class-level sets for configuration categorization
 
@@ -1716,6 +1730,8 @@ class HindsightConfig:
         "file_storage_gcs_service_account_key",
         "file_storage_azure_account_key",
         # File parser credentials
+        "file_parser_markitdown_ocr_api_key",
+        "file_parser_markitdown_ocr_base_url",
         "file_parser_iris_token",
         "file_parser_llama_parse_api_key",
     }
@@ -2424,6 +2440,18 @@ def from_env(cls) -> "HindsightConfig":
             file_parser_allowlist=_parse_str_list(os.getenv(ENV_FILE_PARSER_ALLOWLIST))
             if os.getenv(ENV_FILE_PARSER_ALLOWLIST)
             else None,
+            file_parser_markitdown_ocr_enabled=os.getenv(
+                ENV_FILE_PARSER_MARKITDOWN_OCR_ENABLED,
+                str(DEFAULT_FILE_PARSER_MARKITDOWN_OCR_ENABLED),
+            ).lower()
+            in ("1", "true", "yes", "on"),
+            file_parser_markitdown_ocr_api_key=os.getenv(ENV_FILE_PARSER_MARKITDOWN_OCR_API_KEY) or None,
+            file_parser_markitdown_ocr_base_url=os.getenv(ENV_FILE_PARSER_MARKITDOWN_OCR_BASE_URL) or None,
+            file_parser_markitdown_ocr_model=os.getenv(ENV_FILE_PARSER_MARKITDOWN_OCR_MODEL) or None,
+            file_parser_markitdown_ocr_prompt=os.getenv(
+                ENV_FILE_PARSER_MARKITDOWN_OCR_PROMPT,
+                DEFAULT_FILE_PARSER_MARKITDOWN_OCR_PROMPT,
+            ),
             file_parser_iris_token=os.getenv(ENV_FILE_PARSER_IRIS_TOKEN) or None,
             file_parser_iris_org_id=os.getenv(ENV_FILE_PARSER_IRIS_ORG_ID) or None,
             file_parser_llama_parse_api_key=os.getenv(ENV_FILE_PARSER_LLAMA_PARSE_API_KEY) or None,

hindsight-api-slim/hindsight_api/engine/memory_engine.py

@@ -2755,7 +2755,15 @@ async def _init_connection(conn: asyncpg.Connection) -> None:
 
         self._parser_registry = FileParserRegistry()
         try:
-            self._parser_registry.register(MarkitdownParser())
+            self._parser_registry.register(
+                MarkitdownParser(
+                    ocr_enabled=config.file_parser_markitdown_ocr_enabled,
+                    ocr_api_key=config.file_parser_markitdown_ocr_api_key,
+                    ocr_base_url=config.file_parser_markitdown_ocr_base_url,
+                    ocr_model=config.file_parser_markitdown_ocr_model,
+                    ocr_prompt=config.file_parser_markitdown_ocr_prompt,
+                )
+            )
             logger.debug("Registered markitdown parser")
         except ImportError:
             logger.warning("markitdown not available - file parsing disabled")

hindsight-api-slim/hindsight_api/engine/parsers/markitdown.py

@@ -3,43 +3,116 @@
 import asyncio
 import logging
 import tempfile
+from dataclasses import dataclass
 from pathlib import Path
 
+from hindsight_api.config import DEFAULT_FILE_PARSER_MARKITDOWN_OCR_PROMPT
+
 from .base import FileParser
 
 logger = logging.getLogger(__name__)
 
 
+@dataclass(frozen=True)
+class MarkitdownOcrOptions:
+    """OpenAI-compatible OCR options passed through to MarkItDown."""
+
+    # Keep this typed as object so the OpenAI SDK import stays lazy for non-OCR users.
+    llm_client: object
+    llm_model: str
+    llm_prompt: str
+
+
 class MarkitdownParser(FileParser):
     """
     Markitdown file parser.
 
     Uses Microsoft's markitdown library to convert various file formats
-    to markdown including PDF, Office docs, images (via OCR), audio, HTML.
+    to markdown including PDF, Office docs, images with optional OCR,
+    audio, HTML.
 
     Supported formats:
     - PDF (.pdf)
     - Word (.docx, .doc)
     - PowerPoint (.pptx, .ppt)
     - Excel (.xlsx, .xls)
-    - Images (.jpg, .jpeg, .png) - with OCR
+    - Images (.jpg, .jpeg, .png) - optional OCR
     - HTML (.html, .htm)
     - Text (.txt, .md)
     - Audio (.mp3, .wav) - with transcription
     """
 
-    def __init__(self):
+    def __init__(
+        self,
+        *,
+        ocr_enabled: bool = False,
+        ocr_api_key: str | None = None,
+        ocr_base_url: str | None = None,
+        ocr_model: str | None = None,
+        ocr_prompt: str | None = None,
+    ):
         """Initialize markitdown parser."""
         # Lazy import to avoid requiring markitdown for all users
         try:
             from markitdown import MarkItDown
-
-            self._markitdown = MarkItDown()
         except ImportError as e:
             raise ImportError(
                 "markitdown package is required for file parsing. Install with: pip install markitdown"
             ) from e
 
+        self._ocr_enabled = ocr_enabled
+        if ocr_enabled:
+            ocr_options = self._build_ocr_options(
+                api_key=ocr_api_key,
+                base_url=ocr_base_url,
+                model=ocr_model,
+                prompt=ocr_prompt,
+            )
+            self._markitdown = MarkItDown(
+                llm_client=ocr_options.llm_client,
+                llm_model=ocr_options.llm_model,
+                llm_prompt=ocr_options.llm_prompt,
+            )
+        else:
+            self._markitdown = MarkItDown()
+
+    def _build_ocr_options(
+        self,
+        *,
+        api_key: str | None,
+        base_url: str | None,
+        model: str | None,
+        prompt: str | None,
+    ) -> MarkitdownOcrOptions:
+        """Build MarkItDown options for OpenAI-compatible image OCR."""
+        if not model or not model.strip():
+            raise ValueError(
+                "Markitdown OCR is enabled but no model is configured. "
+                "Set HINDSIGHT_API_FILE_PARSER_MARKITDOWN_OCR_MODEL to an OpenAI-compatible OCR/vision model "
+                "with image-input support."
+            )
+        if not api_key:
+            raise ValueError(
+                "Markitdown OCR is enabled but no API key is configured. "
+                "Set HINDSIGHT_API_FILE_PARSER_MARKITDOWN_OCR_API_KEY."
+            )
+        if not base_url or not base_url.strip():
+            raise ValueError(
+                "Markitdown OCR is enabled but no base URL is configured. "
+                "Set HINDSIGHT_API_FILE_PARSER_MARKITDOWN_OCR_BASE_URL to an OpenAI-compatible OCR/vision endpoint."
+            )
+
+        try:
+            from openai import OpenAI
+        except ImportError as e:
+            raise RuntimeError("openai package is required when Markitdown OCR is enabled.") from e
+
+        return MarkitdownOcrOptions(
+            llm_client=OpenAI(api_key=api_key, base_url=base_url.strip()),
+            llm_model=model.strip(),
+            llm_prompt=prompt or DEFAULT_FILE_PARSER_MARKITDOWN_OCR_PROMPT,
+        )
+
     async def convert(self, file_data: bytes, filename: str) -> str:
         """Parse file to markdown using markitdown."""
         # markitdown is synchronous, so we run it in executor to avoid blocking
@@ -48,6 +121,13 @@ async def convert(self, file_data: bytes, filename: str) -> str:
 
     def _convert_sync(self, file_data: bytes, filename: str) -> str:
         """Synchronous parsing (runs in thread pool)."""
+        if self._is_image_file(filename) and not self._ocr_enabled:
+            raise RuntimeError(
+                "Image OCR is not enabled for the markitdown parser. "
+                "Set HINDSIGHT_API_FILE_PARSER_MARKITDOWN_OCR_ENABLED=true and configure an OpenAI-compatible "
+                "OCR/vision endpoint with image-input support, or choose an OCR-capable parser."
+            )
+
         # Write to temp file (markitdown requires file path)
         with tempfile.NamedTemporaryFile(suffix=Path(filename).suffix, delete=False) as tmp:
             tmp.write(file_data)
@@ -73,6 +153,11 @@ def _convert_sync(self, file_data: bytes, filename: str) -> str:
             except Exception:
                 pass
 
+    @staticmethod
+    def _is_image_file(filename: str) -> bool:
+        """Return whether the file type needs OCR to extract useful text."""
+        return Path(filename).suffix.lower() in {".jpg", ".jpeg", ".png"}
+
     def supports(self, filename: str, content_type: str | None = None) -> bool:
         """Check if markitdown supports this file type."""
         # Supported extensions (from markitdown docs)
@@ -85,7 +170,7 @@ def supports(self, filename: str, content_type: str | None = None) -> bool:
             ".ppt",
             ".xlsx",
             ".xls",
-            # Images (with OCR)
+            # Images (optional OCR)
             ".jpg",
             ".jpeg",
             ".png",

hindsight-api-slim/tests/test_config_validation.py

@@ -452,6 +452,53 @@ def test_llm_output_language_empty_string_is_unset(monkeypatch):
     assert config.llm_output_language is None
 
 
+def test_markitdown_ocr_defaults_disabled(monkeypatch):
+    from hindsight_api.config import HindsightConfig
+
+    monkeypatch.setenv("HINDSIGHT_API_LLM_PROVIDER", "mock")
+
+    config = HindsightConfig.from_env()
+    assert config.file_parser_markitdown_ocr_enabled is False
+
+
+def test_markitdown_ocr_does_not_fall_back_to_main_llm_config(monkeypatch):
+    from hindsight_api.config import DEFAULT_FILE_PARSER_MARKITDOWN_OCR_PROMPT, HindsightConfig
+
+    monkeypatch.setenv("HINDSIGHT_API_FILE_PARSER_MARKITDOWN_OCR_ENABLED", "true")
+    monkeypatch.setenv("HINDSIGHT_API_LLM_PROVIDER", "anthropic")
+    monkeypatch.setenv("HINDSIGHT_API_LLM_API_KEY", "main-key")
+    monkeypatch.setenv("HINDSIGHT_API_LLM_BASE_URL", "https://main.example/v1")
+    monkeypatch.setenv("HINDSIGHT_API_LLM_MODEL", "main-vision-model")
+
+    config = HindsightConfig.from_env()
+    assert config.file_parser_markitdown_ocr_enabled is True
+    assert config.file_parser_markitdown_ocr_api_key is None
+    assert config.file_parser_markitdown_ocr_base_url is None
+    assert config.file_parser_markitdown_ocr_model is None
+    assert config.file_parser_markitdown_ocr_prompt == DEFAULT_FILE_PARSER_MARKITDOWN_OCR_PROMPT
+
+
+def test_markitdown_ocr_uses_explicit_config(monkeypatch):
+    from hindsight_api.config import HindsightConfig
+
+    monkeypatch.setenv("HINDSIGHT_API_FILE_PARSER_MARKITDOWN_OCR_ENABLED", "true")
+    monkeypatch.setenv("HINDSIGHT_API_FILE_PARSER_MARKITDOWN_OCR_API_KEY", "parser-key")
+    monkeypatch.setenv("HINDSIGHT_API_FILE_PARSER_MARKITDOWN_OCR_BASE_URL", "https://parser.example/v1")
+    monkeypatch.setenv("HINDSIGHT_API_FILE_PARSER_MARKITDOWN_OCR_MODEL", "parser-vision-model")
+    monkeypatch.setenv("HINDSIGHT_API_FILE_PARSER_MARKITDOWN_OCR_PROMPT", "Extract this document exactly.")
+    monkeypatch.setenv("HINDSIGHT_API_LLM_PROVIDER", "mock")
+    monkeypatch.setenv("HINDSIGHT_API_LLM_API_KEY", "main-key")
+    monkeypatch.setenv("HINDSIGHT_API_LLM_BASE_URL", "https://main.example/v1")
+    monkeypatch.setenv("HINDSIGHT_API_LLM_MODEL", "main-vision-model")
+
+    config = HindsightConfig.from_env()
+    assert config.file_parser_markitdown_ocr_enabled is True
+    assert config.file_parser_markitdown_ocr_api_key == "parser-key"
+    assert config.file_parser_markitdown_ocr_base_url == "https://parser.example/v1"
+    assert config.file_parser_markitdown_ocr_model == "parser-vision-model"
+    assert config.file_parser_markitdown_ocr_prompt == "Extract this document exactly."
+
+
 def test_llm_reasoning_effort_defaults_to_low(monkeypatch):
     from hindsight_api.config import HindsightConfig