feat: send PDFs as native file content parts for OpenAI

When a model declares `acceptedFileMimetypes` including
"application/pdf", PDFs are now sent as OpenAI `file` content parts
with base64 data instead of being wrapped in XML tags. This lets
OpenAI (and compatible providers) process PDFs natively.

Non-PDF files and models without `acceptedFileMimetypes` are
unaffected — existing text extraction behavior is preserved.

Depends on: huggingface#2189

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

Author: tessaherself

src/lib/server/endpoints/openai/endpointOai.ts

@@ -174,7 +174,12 @@ export async function endpointOai(
 		}) => {
 			// Format messages for the chat API, handling multimodal content if supported
 			let messagesOpenAI: OpenAI.Chat.Completions.ChatCompletionMessageParam[] =
-				await prepareMessagesWithFiles(messages, imageProcessor, isMultimodal ?? model.multimodal);
+				await prepareMessagesWithFiles(
+					messages,
+					imageProcessor,
+					isMultimodal ?? model.multimodal,
+					model.acceptedFileMimetypes
+				);
 
 			// Normalize preprompt and handle empty values
 			const normalizedPreprompt = typeof preprompt === "string" ? preprompt.trim() : "";

src/lib/server/textGeneration/mcp/runMcpFlow.ts

@@ -344,7 +344,8 @@ export async function* runMcpFlow({
 		let messagesOpenAI: ChatCompletionMessageParam[] = await prepareMessagesWithFiles(
 			messages,
 			imageProcessor,
-			mmEnabled
+			mmEnabled,
+			model.acceptedFileMimetypes
 		);
 		const toolPreprompt = buildToolPreprompt(oaTools);
 		const prepromptPieces: string[] = [];

src/lib/server/textGeneration/utils/prepareFiles.ts

@@ -4,33 +4,51 @@ import type { OpenAI } from "openai";
 import { TEXT_MIME_ALLOWLIST } from "$lib/constants/mime";
 import type { makeImageProcessor } from "$lib/server/endpoints/images";
 
+/** MIME types that OpenAI handles natively as file content parts */
+const NATIVE_FILE_MIMETYPES = ["application/pdf"] as const;
+
+function matchesMimeAllowlist(mime: string, allowlist: readonly string[]): boolean {
+	const normalizedMime = (mime || "").toLowerCase();
+	const [fileType, fileSubtype] = normalizedMime.split("/");
+	return allowlist.some((allowed) => {
+		const [type, subtype] = allowed.toLowerCase().split("/");
+		const typeOk = type === "*" || type === fileType;
+		const subOk = subtype === "*" || subtype === fileSubtype;
+		return typeOk && subOk;
+	});
+}
+
 /**
  * Prepare chat messages for OpenAI-compatible multimodal payloads.
  * - Processes images via the provided imageProcessor (resize/convert) when multimodal is enabled.
+ * - Sends PDFs as native file content parts when the model accepts them.
  * - Injects text-file content into the user message text.
  * - Leaves messages untouched when no files or multimodal disabled.
  */
 export async function prepareMessagesWithFiles(
 	messages: EndpointMessage[],
 	imageProcessor: ReturnType<typeof makeImageProcessor>,
-	isMultimodal: boolean
+	isMultimodal: boolean,
+	acceptedFileMimetypes?: string[]
 ): Promise<OpenAI.Chat.Completions.ChatCompletionMessageParam[]> {
 	return Promise.all(
 		messages.map(async (message) => {
 			if (message.from === "user" && message.files && message.files.length > 0) {
-				const { imageParts, textContent } = await prepareFiles(
+				const { imageParts, fileParts, textContent } = await prepareFiles(
 					imageProcessor,
 					message.files,
-					isMultimodal
+					isMultimodal,
+					acceptedFileMimetypes
 				);
 
 				let messageText = message.content;
 				if (textContent.length > 0) {
 					messageText = textContent + "\n\n" + message.content;
 				}
 
-				if (imageParts.length > 0 && isMultimodal) {
-					const parts = [{ type: "text" as const, text: messageText }, ...imageParts];
+				const multimodalParts = [...imageParts, ...fileParts];
+				if (multimodalParts.length > 0) {
+					const parts = [{ type: "text" as const, text: messageText }, ...multimodalParts];
 					return { role: message.from, content: parts };
 				}
 
@@ -44,22 +62,25 @@ export async function prepareMessagesWithFiles(
 async function prepareFiles(
 	imageProcessor: ReturnType<typeof makeImageProcessor>,
 	files: MessageFile[],
-	isMultimodal: boolean
+	isMultimodal: boolean,
+	acceptedFileMimetypes?: string[]
 ): Promise<{
 	imageParts: OpenAI.Chat.Completions.ChatCompletionContentPartImage[];
+	fileParts: OpenAI.Chat.Completions.ChatCompletionContentPart.File[];
 	textContent: string;
 }> {
 	const imageFiles = files.filter((file) => file.mime.startsWith("image/"));
-	const textFiles = files.filter((file) => {
-		const mime = (file.mime || "").toLowerCase();
-		const [fileType, fileSubtype] = mime.split("/");
-		return TEXT_MIME_ALLOWLIST.some((allowed) => {
-			const [type, subtype] = allowed.toLowerCase().split("/");
-			const typeOk = type === "*" || type === fileType;
-			const subOk = subtype === "*" || subtype === fileSubtype;
-			return typeOk && subOk;
-		});
-	});
+	const textFiles = files.filter((file) => matchesMimeAllowlist(file.mime, TEXT_MIME_ALLOWLIST));
+
+	// Files that the model accepts natively (e.g. PDFs via OpenAI's file content part)
+	const nativeFiles = files.filter(
+		(file) =>
+			!file.mime.startsWith("image/") &&
+			!matchesMimeAllowlist(file.mime, TEXT_MIME_ALLOWLIST) &&
+			acceptedFileMimetypes &&
+			matchesMimeAllowlist(file.mime, acceptedFileMimetypes) &&
+			matchesMimeAllowlist(file.mime, NATIVE_FILE_MIMETYPES)
+	);
 
 	let imageParts: OpenAI.Chat.Completions.ChatCompletionContentPartImage[] = [];
 	if (isMultimodal && imageFiles.length > 0) {
@@ -73,6 +94,17 @@ async function prepareFiles(
 		}));
 	}
 
+	// Send natively-supported files as OpenAI file content parts
+	const fileParts: OpenAI.Chat.Completions.ChatCompletionContentPart.File[] = nativeFiles.map(
+		(file) => ({
+			type: "file" as const,
+			file: {
+				filename: file.name,
+				file_data: `data:${file.mime};base64,${file.value}`,
+			},
+		})
+	);
+
 	let textContent = "";
 	if (textFiles.length > 0) {
 		const textParts = await Promise.all(
@@ -84,5 +116,5 @@ async function prepareFiles(
 		textContent = textParts.join("\n\n");
 	}
 
-	return { imageParts, textContent };
+	return { imageParts, fileParts, textContent };
 }