fix(responses): preserve image inputs with placeholder

Author: Marvae

src/routes/responses/utils.ts

@@ -80,6 +80,26 @@ export const hasVisionInput = (payload: ResponsesPayload): boolean => {
 const DATA_URL_PREFIX = "data:"
 const BASE64_MARKER = ";base64,"
 const IMAGE_MEDIA_TYPE_PATTERN = /^image\/[a-zA-Z0-9.+-]+$/
+// Static 96x32 PNG reading "Image too large / Redacted".
+const REDACTED_IMAGE_PLACEHOLDER_DATA_URL =
+  "data:image/png;base64,"
+  + [
+    "iVBORw0KGgoAAAANSUhEUgAAAGAAAAAgCAMAAADaHo1mAAADAFBMVEX///8fKTfR1dsAAAAAAAAAAAAAAAAAAAAAAAAAAAAA",
+    "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA",
+    "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA",
+    "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA",
+    "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA",
+    "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA",
+    "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA",
+    "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA",
+    "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA",
+    "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA",
+    "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA",
+    "AAAAAAAAAAAAAAAAAAAAAACae8QWAAAAvElEQVR42u1WixKAIAhj/f9Hdz2BXJiVed3pVSYtpgwsGSo3GaRq6wSd4F8EyIJx",
+    "ydSUAMB8il51sHT2fiVQu8czguQwXWAyFvswIJhmoS9gmzYlcFiHj1aAgzcJVgCyguYhAhNZmMhYQZs1EJnnIAqKiuHjSrZT",
+    "ucSQ4s8JkKDDIYr3IuR8vEWgqroKP9b1bYKk2wfgeVmqATQLXdXamsXdEKkz3QXEEeTTuWWImMhW6qci94/+hwSVf99HqVoD",
+    "OAuj2SEAAAAASUVORK5CYII=",
+  ].join("")
 
 export const sanitizeOversizedInputImages = (
   payload: ResponsesPayload,
@@ -97,7 +117,7 @@ export const sanitizeOversizedInputImages = (
   let count = 0
   for (const image of collectInputImageDataUrls(payload.input)) {
     if (limit !== undefined && image.decodedBytes > limit) {
-      replaceInputImageWithText(image, `max ${limit} bytes`)
+      replaceInputImageWithPlaceholder(image)
       count += 1
     }
   }
@@ -112,7 +132,7 @@ export const sanitizeAllInputImages = (payload: ResponsesPayload): number => {
 
   let count = 0
   for (const image of collectInputImageDataUrls(payload.input)) {
-    replaceInputImageWithText(image, "upstream rejected payload as too large")
+    replaceInputImageWithPlaceholder(image)
     count += 1
   }
   return count
@@ -204,15 +224,12 @@ const getBase64DecodedByteLength = (
   return Math.max(0, Math.floor((base64Length * 3) / 4) - padding)
 }
 
-const replaceInputImageWithText = (
-  image: InputImageDataUrl,
-  reason: string,
-): void => {
-  image.record.type = "input_text"
-  image.record.text = `[omitted input image: ${image.mediaType}, ${image.decodedBytes} bytes, ${reason}]`
-  delete image.record.image_url
+const replaceInputImageWithPlaceholder = (image: InputImageDataUrl): void => {
+  image.record.type = "input_image"
+  image.record.image_url = REDACTED_IMAGE_PLACEHOLDER_DATA_URL
+  image.record.detail = "low"
+  delete image.record.text
   delete image.record.file_id
-  delete image.record.detail
 }
 
 export const resolveResponsesCompactThreshold = (

tests/responses-handler.test.ts

@@ -313,18 +313,20 @@ describe("responses handler token usage", () => {
 
     expect(response.status).toBe(200)
     expect(createResponses).toHaveBeenCalledTimes(1)
-    expect(createResponses.mock.calls[0][0].input).toEqual([
-      {
-        content: [
-          { text: "look", type: "input_text" },
-          {
-            text: "[omitted input image: image/png, 12 bytes, max 8 bytes]",
-            type: "input_text",
-          },
-        ],
-        role: "user",
-      },
-    ])
+    const image = (
+      createResponses.mock.calls[0][0].input as Array<{
+        content: Array<{
+          detail?: string
+          image_url?: string
+          text?: string
+          type: string
+        }>
+      }>
+    )[0].content[1]
+    expect(image.type).toBe("input_image")
+    expect(image.detail).toBe("low")
+    expect(image.image_url?.startsWith("data:image/png;base64,")).toBe(true)
+    expect(image.text).toBeUndefined()
   })
 
   test("preserves multiple input images before forwarding to Copilot Responses", async () => {
@@ -446,19 +448,24 @@ describe("responses handler token usage", () => {
 
     expect(response.status).toBe(200)
     expect(createResponses).toHaveBeenCalledTimes(2)
-    expect(createResponses.mock.calls[1][0].input).toEqual([
-      {
-        content: [
-          { text: "look", type: "input_text" },
-          {
-            text: "[omitted input image: image/png, 6 bytes, upstream rejected payload as too large]",
-            type: "input_text",
-          },
-        ],
-        role: "user",
-      },
-    ])
-    expect(createResponses.mock.calls[1][1]?.vision).toBe(false)
+    const retryImage = (
+      createResponses.mock.calls[1][0].input as Array<{
+        content: Array<{
+          detail?: string
+          image_url?: string
+          text?: string
+          type: string
+        }>
+      }>
+    )[0].content[1]
+    expect(retryImage.type).toBe("input_image")
+    expect(retryImage.detail).toBe("low")
+    expect(retryImage.image_url?.startsWith("data:image/png;base64,")).toBe(
+      true,
+    )
+    expect(retryImage.image_url).not.toBe(imageUrl)
+    expect(retryImage.text).toBeUndefined()
+    expect(createResponses.mock.calls[1][1]?.vision).toBe(true)
   })
 
   test("records usage from failed streaming responses and falls back to interaction id", async () => {

tests/responses-image-sanitizer.test.ts

@@ -28,24 +28,27 @@ const makePayload = (imageUrl: string): ResponsesPayload =>
   }) as unknown as ResponsesPayload
 
 describe("sanitizeOversizedInputImages", () => {
-  test("replaces oversized input images with text markers", () => {
+  test("replaces oversized input images with placeholder images", () => {
     const payload = makePayload(tinyPngDataUrl)
 
     const sanitized = sanitizeOversizedInputImages(payload, 67)
 
     expect(sanitized).toBe(1)
-    expect(payload.input).toEqual([
-      {
-        content: [
-          { text: "look", type: "input_text" },
-          {
-            text: "[omitted input image: image/png, 68 bytes, max 67 bytes]",
-            type: "input_text",
-          },
-        ],
-        role: "user",
-      },
-    ])
+    const image = (
+      payload.input as Array<{
+        content: Array<{
+          detail?: string
+          image_url?: string
+          text?: string
+          type: string
+        }>
+      }>
+    )[0].content[1]
+    expect(image.type).toBe("input_image")
+    expect(image.detail).toBe("low")
+    expect(image.image_url?.startsWith("data:image/png;base64,")).toBe(true)
+    expect(image.image_url).not.toBe(tinyPngDataUrl)
+    expect(image.text).toBeUndefined()
   })
 
   test("keeps input images within the model size limit", () => {
@@ -64,20 +67,22 @@ describe("sanitizeOversizedInputImages", () => {
     ).toEqual({ detail: "low", image_url: imageUrl, type: "input_image" })
   })
 
-  test("removes all input images for a retry after payload rejection", () => {
+  test("replaces all input images for a retry after payload rejection", () => {
+    const firstImageUrl = imageDataUrl(1024)
+    const secondImageUrl = imageDataUrl(1024)
     const payload = {
       input: [
         {
           content: [
             { text: "look", type: "input_text" },
             {
               detail: "low",
-              image_url: imageDataUrl(1024),
+              image_url: firstImageUrl,
               type: "input_image",
             },
             {
               detail: "low",
-              image_url: imageDataUrl(1024),
+              image_url: secondImageUrl,
               type: "input_image",
             },
           ],
@@ -90,6 +95,8 @@ describe("sanitizeOversizedInputImages", () => {
     const sanitized = sanitizeAllInputImages(payload)
 
     expect(sanitized).toBe(2)
-    expect(JSON.stringify(payload)).not.toContain("data:image/png;base64")
+    expect(JSON.stringify(payload)).not.toContain(firstImageUrl)
+    expect(JSON.stringify(payload)).not.toContain(secondImageUrl)
+    expect(JSON.stringify(payload)).toContain("data:image/png;base64")
   })
 })