fix(responses): estimate input image bytes from data URLs

Author: Marvae

src/routes/responses/utils.ts

@@ -77,7 +77,9 @@ export const hasVisionInput = (payload: ResponsesPayload): boolean => {
   return values.some((item) => containsVisionContent(item))
 }
 
-const IMAGE_DATA_URL_PATTERN = /^data:(image\/[a-zA-Z0-9.+-]+);base64,(.*)$/s
+const DATA_URL_PREFIX = "data:"
+const BASE64_MARKER = ";base64,"
+const IMAGE_MEDIA_TYPE_PATTERN = /^image\/[a-zA-Z0-9.+-]+$/
 
 export const sanitizeOversizedInputImages = (
   payload: ResponsesPayload,
@@ -163,13 +165,24 @@ const getInputImageDataUrl = (
     return null
   }
 
-  const match = record.image_url.match(IMAGE_DATA_URL_PATTERN)
-  if (!match) {
+  const imageUrl = record.image_url
+  const base64MarkerIndex = imageUrl.indexOf(BASE64_MARKER)
+  if (
+    !imageUrl.startsWith(DATA_URL_PREFIX)
+    || base64MarkerIndex <= DATA_URL_PREFIX.length
+  ) {
+    return null
+  }
+
+  const mediaType = imageUrl.slice(DATA_URL_PREFIX.length, base64MarkerIndex)
+  if (!IMAGE_MEDIA_TYPE_PATTERN.test(mediaType)) {
     return null
   }
 
-  const mediaType = match[1]
-  const decodedBytes = Buffer.byteLength(match[2], "base64")
+  const decodedBytes = getBase64DecodedByteLength(
+    imageUrl,
+    base64MarkerIndex + BASE64_MARKER.length,
+  )
 
   return {
     decodedBytes,
@@ -178,6 +191,19 @@ const getInputImageDataUrl = (
   }
 }
 
+const getBase64DecodedByteLength = (
+  value: string,
+  base64Start: number,
+): number => {
+  const base64Length = value.length - base64Start
+  const padding =
+    value.endsWith("==") ? 2
+    : value.endsWith("=") ? 1
+    : 0
+
+  return Math.max(0, Math.floor((base64Length * 3) / 4) - padding)
+}
+
 const replaceInputImageWithText = (
   image: InputImageDataUrl,
   reason: string,

tests/responses-image-sanitizer.test.ts

@@ -10,6 +10,9 @@ import {
 const imageDataUrl = (base64Length: number): string =>
   `data:image/png;base64,${"A".repeat(base64Length)}`
 
+const tinyPngDataUrl =
+  "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+/p9sAAAAASUVORK5CYII="
+
 const makePayload = (imageUrl: string): ResponsesPayload =>
   ({
     input: [
@@ -26,17 +29,17 @@ const makePayload = (imageUrl: string): ResponsesPayload =>
 
 describe("sanitizeOversizedInputImages", () => {
   test("replaces oversized input images with text markers", () => {
-    const payload = makePayload(imageDataUrl(16))
+    const payload = makePayload(tinyPngDataUrl)
 
-    const sanitized = sanitizeOversizedInputImages(payload, 8)
+    const sanitized = sanitizeOversizedInputImages(payload, 67)
 
     expect(sanitized).toBe(1)
     expect(payload.input).toEqual([
       {
         content: [
           { text: "look", type: "input_text" },
           {
-            text: "[omitted input image: image/png, 12 bytes, max 8 bytes]",
+            text: "[omitted input image: image/png, 68 bytes, max 67 bytes]",
             type: "input_text",
           },
         ],