fix(bedrock): normalize image MIME types

- Detect image MIME types from bytes before sending vision payloads.

- Correct stale base64 image media types in request history.

- Bump extension version to 4.99.4.

Author: allquixotic

src/api/transform/__tests__/bedrock-converse-format.spec.ts

@@ -68,6 +68,34 @@ describe("convertToBedrockConverseMessages", () => {
 		}
 	})
 
+	it("uses detected image bytes instead of stale media_type for Bedrock image format", () => {
+		const jpegData = Buffer.from([0xff, 0xd8, 0xff, 0xe0, 0x00, 0x10]).toString("base64")
+		const messages: Anthropic.Messages.MessageParam[] = [
+			{
+				role: "user",
+				content: [
+					{
+						type: "image",
+						source: {
+							type: "base64",
+							data: jpegData,
+							media_type: "image/png" as const,
+						},
+					},
+				],
+			},
+		]
+
+		const result = convertToBedrockConverseMessages(messages)
+		const imageBlock = result[0].content?.[0] as ContentBlock
+
+		if ("image" in imageBlock && imageBlock.image) {
+			expect(imageBlock.image.format).toBe("jpeg")
+		} else {
+			expect.fail("Expected image block not found")
+		}
+	})
+
 	it("converts tool use messages correctly (native tools format; default)", () => {
 		const messages: Anthropic.Messages.MessageParam[] = [
 			{

src/api/transform/__tests__/image-cleaning.spec.ts

@@ -103,6 +103,51 @@ describe("maybeRemoveImageBlocks", () => {
 		expect(apiHandler.getModel).toHaveBeenCalled()
 	})
 
+	it("should normalize image MIME type from base64 bytes when API handler supports images", () => {
+		const apiHandler = createMockApiHandler(true)
+		const jpegData = Buffer.from([0xff, 0xd8, 0xff, 0xe0, 0x00, 0x10]).toString("base64")
+		const messages: ApiMessage[] = [
+			{
+				role: "user",
+				content: [
+					{
+						type: "text",
+						text: "Check this screenshot:",
+					},
+					{
+						type: "image",
+						source: {
+							type: "base64",
+							media_type: "image/png",
+							data: jpegData,
+						},
+					},
+				],
+			},
+		]
+
+		const result = maybeRemoveImageBlocks(messages, apiHandler)
+
+		expect((result[0].content as any[])[1].source.media_type).toBe("image/jpeg")
+		expect(messages[0]).toEqual({
+			role: "user",
+			content: [
+				{
+					type: "text",
+					text: "Check this screenshot:",
+				},
+				{
+					type: "image",
+					source: {
+						type: "base64",
+						media_type: "image/png",
+						data: jpegData,
+					},
+				},
+			],
+		})
+	})
+
 	it("should convert image blocks to text descriptions when API handler doesn't support images", () => {
 		const apiHandler = createMockApiHandler(false)
 		const messages: ApiMessage[] = [

src/api/transform/bedrock-converse-format.ts

@@ -1,6 +1,7 @@
 import { Anthropic } from "@anthropic-ai/sdk"
 import { ConversationRole, Message, ContentBlock } from "@aws-sdk/client-bedrock-runtime"
 import { sanitizeOpenAiCallId } from "../../utils/tool-id"
+import { detectImageMimeType } from "../../utils/imageMime"
 
 interface BedrockMessageContent {
 	type: "text" | "image" | "video" | "tool_use" | "tool_result"
@@ -71,8 +72,9 @@ export function convertToBedrockConverseMessages(anthropicMessages: Anthropic.Me
 					byteArray = messageBlock.source.data
 				}
 
+				const mediaType = detectImageMimeType(byteArray) || messageBlock.source.media_type
 				// Extract format from media_type (e.g., "image/jpeg" -> "jpeg")
-				const format = messageBlock.source.media_type.split("/")[1]
+				const format = mediaType.split("/")[1]
 				if (!["png", "jpeg", "gif", "webp"].includes(format)) {
 					throw new Error(`Unsupported image format: ${format}`)
 				}

src/api/transform/image-cleaning.ts

@@ -1,6 +1,26 @@
 import { ApiMessage } from "../../core/task-persistence/apiMessages"
 
 import { ApiHandler } from "../index"
+import { detectBase64ImageMimeType } from "../../utils/imageMime"
+
+function normalizeImageBlockMimeType(block: any): any {
+	if (block?.type !== "image" || block.source?.type !== "base64" || typeof block.source.data !== "string") {
+		return block
+	}
+
+	const detectedMimeType = detectBase64ImageMimeType(block.source.data)
+	if (!detectedMimeType || detectedMimeType === block.source.media_type) {
+		return block
+	}
+
+	return {
+		...block,
+		source: {
+			...block.source,
+			media_type: detectedMimeType,
+		},
+	}
+}
 
 /* Removes image blocks from messages if they are not supported by the Api Handler */
 export function maybeRemoveImageBlocks(messages: ApiMessage[], apiHandler: ApiHandler): ApiMessage[] {
@@ -25,6 +45,8 @@ export function maybeRemoveImageBlocks(messages: ApiMessage[], apiHandler: ApiHa
 					}
 					return block
 				})
+			} else {
+				content = content.map(normalizeImageBlockMimeType)
 			}
 		}
 		return { ...message, content }

src/core/tools/helpers/__tests__/imageHelpers.spec.ts

@@ -0,0 +1,30 @@
+import path from "path"
+import * as fs from "fs/promises"
+import * as os from "os"
+
+import { afterEach, describe, expect, it } from "vitest"
+
+import { readImageAsDataUrlWithBuffer } from "../imageHelpers"
+
+describe("imageHelpers", () => {
+	let tempDir: string | undefined
+
+	afterEach(async () => {
+		if (tempDir) {
+			await fs.rm(tempDir, { recursive: true, force: true })
+			tempDir = undefined
+		}
+	})
+
+	it("labels image data URLs using detected bytes before the file extension", async () => {
+		tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "zoo-image-helper-"))
+		const mislabeledPngPath = path.join(tempDir, "ssm-screenshot.png")
+		const jpegBytes = Buffer.from([0xff, 0xd8, 0xff, 0xe0, 0x00, 0x10])
+		await fs.writeFile(mislabeledPngPath, jpegBytes)
+
+		const result = await readImageAsDataUrlWithBuffer(mislabeledPngPath)
+
+		expect(result.dataUrl).toBe(`data:image/jpeg;base64,${jpegBytes.toString("base64")}`)
+		expect(result.buffer).toEqual(jpegBytes)
+	})
+})

src/core/tools/helpers/imageHelpers.ts

@@ -3,6 +3,8 @@ import * as fs from "fs/promises"
 import { t } from "../../../i18n"
 import prettyBytes from "pretty-bytes"
 
+import { detectImageMimeType } from "../../../utils/imageMime"
+
 /**
  * Default maximum allowed image file size in bytes (5MB)
  */
@@ -77,7 +79,7 @@ export async function readImageAsDataUrlWithBuffer(filePath: string): Promise<{
 	const base64 = fileBuffer.toString("base64")
 	const ext = path.extname(filePath).toLowerCase()
 
-	const mimeType = IMAGE_MIME_TYPES[ext] || "image/png"
+	const mimeType = detectImageMimeType(fileBuffer) || IMAGE_MIME_TYPES[ext] || "image/png"
 	const dataUrl = `data:${mimeType};base64,${base64}`
 
 	return { dataUrl, buffer: fileBuffer }

src/package.json

@@ -3,7 +3,7 @@
 	"displayName": "%extension.displayName%",
 	"description": "%extension.description%",
 	"publisher": "allquixotic",
-	"version": "4.99.3",
+	"version": "4.99.4",
 	"icon": "assets/icons/icon.png",
 	"galleryBanner": {
 		"color": "#617A91",

src/utils/__tests__/imageMime.spec.ts

@@ -0,0 +1,22 @@
+import { describe, expect, it } from "vitest"
+
+import { detectBase64ImageMimeType, detectImageMimeType } from "../imageMime"
+
+describe("imageMime", () => {
+	it("detects JPEG bytes even when the caller expected PNG", () => {
+		const bytes = Uint8Array.from([0xff, 0xd8, 0xff, 0xe0, 0x00, 0x10])
+
+		expect(detectImageMimeType(bytes)).toBe("image/jpeg")
+		expect(detectBase64ImageMimeType(Buffer.from(bytes).toString("base64"))).toBe("image/jpeg")
+	})
+
+	it("detects PNG bytes", () => {
+		const bytes = Uint8Array.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a])
+
+		expect(detectImageMimeType(bytes)).toBe("image/png")
+	})
+
+	it("returns undefined for unknown bytes", () => {
+		expect(detectImageMimeType(Buffer.from("not an image"))).toBeUndefined()
+	})
+})

src/utils/imageMime.ts

@@ -0,0 +1,99 @@
+const PNG_SIGNATURE = [0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]
+const JPEG_SIGNATURE = [0xff, 0xd8, 0xff]
+const GIF87A_SIGNATURE = "GIF87a"
+const GIF89A_SIGNATURE = "GIF89a"
+const RIFF_SIGNATURE = "RIFF"
+const WEBP_SIGNATURE = "WEBP"
+const BMP_SIGNATURE = "BM"
+const ICO_SIGNATURE = [0x00, 0x00, 0x01, 0x00]
+const TIFF_LE_SIGNATURE = [0x49, 0x49, 0x2a, 0x00]
+const TIFF_BE_SIGNATURE = [0x4d, 0x4d, 0x00, 0x2a]
+const ISO_BMFF_FILE_TYPE_BOX = "ftyp"
+const AVIF_BRANDS = new Set(["avif", "avis"])
+
+function hasBytes(bytes: Uint8Array, signature: readonly number[], offset = 0): boolean {
+	if (bytes.length < offset + signature.length) {
+		return false
+	}
+
+	return signature.every((value, index) => bytes[offset + index] === value)
+}
+
+function hasAscii(bytes: Uint8Array, signature: string, offset = 0): boolean {
+	if (bytes.length < offset + signature.length) {
+		return false
+	}
+
+	for (let i = 0; i < signature.length; i++) {
+		if (bytes[offset + i] !== signature.charCodeAt(i)) {
+			return false
+		}
+	}
+
+	return true
+}
+
+function asciiSlice(bytes: Uint8Array, start: number, end: number): string {
+	if (bytes.length < end) {
+		return ""
+	}
+
+	return String.fromCharCode(...bytes.slice(start, end))
+}
+
+function looksLikeSvg(bytes: Uint8Array): boolean {
+	const sample = new TextDecoder("utf-8", { fatal: false }).decode(bytes.slice(0, 512)).trimStart()
+	return (
+		sample.startsWith("<svg") ||
+		sample.startsWith("<!DOCTYPE svg") ||
+		(sample.startsWith("<?xml") && sample.includes("<svg"))
+	)
+}
+
+export function detectImageMimeType(bytes: Uint8Array): string | undefined {
+	if (hasBytes(bytes, PNG_SIGNATURE)) {
+		return "image/png"
+	}
+
+	if (hasBytes(bytes, JPEG_SIGNATURE)) {
+		return "image/jpeg"
+	}
+
+	if (hasAscii(bytes, GIF87A_SIGNATURE) || hasAscii(bytes, GIF89A_SIGNATURE)) {
+		return "image/gif"
+	}
+
+	if (hasAscii(bytes, RIFF_SIGNATURE) && hasAscii(bytes, WEBP_SIGNATURE, 8)) {
+		return "image/webp"
+	}
+
+	if (hasAscii(bytes, BMP_SIGNATURE)) {
+		return "image/bmp"
+	}
+
+	if (hasBytes(bytes, ICO_SIGNATURE)) {
+		return "image/x-icon"
+	}
+
+	if (hasBytes(bytes, TIFF_LE_SIGNATURE) || hasBytes(bytes, TIFF_BE_SIGNATURE)) {
+		return "image/tiff"
+	}
+
+	if (hasAscii(bytes, ISO_BMFF_FILE_TYPE_BOX, 4) && AVIF_BRANDS.has(asciiSlice(bytes, 8, 12))) {
+		return "image/avif"
+	}
+
+	if (looksLikeSvg(bytes)) {
+		return "image/svg+xml"
+	}
+
+	return undefined
+}
+
+export function detectBase64ImageMimeType(data: string): string | undefined {
+	try {
+		return detectImageMimeType(Buffer.from(data, "base64"))
+	} catch {
+		return undefined
+	}
+}