fix(core): sniff MCP image MIME types

packages/core/src/tools/mcp-tool.test.ts

@@ -51,6 +51,8 @@ const createSdkResponse = (
   },
 ];
 
+const WEBP_BASE64_DATA = 'UklGRgAAAABXRUJQ';
+
 describe('generateValidName', () => {
   it('should return a valid name for a simple function', () => {
     expect(generateValidName('myFunction')).toBe('mcp_myFunction');
@@ -499,6 +501,38 @@ describe('DiscoveredMCPTool', () => {
       expect(toolResult.returnDisplay).toBe('[Audio: audio/mp3]');
     });
 
+    it('should correct a mislabeled WebP image block', async () => {
+      const params = { param: 'look' };
+      mockCallTool.mockResolvedValue(
+        createSdkResponse(serverToolName, {
+          content: [
+            {
+              type: 'image',
+              data: WEBP_BASE64_DATA,
+              mimeType: 'image/png',
+            },
+          ],
+        }),
+      );
+
+      const invocation = tool.build(params);
+      const toolResult = await invocation.execute({
+        abortSignal: new AbortController().signal,
+      });
+      expect(toolResult.llmContent).toEqual([
+        {
+          text: `[Tool '${serverToolName}' provided the following image data with mime-type: image/webp]`,
+        },
+        {
+          inlineData: {
+            mimeType: 'image/webp',
+            data: WEBP_BASE64_DATA,
+          },
+        },
+      ]);
+      expect(toolResult.returnDisplay).toBe('[Image: image/webp]');
+    });
+
     it('should handle a ResourceLinkBlock response', async () => {
       const params = { param: 'get' };
       mockCallTool.mockResolvedValue(
@@ -592,6 +626,41 @@ describe('DiscoveredMCPTool', () => {
       );
     });
 
+    it('should correct a mislabeled WebP embedded resource block', async () => {
+      const params = { param: 'get' };
+      mockCallTool.mockResolvedValue(
+        createSdkResponse(serverToolName, {
+          content: [
+            {
+              type: 'resource',
+              resource: {
+                uri: 'file:///path/to/image.webp',
+                blob: WEBP_BASE64_DATA,
+                mimeType: 'image/png',
+              },
+            },
+          ],
+        }),
+      );
+
+      const invocation = tool.build(params);
+      const toolResult = await invocation.execute({
+        abortSignal: new AbortController().signal,
+      });
+      expect(toolResult.llmContent).toEqual([
+        {
+          text: `[Tool '${serverToolName}' provided the following embedded resource with mime-type: image/webp]`,
+        },
+        {
+          inlineData: {
+            mimeType: 'image/webp',
+            data: WEBP_BASE64_DATA,
+          },
+        },
+      ]);
+      expect(toolResult.returnDisplay).toBe('[Embedded Resource: image/webp]');
+    });
+
     it('should handle a mix of content block types', async () => {
       const params = { param: 'complex' };
       mockCallTool.mockResolvedValue(

packages/core/src/tools/mcp-tool.ts

@@ -451,19 +451,69 @@ function transformTextBlock(block: McpTextBlock): Part {
   return { text: block.text };
 }
 
+function startsWithBytes(buffer: Buffer, bytes: number[]): boolean {
+  if (buffer.length < bytes.length) {
+    return false;
+  }
+
+  return bytes.every((byte, index) => buffer[index] === byte);
+}
+
+function detectImageMimeTypeFromBase64(data: string): string | undefined {
+  const prefix = data.slice(0, 32);
+  const buffer = Buffer.from(prefix, 'base64');
+
+  if (
+    startsWithBytes(buffer, [0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a])
+  ) {
+    return 'image/png';
+  }
+
+  if (startsWithBytes(buffer, [0xff, 0xd8, 0xff])) {
+    return 'image/jpeg';
+  }
+
+  const gifSignature = buffer.subarray(0, 6).toString('ascii');
+  if (gifSignature === 'GIF87a' || gifSignature === 'GIF89a') {
+    return 'image/gif';
+  }
+
+  if (
+    buffer.length >= 12 &&
+    buffer.subarray(0, 4).toString('ascii') === 'RIFF' &&
+    buffer.subarray(8, 12).toString('ascii') === 'WEBP'
+  ) {
+    return 'image/webp';
+  }
+
+  return undefined;
+}
+
+function getMimeTypeForInlineData(
+  declaredMimeType: string,
+  data: string,
+): string {
+  return detectImageMimeTypeFromBase64(data) ?? declaredMimeType;
+}
+
 function transformImageAudioBlock(
   block: McpMediaBlock,
   toolName: string,
 ): Part[] {
+  const mimeType =
+    block.type === 'image'
+      ? getMimeTypeForInlineData(block.mimeType, block.data)
+      : block.mimeType;
+
   return [
     {
       text: `[Tool '${toolName}' provided the following ${
         block.type
-      } data with mime-type: ${block.mimeType}]`,
+      } data with mime-type: ${mimeType}]`,
     },
     {
       inlineData: {
-        mimeType: block.mimeType,
+        mimeType,
         data: block.data,
       },
     },
@@ -479,7 +529,8 @@ function transformResourceBlock(
     return { text: resource.text };
   }
   if (resource?.blob) {
-    const mimeType = resource.mimeType || 'application/octet-stream';
+    const declaredMimeType = resource.mimeType || 'application/octet-stream';
+    const mimeType = getMimeTypeForInlineData(declaredMimeType, resource.blob);
     return [
       {
         text: `[Tool '${toolName}' provided the following embedded resource with mime-type: ${mimeType}]`,
@@ -561,7 +612,7 @@ function getStringifiedResultForDisplay(rawResponse: Part[]): string {
       case 'text':
         return block.text;
       case 'image':
-        return `[Image: ${block.mimeType}]`;
+        return `[Image: ${getMimeTypeForInlineData(block.mimeType, block.data)}]`;
       case 'audio':
         return `[Audio: ${block.mimeType}]`;
       case 'resource_link':
@@ -570,9 +621,13 @@ function getStringifiedResultForDisplay(rawResponse: Part[]): string {
         if (block.resource?.text) {
           return block.resource.text;
         }
-        return `[Embedded Resource: ${
-          block.resource?.mimeType || 'unknown type'
-        }]`;
+        if (block.resource?.blob) {
+          return `[Embedded Resource: ${getMimeTypeForInlineData(
+            block.resource.mimeType || 'unknown type',
+            block.resource.blob,
+          )}]`;
+        }
+        return `[Embedded Resource: unknown type]`;
       default:
         return `[Unknown content type: ${(block as { type: string }).type}]`;
     }