feat(rust): support binary tool results (#1222)

* feat(rust): support binary tool results

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

* fix: default empty MCP resource MIME types

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

---------

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
Co-authored-by: Stephen Toub <stoub@microsoft.com>

Author: 3 people

nodejs/src/types.ts

@@ -322,9 +322,13 @@ export function convertMcpCallToolResult(callResult: McpCallToolResult): ToolRes
                     textParts.push(block.resource.text);
                 }
                 if (block.resource?.blob) {
+                    const mimeType = block.resource.mimeType;
                     binaryResults.push({
                         data: block.resource.blob,
-                        mimeType: block.resource.mimeType ?? "application/octet-stream",
+                        mimeType:
+                            typeof mimeType === "string" && mimeType
+                                ? mimeType
+                                : "application/octet-stream",
                         type: "resource",
                         description: block.resource.uri,
                     });

nodejs/test/call-tool-result.test.ts

@@ -130,12 +130,17 @@ describe("convertMcpCallToolResult", () => {
                     type: "resource",
                     resource: { uri: "file:///data.bin", blob: "binarydata" },
                 },
+                {
+                    type: "resource",
+                    resource: { uri: "file:///empty-mime.bin", blob: "binarydata2", mimeType: "" },
+                },
             ],
         };
 
         const result = convertMcpCallToolResult(input);
 
         expect(result.binaryResultsForLlm![0]!.mimeType).toBe("application/octet-stream");
+        expect(result.binaryResultsForLlm![1]!.mimeType).toBe("application/octet-stream");
     });
 
     it("handles text block with missing text field without corrupting output", () => {

python/copilot/tools.py

@@ -307,14 +307,14 @@ def convert_mcp_call_tool_result(call_result: dict[str, Any]) -> ToolResult:
                 text_parts.append(text)
             blob = resource.get("blob")
             if isinstance(blob, str) and blob:
-                mime_type = resource.get("mimeType", "application/octet-stream")
+                mime_type = resource.get("mimeType")
+                if not isinstance(mime_type, str) or not mime_type:
+                    mime_type = "application/octet-stream"
                 uri = resource.get("uri", "")
                 binary_results.append(
                     ToolBinaryResult(
                         data=blob,
-                        mime_type=mime_type
-                        if isinstance(mime_type, str)
-                        else "application/octet-stream",
+                        mime_type=mime_type,
                         type="resource",
                         description=uri if isinstance(uri, str) else "",
                     )

python/test_tools.py

@@ -373,8 +373,34 @@ def test_resource_blob_to_binary(self):
         assert result.binary_results_for_llm is not None
         assert len(result.binary_results_for_llm) == 1
         assert result.binary_results_for_llm[0].data == "blobdata"
+        assert result.binary_results_for_llm[0].mime_type == "image/png"
         assert result.binary_results_for_llm[0].description == "file:///img.png"
 
+    def test_resource_blob_defaults_missing_or_empty_mime_type(self):
+        result = convert_mcp_call_tool_result(
+            {
+                "content": [
+                    {
+                        "type": "resource",
+                        "resource": {"uri": "file:///data.bin", "blob": "binarydata"},
+                    },
+                    {
+                        "type": "resource",
+                        "resource": {
+                            "uri": "file:///empty-mime.bin",
+                            "blob": "binarydata2",
+                            "mimeType": "",
+                        },
+                    },
+                ],
+            }
+        )
+
+        assert result.binary_results_for_llm is not None
+        assert len(result.binary_results_for_llm) == 2
+        assert result.binary_results_for_llm[0].mime_type == "application/octet-stream"
+        assert result.binary_results_for_llm[1].mime_type == "application/octet-stream"
+
     def test_empty_content_array(self):
         result = convert_mcp_call_tool_result({"content": []})
         assert result.text_result_for_llm == ""

rust/src/tool.rs

@@ -139,6 +139,7 @@ pub fn convert_mcp_call_tool_result(value: &serde_json::Value) -> Option<ToolRes
                     let mime_type = resource
                         .get("mimeType")
                         .and_then(serde_json::Value::as_str)
+                        .filter(|s| !s.is_empty())
                         .unwrap_or("application/octet-stream");
                     let description = resource
                         .get("uri")
@@ -523,12 +524,130 @@ mod tests {
             .expect("binary results should be captured");
         assert_eq!(binary_results.len(), 2);
         assert_eq!(binary_results[0].r#type, "image");
+        assert_eq!(binary_results[0].data, "aW1n");
+        assert_eq!(binary_results[0].mime_type, "image/png");
         assert_eq!(
             binary_results[1].description.as_deref(),
             Some("file:///tmp/data.bin")
         );
     }
 
+    #[test]
+    fn convert_mcp_call_tool_result_converts_image_content() {
+        let result = convert_mcp_call_tool_result(&serde_json::json!({
+            "content": [
+                { "type": "image", "data": "aW1hZ2U=", "mimeType": "image/jpeg" }
+            ]
+        }))
+        .expect("valid CallToolResult should convert");
+
+        let ToolResult::Expanded(expanded) = result else {
+            panic!("expected expanded tool result");
+        };
+
+        assert_eq!(expanded.text_result_for_llm, "");
+        assert_eq!(expanded.result_type, "success");
+        let binary_results = expanded
+            .binary_results_for_llm
+            .expect("image result should be captured");
+        assert_eq!(binary_results.len(), 1);
+        assert_eq!(binary_results[0].data, "aW1hZ2U=");
+        assert_eq!(binary_results[0].mime_type, "image/jpeg");
+        assert_eq!(binary_results[0].r#type, "image");
+        assert!(binary_results[0].description.is_none());
+    }
+
+    #[test]
+    fn convert_mcp_call_tool_result_converts_resource_blob_content() {
+        let result = convert_mcp_call_tool_result(&serde_json::json!({
+            "content": [
+                {
+                    "type": "resource",
+                    "resource": {
+                        "uri": "file:///tmp/report.pdf",
+                        "blob": "cGRm",
+                        "mimeType": "application/pdf"
+                    }
+                }
+            ]
+        }))
+        .expect("valid CallToolResult should convert");
+
+        let ToolResult::Expanded(expanded) = result else {
+            panic!("expected expanded tool result");
+        };
+
+        let binary_results = expanded
+            .binary_results_for_llm
+            .expect("resource result should be captured");
+        assert_eq!(binary_results.len(), 1);
+        assert_eq!(binary_results[0].data, "cGRm");
+        assert_eq!(binary_results[0].mime_type, "application/pdf");
+        assert_eq!(binary_results[0].r#type, "resource");
+        assert_eq!(
+            binary_results[0].description.as_deref(),
+            Some("file:///tmp/report.pdf")
+        );
+    }
+
+    #[test]
+    fn convert_mcp_call_tool_result_defaults_resource_blob_mime_type() {
+        let result = convert_mcp_call_tool_result(&serde_json::json!({
+            "content": [
+                {
+                    "type": "resource",
+                    "resource": {
+                        "uri": "file:///tmp/data.bin",
+                        "blob": "Ymlu"
+                    }
+                },
+                {
+                    "type": "resource",
+                    "resource": {
+                        "blob": "YmluMg==",
+                        "mimeType": ""
+                    }
+                }
+            ]
+        }))
+        .expect("valid CallToolResult should convert");
+
+        let ToolResult::Expanded(expanded) = result else {
+            panic!("expected expanded tool result");
+        };
+
+        let binary_results = expanded
+            .binary_results_for_llm
+            .expect("resource blobs should be captured");
+        assert_eq!(binary_results.len(), 2);
+        assert_eq!(binary_results[0].mime_type, "application/octet-stream");
+        assert_eq!(binary_results[1].mime_type, "application/octet-stream");
+    }
+
+    #[test]
+    fn convert_mcp_call_tool_result_omits_binary_results_without_binary_content() {
+        let result = convert_mcp_call_tool_result(&serde_json::json!({
+            "content": [
+                { "type": "text", "text": "hello" },
+                {
+                    "type": "resource",
+                    "resource": {
+                        "uri": "file:///tmp/readme.md",
+                        "text": "resource text"
+                    }
+                }
+            ]
+        }))
+        .expect("valid CallToolResult should convert");
+
+        let ToolResult::Expanded(expanded) = result else {
+            panic!("expected expanded tool result");
+        };
+
+        assert_eq!(expanded.text_result_for_llm, "hello\nresource text");
+        assert!(expanded.binary_results_for_llm.is_none());
+    }
+
     #[tokio::test]
     async fn tool_handler_call_returns_result() {
         let tool = EchoTool;

rust/src/types.rs

@@ -3125,7 +3125,8 @@ mod tests {
         Attachment, AttachmentLineRange, AttachmentSelectionPosition, AttachmentSelectionRange,
         ConnectionState, CustomAgentConfig, DeliveryMode, GitHubReferenceType,
         InfiniteSessionConfig, ProviderConfig, ResumeSessionConfig, SessionConfig, SessionEvent,
-        SessionId, SystemMessageConfig, Tool, ensure_attachment_display_names,
+        SessionId, SystemMessageConfig, Tool, ToolBinaryResult, ToolResult, ToolResultExpanded,
+        ToolResultResponse, ensure_attachment_display_names,
     };
     use crate::generated::session_events::TypedSessionEvent;
 
@@ -3157,6 +3158,64 @@ mod tests {
         assert!(tool.parameters.is_empty());
     }
 
+    #[test]
+    fn tool_result_expanded_serializes_binary_results_for_llm() {
+        let response = ToolResultResponse {
+            result: ToolResult::Expanded(ToolResultExpanded {
+                text_result_for_llm: "rendered chart".to_string(),
+                result_type: "success".to_string(),
+                binary_results_for_llm: Some(vec![ToolBinaryResult {
+                    data: "aW1n".to_string(),
+                    mime_type: "image/png".to_string(),
+                    r#type: "image".to_string(),
+                    description: Some("chart preview".to_string()),
+                }]),
+                session_log: None,
+                error: None,
+                tool_telemetry: None,
+            }),
+        };
+
+        let wire = serde_json::to_value(&response).unwrap();
+
+        assert_eq!(
+            wire,
+            json!({
+                "result": {
+                    "textResultForLlm": "rendered chart",
+                    "resultType": "success",
+                    "binaryResultsForLlm": [
+                        {
+                            "data": "aW1n",
+                            "mimeType": "image/png",
+                            "type": "image",
+                            "description": "chart preview"
+                        }
+                    ]
+                }
+            })
+        );
+    }
+
+    #[test]
+    fn tool_result_expanded_omits_binary_results_for_llm_when_none() {
+        let response = ToolResultResponse {
+            result: ToolResult::Expanded(ToolResultExpanded {
+                text_result_for_llm: "ok".to_string(),
+                result_type: "success".to_string(),
+                binary_results_for_llm: None,
+                session_log: None,
+                error: None,
+                tool_telemetry: None,
+            }),
+        };
+
+        let wire = serde_json::to_value(&response).unwrap();
+
+        assert_eq!(wire["result"]["textResultForLlm"], "ok");
+        assert!(wire["result"].get("binaryResultsForLlm").is_none());
+    }
+
     #[test]
     fn session_config_default_enables_permission_flow_flags() {
         let cfg = SessionConfig::default();