feat(examples): Add widget interaction tools to budget-allocator, shadertoy, wiki-explorer, and threejs

ochafik · ochafik · commit 8f592a354a9c · 2026-01-17T03:15:01.000Z
budget-allocator:
- get-allocations: Get current budget allocations
- set-allocation: Set allocation for a category
- set-total-budget: Adjust total budget
- set-company-stage: Change stage for benchmarks
- get-benchmark-comparison: Compare against benchmarks

shadertoy:
- set-shader-source: Update shader source code
- get-shader-info: Get shader source and compilation status
- Sends errors via updateModelContext

wiki-explorer:
- search-article: Search for Wikipedia articles
- get-current-article: Get current article info
- highlight-node: Highlight a graph node
- get-visible-nodes: List visible nodes

threejs:
- set-scene-source: Update the Three.js scene source code
- get-scene-info: Get current scene state and any errors
- Sends syntax errors to model via updateModelContext
diff --git a/examples/budget-allocator-server/src/mcp-app.ts b/examples/budget-allocator-server/src/mcp-app.ts
@@ -680,12 +680,13 @@ app.registerTool(
   "set-allocation",
   {
     title: "Set Category Allocation",
-    description:
-      "Set the allocation percentage for a specific budget category",
+    description: "Set the allocation percentage for a specific budget category",
     inputSchema: z.object({
       categoryId: z
         .string()
-        .describe("Category ID (e.g., 'rd', 'sales', 'marketing', 'ops', 'ga')"),
+        .describe(
+          "Category ID (e.g., 'rd', 'sales', 'marketing', 'ops', 'ga')",
+        ),
       percent: z
         .number()
         .min(0)
@@ -703,7 +704,9 @@ app.registerTool(
       };
     }
 
-    const category = state.config.categories.find((c) => c.id === args.categoryId);
+    const category = state.config.categories.find(
+      (c) => c.id === args.categoryId,
+    );
     if (!category) {
       return {
         content: [
@@ -858,9 +861,7 @@ app.registerTool(
   async () => {
     if (!state.config || !state.analytics) {
       return {
-        content: [
-          { type: "text" as const, text: "Error: Data not loaded" },
-        ],
+        content: [{ type: "text" as const, text: "Error: Data not loaded" }],
         isError: true,
       };
     }
diff --git a/examples/shadertoy-server/src/mcp-app.ts b/examples/shadertoy-server/src/mcp-app.ts
@@ -239,12 +239,29 @@ app.registerTool(
     description:
       "Update the shader source code. Compiles and runs the new shader immediately.",
     inputSchema: z.object({
-      fragmentShader: z.string().describe("The main fragment shader source code (mainImage function)"),
-      common: z.string().optional().describe("Common code shared across all shaders"),
-      bufferA: z.string().optional().describe("Buffer A shader source (for multi-pass rendering)"),
-      bufferB: z.string().optional().describe("Buffer B shader source (for multi-pass rendering)"),
-      bufferC: z.string().optional().describe("Buffer C shader source (for multi-pass rendering)"),
-      bufferD: z.string().optional().describe("Buffer D shader source (for multi-pass rendering)"),
+      fragmentShader: z
+        .string()
+        .describe("The main fragment shader source code (mainImage function)"),
+      common: z
+        .string()
+        .optional()
+        .describe("Common code shared across all shaders"),
+      bufferA: z
+        .string()
+        .optional()
+        .describe("Buffer A shader source (for multi-pass rendering)"),
+      bufferB: z
+        .string()
+        .optional()
+        .describe("Buffer B shader source (for multi-pass rendering)"),
+      bufferC: z
+        .string()
+        .optional()
+        .describe("Buffer C shader source (for multi-pass rendering)"),
+      bufferD: z
+        .string()
+        .optional()
+        .describe("Buffer D shader source (for multi-pass rendering)"),
     }),
   },
   async (args) => {
@@ -272,8 +289,7 @@ app.registerTool(
   "get-shader-info",
   {
     title: "Get Shader Info",
-    description:
-      "Get the current shader source code and compilation status.",
+    description: "Get the current shader source code and compilation status.",
   },
   async () => {
     log.info("get-shader-info tool called");
diff --git a/examples/threejs-server/src/mcp-app-wrapper.tsx b/examples/threejs-server/src/mcp-app-wrapper.tsx
@@ -7,15 +7,30 @@
 import type { App, McpUiHostContext } from "@modelcontextprotocol/ext-apps";
 import { useApp } from "@modelcontextprotocol/ext-apps/react";
 import type { CallToolResult } from "@modelcontextprotocol/sdk/types.js";
-import { StrictMode, useState, useCallback, useEffect } from "react";
+import { StrictMode, useState, useCallback, useEffect, useRef } from "react";
 import { createRoot } from "react-dom/client";
+import { z } from "zod";
 import ThreeJSApp from "./threejs-app.tsx";
 import "./global.css";
 
 // =============================================================================
 // Types
 // =============================================================================
 
+/**
+ * Scene state tracked for widget interaction tools.
+ */
+export interface SceneState {
+  /** Current Three.js code */
+  code: string | null;
+  /** Canvas height */
+  height: number;
+  /** Last error message if any */
+  error: string | null;
+  /** Whether the scene is currently rendering */
+  isRendering: boolean;
+}
+
 /**
  * Props passed to the widget component.
  * This interface can be reused for other widgets.
@@ -37,6 +52,96 @@ export interface WidgetProps<TToolInput = Record<string, unknown>> {
   openLink: App["openLink"];
   /** Send log messages to the host */
   sendLog: App["sendLog"];
+  /** Callback to report scene errors */
+  onSceneError: (error: string | null) => void;
+  /** Callback to report scene is rendering */
+  onSceneRendering: (isRendering: boolean) => void;
+}
+
+// =============================================================================
+// Widget Interaction Tools
+// =============================================================================
+
+/**
+ * Registers widget interaction tools on the App instance.
+ * These tools allow the model to interact with the Three.js scene.
+ */
+function registerWidgetTools(
+  app: App,
+  sceneStateRef: React.RefObject<SceneState>,
+): void {
+  // Tool: set-scene-source - Update the scene source/configuration
+  app.registerTool(
+    "set-scene-source",
+    {
+      title: "Set Scene Source",
+      description:
+        "Update the Three.js scene source code. The code will be executed in a sandboxed environment with access to THREE, OrbitControls, EffectComposer, RenderPass, UnrealBloomPass, canvas, width, and height.",
+      inputSchema: z.object({
+        code: z.string().describe("JavaScript code to render the 3D scene"),
+        height: z
+          .number()
+          .int()
+          .positive()
+          .optional()
+          .describe("Height in pixels (optional, defaults to current)"),
+      }),
+      outputSchema: z.object({
+        success: z.boolean(),
+        code: z.string(),
+        height: z.number(),
+      }),
+    },
+    async (args) => {
+      // Update scene state
+      sceneStateRef.current.code = args.code;
+      if (args.height !== undefined) {
+        sceneStateRef.current.height = args.height;
+      }
+      sceneStateRef.current.error = null;
+
+      const result = {
+        success: true,
+        code: args.code,
+        height: sceneStateRef.current.height,
+      };
+
+      return {
+        content: [{ type: "text" as const, text: JSON.stringify(result) }],
+        structuredContent: result,
+      };
+    },
+  );
+
+  // Tool: get-scene-info - Get current scene state and any errors
+  app.registerTool(
+    "get-scene-info",
+    {
+      title: "Get Scene Info",
+      description:
+        "Get the current Three.js scene state including source code, dimensions, rendering status, and any errors.",
+      outputSchema: z.object({
+        code: z.string().nullable(),
+        height: z.number(),
+        error: z.string().nullable(),
+        isRendering: z.boolean(),
+      }),
+    },
+    async () => {
+      const state = sceneStateRef.current;
+      const result = {
+        code: state.code,
+        height: state.height,
+        error: state.error,
+        isRendering: state.isRendering,
+      };
+
+      return {
+        content: [{ type: "text" as const, text: JSON.stringify(result) }],
+        structuredContent: result,
+      };
+    },
+  );
 }
 
 // =============================================================================
@@ -54,14 +159,38 @@ function McpAppWrapper() {
   const [toolResult, setToolResult] = useState<CallToolResult | null>(null);
   const [hostContext, setHostContext] = useState<McpUiHostContext | null>(null);
 
+  // Scene state for widget interaction tools
+  const sceneStateRef = useRef<SceneState>({
+    code: null,
+    height: 400,
+    error: null,
+    isRendering: false,
+  });
+
+  // Reference to app for tools to access updateModelContext
+  const appRef = useRef<App | null>(null);
+
   const { app, error } = useApp({
     appInfo: { name: "Three.js Widget", version: "1.0.0" },
-    capabilities: {},
+    capabilities: { tools: {} },
     onAppCreated: (app) => {
+      appRef.current = app;
+
+      // Register widget interaction tools before connect()
+      registerWidgetTools(app, sceneStateRef);
+
       // Complete tool input (streaming finished)
       app.ontoolinput = (params) => {
-        setToolInputs(params.arguments as Record<string, unknown>);
+        const args = params.arguments as Record<string, unknown>;
+        setToolInputs(args);
         setToolInputsPartial(null);
+        // Update scene state from tool input
+        if (typeof args.code === "string") {
+          sceneStateRef.current.code = args.code;
+        }
+        if (typeof args.height === "number") {
+          sceneStateRef.current.height = args.height;
+        }
       };
       // Partial tool input (streaming in progress)
       app.ontoolinputpartial = (params) => {
@@ -106,6 +235,33 @@ function McpAppWrapper() {
     [app],
   );
 
+  // Callback for scene to report errors
+  const onSceneError = useCallback((sceneError: string | null) => {
+    sceneStateRef.current.error = sceneError;
+
+    // Send errors to model context for awareness
+    if (sceneError && appRef.current) {
+      appRef.current.updateModelContext({
+        content: [
+          {
+            type: "text" as const,
+            text: `Three.js Scene Error: ${sceneError}`,
+          },
+        ],
+        structuredContent: {
+          type: "scene_error",
+          error: sceneError,
+          timestamp: new Date().toISOString(),
+        },
+      });
+    }
+  }, []);
+
+  // Callback for scene to report rendering state
+  const onSceneRendering = useCallback((isRendering: boolean) => {
+    sceneStateRef.current.isRendering = isRendering;
+  }, []);
+
   if (error) {
     return <div className="error">Error: {error.message}</div>;
   }
@@ -124,6 +280,8 @@ function McpAppWrapper() {
       sendMessage={sendMessage}
       openLink={openLink}
       sendLog={sendLog}
+      onSceneError={onSceneError}
+      onSceneRendering={onSceneRendering}
     />
   );
 }
diff --git a/examples/threejs-server/src/threejs-app.tsx b/examples/threejs-server/src/threejs-app.tsx
@@ -173,6 +173,8 @@ export default function ThreeJSApp({
   sendMessage: _sendMessage,
   openLink: _openLink,
   sendLog: _sendLog,
+  onSceneError,
+  onSceneRendering,
 }: ThreeJSAppProps) {
   const [error, setError] = useState<string | null>(null);
   const canvasRef = useRef<HTMLCanvasElement>(null);
@@ -195,11 +197,21 @@ export default function ThreeJSApp({
     if (!code || !canvasRef.current || !containerRef.current) return;
 
     setError(null);
+    onSceneError(null);
+    onSceneRendering(true);
+
     const width = containerRef.current.offsetWidth || 800;
-    executeThreeCode(code, canvasRef.current, width, height).catch((e) =>
-      setError(e instanceof Error ? e.message : "Unknown error"),
-    );
-  }, [code, height]);
+    executeThreeCode(code, canvasRef.current, width, height)
+      .then(() => {
+        onSceneRendering(true);
+      })
+      .catch((e) => {
+        const errorMessage = e instanceof Error ? e.message : "Unknown error";
+        setError(errorMessage);
+        onSceneError(errorMessage);
+        onSceneRendering(false);
+      });
+  }, [code, height, onSceneError, onSceneRendering]);
 
   if (isStreaming || !code) {
     return (
diff --git a/examples/wiki-explorer-server/src/mcp-app.ts b/examples/wiki-explorer-server/src/mcp-app.ts
@@ -420,7 +420,10 @@ app.registerTool(
     const response = result.structuredContent as unknown as ToolResponse;
     if (response && response.page) {
       initialUrl = response.page.url;
-      addNode(response.page.url, response.page.title, "default", { x: 0, y: 0 });
+      addNode(response.page.url, response.page.title, "default", {
+        x: 0,
+        y: 0,
+      });
       graph.warmupTicks(100);
       handleToolResultData(result);
       graph.centerAt(0, 0, 500);
@@ -465,7 +468,9 @@ app.registerTool(
 
     if (!currentUrl) {
       return {
-        content: [{ type: "text" as const, text: "No article is currently selected" }],
+        content: [
+          { type: "text" as const, text: "No article is currently selected" },
+        ],
         structuredContent: {
           hasSelection: false,
           article: null,
@@ -477,7 +482,12 @@ app.registerTool(
 
     if (!node) {
       return {
-        content: [{ type: "text" as const, text: "Selected article not found in graph" }],
+        content: [
+          {
+            type: "text" as const,
+            text: "Selected article not found in graph",
+          },
+        ],
         structuredContent: {
           hasSelection: false,
           article: null,
@@ -512,7 +522,8 @@ app.registerTool(
   "highlight-node",
   {
     title: "Highlight Node",
-    description: "Highlight and center on a specific node in the graph by title or URL",
+    description:
+      "Highlight and center on a specific node in the graph by title or URL",
     inputSchema: z.object({
       identifier: z
         .string()
@@ -526,8 +537,7 @@ app.registerTool(
     // Find node by title (case-insensitive partial match) or exact URL
     const node = graphData.nodes.find(
       (n) =>
-        n.url === identifier ||
-        n.title.toLowerCase().includes(lowerIdentifier),
+        n.url === identifier || n.title.toLowerCase().includes(lowerIdentifier),
     );
 
     if (!node) {
