feat: tool will reach to client and client can react on tools, it will support action on web later

Author: iamprashant

package.json

@@ -1,6 +1,6 @@
 {
     "name": "@rapidaai/react",
-    "version": "1.1.67",
+    "version": "1.1.68",
     "description": "An easy to use react client for building generative ai application using Rapida platform.",
     "repository": {
         "type": "git",
@@ -127,4 +127,4 @@
         "ts-protoc-gen": "^0.15.1-pre.a71b34e",
         "typed-emitter": "^2.1.0"
     }
-}
+}

readme.md

@@ -443,7 +443,11 @@ const agent = new VoiceAgent(
     onConfiguration: (cfg) => console.log("Config:", cfg),
     onInterrupt: () => console.log("Interrupted"),
     onInitialization: (init) => console.log("Session started:", init),
-    onDirective: (dir) => console.log("Directive:", dir),
+    onToolCall: (toolCall) => {
+      console.log("Tool call:", toolCall);
+      // Return a result to send back to server, or void to skip
+      return { status: "completed" };
+    },
   }
 );
 ```
@@ -1092,7 +1096,7 @@ const updated = await UpdateNotificationSetting(config, request, authHeader);
 | `InputOptions` | Input channel config: `channels`, `channel`, `device`, `iceServers`. |
 | `OutputOptions` | Output channel config: `channels`, `channel`, `device`. |
 | `UserIdentifier` | User identity: `id` and optional `name`. |
-| `AgentCallback` | Callback interface: `onAssistantMessage`, `onUserMessage`, `onConfiguration`, `onInterrupt`, `onDirective`, `onInitialization`, `onConnectionStateChange`, `onConnected`, `onDisconnected`, `onError`. |
+| `AgentCallback` | Callback interface: `onAssistantMessage`, `onUserMessage`, `onConfiguration`, `onInterrupt`, `onToolCall`, `onToolCallResult`, `onInitialization`, `onConnectionStateChange`, `onConnected`, `onDisconnected`, `onError`. |
 | `Channel` | Enum: `Channel.Audio`, `Channel.Text` |
 | `ConnectionState` | Enum: `ConnectionState.Disconnected`, `ConnectionState.Connecting`, `ConnectionState.Connected` |
 | `Message` | Message object: `id`, `role`, `messages[]`, `feedback?`, `time`, `status` |

src/agents/index.ts

@@ -80,7 +80,8 @@ export function describeResponse(res: WebTalkResponse): string {
   if (res.hasAssistant()) parts.push(`Assistant("${res.getAssistant()?.getText()?.substring(0, 80) ?? ""}"`);
   if (res.hasUser()) parts.push(`User("${res.getUser()?.getText()?.substring(0, 80) ?? ""}"`);
   if (res.hasInterruption()) parts.push("Interruption");
-  if (res.hasDirective()) parts.push("Directive");
+  if (res.hasToolcall?.()) parts.push("ToolCall");
+  if (res.hasToolcallresult?.()) parts.push("ToolCallResult");
   if (res.hasSignaling()) parts.push("Signaling");
   return parts.length > 0 ? parts.join(" + ") : "Empty";
 }

src/agents/voice-agent.ts

@@ -40,7 +40,6 @@ import {
   ConversationUserMessage,
 } from "@/rapida/types/agent-callback";
 import { ConnectionState } from "@/rapida/types/connection-state";
-import { ConversationDirective } from "../clients/protos/talk-api_pb";
 
 const LOG_PREFIX = "[Rapida:VoiceAgent]";
 
@@ -144,12 +143,31 @@ export class VoiceAgent extends Agent {
 
       },
 
-      onDirective: (directive) => {
-        // console.log(`${LOG_PREFIX} callback -> onDirective`, directive);
-        if (directive && directive.type === ConversationDirective.DirectiveType.END_CONVERSATION) {
-          this.disconnect();
+      onToolCall: (toolCall) => {
+        const sendResult = (result: Record<string, string>) => {
+          this.webrtcTransport?.sendToolCallResult(
+            toolCall.id, toolCall.toolid, toolCall.name, toolCall.action, result,
+          );
+        };
+
+        for (const cb of this.agentCallbacks) {
+          const r = cb.onToolCall?.(toolCall);
+          if (r && typeof (r as Promise<unknown>).then === "function") {
+            (r as Promise<Record<string, string>>).then((res) => { if (res) sendResult(res); });
+            return;
+          }
+          if (r) {
+            sendResult(r as Record<string, string>);
+            return;
+          }
         }
       },
+
+      onToolCallResult: (result) => {
+        this.agentCallbacks.forEach((cb) => {
+          cb.onToolCallResult?.(result);
+        });
+      },
       onConversationEvent: (event) => {
         this.agentCallbacks.forEach((cb) => {
           cb.onConversationEvent?.(event);

src/audio/grpc-signaling-manager.ts

@@ -28,6 +28,8 @@ import { AgentCallback } from "@/rapida/types/agent-callback";
 import { WebTalk } from "@/rapida/clients/webrtc";
 import {
   ConversationUserMessage,
+  ConversationToolCallResult,
+  ToolCallActionMap,
 } from "@/rapida/clients/protos/talk-api_pb";
 import {
   WebTalkRequest,
@@ -195,6 +197,36 @@ export class GrpcSignalingManager {
     }
   }
 
+  /** Send a tool call result back to the server */
+  sendToolCallResult(
+    id: string,
+    toolId: string,
+    name: string,
+    action: ToolCallActionMap[keyof ToolCallActionMap],
+    result: Record<string, string>,
+  ): void {
+    if (!this.grpcStream) return;
+    this.ensureInitializationSent();
+
+    try {
+      const request = new WebTalkRequest();
+      const toolCallResult = new ConversationToolCallResult();
+      toolCallResult.setId(id);
+      toolCallResult.setToolid(toolId);
+      toolCallResult.setName(name);
+      toolCallResult.setAction(action);
+      const resultMap = toolCallResult.getResultMap();
+      for (const [k, v] of Object.entries(result)) {
+        resultMap.set(k, v);
+      }
+      request.setToolcallresult(toolCallResult);
+      this.grpcStream.write(request);
+    } catch (error) {
+      console.error(`${LOG_PREFIX} Failed to send tool call result`, error);
+      this.callbacks.onError?.(new Error(`Failed to send tool call result: ${error}`));
+    }
+  }
+
   /** Send an SDP answer via ClientSignaling */
   sendWebRTCAnswer(sdp: string): void {
     if (!this.grpcStream) return;

src/audio/message-protocol-handler.ts

@@ -43,7 +43,7 @@ import type { AudioMediaManager } from "./audio-media-manager";
  * - Parse each response and route to the correct manager / callback
  * - Handle ServerSignaling (config, SDP, ICE, ready, clear, error)
  * - Handle conversation init/config responses
- * - Handle assistant / user / interruption / directive messages
+ * - Handle assistant / user / interruption / tool call messages
  *
  * This is pure dispatch logic — it holds no state of its own.
  */
@@ -115,10 +115,21 @@ export class MessageProtocolHandler {
       if (interruption) this.callbacks.onInterrupt?.(interruption.toObject());
     }
 
-    // Directive / action
-    if (response.hasDirective()) {
-      const directive = response.getDirective();
-      if (directive) this.callbacks.onDirective?.(directive.toObject());
+    // Tool call (server invokes a tool — client may need to act)
+    if (response.hasToolcall()) {
+      const toolCall = response.getToolcall();
+      if (toolCall) this.callbacks.onToolCall?.(toolCall.toObject());
+    }
+
+    // Tool call result (server-side tool completed — informational)
+    if (response.hasToolcallresult()) {
+      const result = response.getToolcallresult();
+      if (result) this.callbacks.onToolCallResult?.(result.toObject());
+    }
+
+    // Disconnection (server requests disconnect)
+    if (response.hasDisconnection()) {
+      this.callbacks.onDisconnected?.();
     }
 
     // Pipeline conversation event (STT, TTS, LLM, session, etc.)

src/audio/webrtc-grpc-transport.ts

@@ -209,6 +209,9 @@ export class WebRTCGrpcTransport {
   sendConversationInitialization(): void { this.signaling.sendConversationInitialization(); }
   sendConversationConfiguration(): void { this.signaling.sendConversationConfiguration(); }
   sendText(text: string): void { this.signaling.sendText(text); }
+  sendToolCallResult(id: string, toolId: string, name: string, action: 0 | 1 | 2, result: Record<string, string>): void {
+    this.signaling.sendToolCallResult(id, toolId, name, action, result);
+  }
 
   // ---------------------------------------------------------------------------
   // Audio controls (delegated to audio & peer managers)

src/clients/protos/agentkit_pb.d.ts

@@ -90,20 +90,15 @@ export class TalkOutput extends jspb.Message {
   getAssistant(): talk_api_pb.ConversationAssistantMessage | undefined;
   setAssistant(value?: talk_api_pb.ConversationAssistantMessage): void;
 
-  hasTool(): boolean;
-  clearTool(): void;
-  getTool(): talk_api_pb.ConversationToolCall | undefined;
-  setTool(value?: talk_api_pb.ConversationToolCall): void;
+  hasToolcall(): boolean;
+  clearToolcall(): void;
+  getToolcall(): talk_api_pb.ConversationToolCall | undefined;
+  setToolcall(value?: talk_api_pb.ConversationToolCall): void;
 
-  hasToolresult(): boolean;
-  clearToolresult(): void;
-  getToolresult(): talk_api_pb.ConversationToolResult | undefined;
-  setToolresult(value?: talk_api_pb.ConversationToolResult): void;
-
-  hasDirective(): boolean;
-  clearDirective(): void;
-  getDirective(): talk_api_pb.ConversationDirective | undefined;
-  setDirective(value?: talk_api_pb.ConversationDirective): void;
+  hasToolcallresult(): boolean;
+  clearToolcallresult(): void;
+  getToolcallresult(): talk_api_pb.ConversationToolCallResult | undefined;
+  setToolcallresult(value?: talk_api_pb.ConversationToolCallResult): void;
 
   hasError(): boolean;
   clearError(): void;
@@ -128,9 +123,8 @@ export namespace TalkOutput {
     initialization?: talk_api_pb.ConversationInitialization.AsObject,
     interruption?: talk_api_pb.ConversationInterruption.AsObject,
     assistant?: talk_api_pb.ConversationAssistantMessage.AsObject,
-    tool?: talk_api_pb.ConversationToolCall.AsObject,
-    toolresult?: talk_api_pb.ConversationToolResult.AsObject,
-    directive?: talk_api_pb.ConversationDirective.AsObject,
+    toolcall?: talk_api_pb.ConversationToolCall.AsObject,
+    toolcallresult?: talk_api_pb.ConversationToolCallResult.AsObject,
     error?: common_pb.Error.AsObject,
   }
 
@@ -139,9 +133,8 @@ export namespace TalkOutput {
     INITIALIZATION = 9,
     INTERRUPTION = 10,
     ASSISTANT = 12,
-    TOOL = 13,
-    TOOLRESULT = 14,
-    DIRECTIVE = 16,
+    TOOLCALL = 13,
+    TOOLCALLRESULT = 14,
     ERROR = 15,
   }
 }