diff --git a/.changeset/honor-ignore-on-enter-toolsets.md b/.changeset/honor-ignore-on-enter-toolsets.md new file mode 100644 index 000000000..6370558fd --- /dev/null +++ b/.changeset/honor-ignore-on-enter-toolsets.md @@ -0,0 +1,5 @@ +--- +'@livekit/agents': patch +--- + +Honor `ToolFlag.IGNORE_ON_ENTER` for tools nested inside `Toolset`s. diff --git a/agents/src/beta/tools/end_call.ts b/agents/src/beta/tools/end_call.ts index 962deda36..d444dd17e 100644 --- a/agents/src/beta/tools/end_call.ts +++ b/agents/src/beta/tools/end_call.ts @@ -8,6 +8,7 @@ import { RealtimeModel, type ToolCalledEvent, type ToolCompletedEvent, + ToolFlag, Toolset, tool, } from '../../llm/index.js'; @@ -68,6 +69,8 @@ export type EndCallToolOptions = { deleteRoom?: boolean; /** Tool output to the LLM for generating the tool response. */ endInstructions?: string | null; + /** Hide the tool during onEnter so the model can't end the call while greeting. */ + ignoreOnEnter?: boolean; /** Callback to call when the tool is called. */ onToolCalled?: (event: EndCallToolCalledEvent) => Promise | void; /** Callback to call when the tool is completed. */ @@ -81,6 +84,7 @@ export function createEndCallTool({ extraDescription = '', deleteRoom = true, endInstructions = 'say goodbye to the user', + ignoreOnEnter = false, onToolCalled, onToolCompleted, }: EndCallToolOptions = {}): Toolset { @@ -116,6 +120,7 @@ export function createEndCallTool({ tool({ name: 'end_call', description: `${END_CALL_DESCRIPTION}\n${extraDescription}`, + flags: ignoreOnEnter ? ToolFlag.IGNORE_ON_ENTER : ToolFlag.NONE, execute: async (_args, { ctx, abortSignal }) => { log().debug('end_call tool called'); const session = ctx.session; diff --git a/agents/src/llm/tool_context.test.ts b/agents/src/llm/tool_context.test.ts index 10bfdc90f..15d492fec 100644 --- a/agents/src/llm/tool_context.test.ts +++ b/agents/src/llm/tool_context.test.ts @@ -5,6 +5,7 @@ import { describe, expect, it } from 'vitest'; import { z } from 'zod'; import * as z3 from 'zod/v3'; import * as z4 from 'zod/v4'; +import { createEndCallTool } from '../beta/tools/end_call.js'; import { voice } from '../index.js'; import { CONFIRM_DUPLICATE_PARAM, @@ -15,6 +16,7 @@ import { type ToolOptions, Toolset, type ToolsetContext, + isFunctionTool, tool, } from './tool_context.js'; import { createToolOptions, oaiParams } from './utils.js'; @@ -701,6 +703,32 @@ describe('ToolContext', () => { expect(ctx.getFunctionTool('b')).toBe(b); }); + it('_exclude hides a toolset member but keeps the toolset', () => { + const a = makeFn('a'); + const b = makeFn('b'); + const direct = makeFn('direct'); + const ts = new Toolset({ id: 'set', tools: [a, b] }); + const ctx = new ToolContext([ts, direct]); + + ctx._exclude([a]); + + expect(ctx.getFunctionTool('a')).toBeUndefined(); + expect(ctx.flatten()).not.toContain(a); + expect(ctx.getFunctionTool('b')).toBe(b); + expect(ctx.getFunctionTool('direct')).toBe(direct); + expect(ctx.toolsets).toEqual([ts]); + }); + + it('_exclude with no tools is a no-op', () => { + const a = makeFn('a'); + const b = makeFn('b'); + const ctx = new ToolContext([a, b]); + + ctx._exclude([]); + + expect(Object.keys(ctx.functionTools).sort()).toEqual(['a', 'b']); + }); + it('copy() yields an independent context with the same tools', () => { const a = makeFn('a'); const ctx = new ToolContext([a]); @@ -768,6 +796,22 @@ describe('ToolContext', () => { }); }); +describe('createEndCallTool', () => { + it('sets IGNORE_ON_ENTER only when requested', () => { + const [defaultTool] = createEndCallTool().tools; + expect(defaultTool).toBeDefined(); + expect(isFunctionTool(defaultTool)).toBe(true); + expect(isFunctionTool(defaultTool) && defaultTool.flags & ToolFlag.IGNORE_ON_ENTER).toBe(0); + + const [ignoredTool] = createEndCallTool({ ignoreOnEnter: true }).tools; + expect(ignoredTool).toBeDefined(); + expect(isFunctionTool(ignoredTool)).toBe(true); + expect( + isFunctionTool(ignoredTool) && ignoredTool.flags & ToolFlag.IGNORE_ON_ENTER, + ).toBeTruthy(); + }); +}); + describe('Toolset', () => { const makeFn = (name: string) => tool({ diff --git a/agents/src/llm/tool_context.ts b/agents/src/llm/tool_context.ts index 6b947a021..d7df12934 100644 --- a/agents/src/llm/tool_context.ts +++ b/agents/src/llm/tool_context.ts @@ -515,7 +515,12 @@ export class ToolContext { } updateTools(tools: ToolContextInit): void { + this._updateTools(tools); + } + + private _updateTools(tools: ToolContextInit, exclude: readonly Tool[] = []): void { const normalizedTools = normalizeToolContextInit(tools); + const excludedTools = new Set(exclude); this._tools = normalizedTools; this._functionToolsMap = new Map(); this._providerTools = []; @@ -523,6 +528,10 @@ export class ToolContext { // eslint-disable-next-line @typescript-eslint/no-explicit-any -- accepts any tool shape const addTool = (tool: any): void => { + if (excludedTools.has(tool)) { + return; + } + if (isToolset(tool)) { for (const inner of tool.tools) { addTool(inner); @@ -556,6 +565,35 @@ export class ToolContext { } } + private _syncFlattened(tools: readonly Tool[]): void { + const current = this.flatten(); + const currentTools = new Set(current); + const nextTools = new Set(tools); + if (currentTools.size === nextTools.size && current.every((tool) => nextTools.has(tool))) { + return; + } + + const added: Extract, Tool>[] = tools + .filter((tool) => !currentTools.has(tool)) + .filter( + (tool): tool is Extract, Tool> => + isFunctionTool(tool) || isProviderTool(tool), + ); + const removed = current.filter((tool) => !nextTools.has(tool)); + const structured = this._tools.filter((tool) => !removed.includes(tool as Tool)); + this._updateTools([...structured, ...added], removed); + } + + /** Hide tools from the callable set while keeping their toolsets intact. @internal */ + _exclude(tools: readonly Tool[]): void { + if (tools.length === 0) { + return; + } + + const excludedTools = new Set(tools); + this._syncFlattened(this.flatten().filter((tool) => !excludedTools.has(tool))); + } + copy(): ToolContext { return new ToolContext([...this._tools]); } diff --git a/agents/src/voice/agent_activity.test.ts b/agents/src/voice/agent_activity.test.ts index e8c36bd00..e7de6cef5 100644 --- a/agents/src/voice/agent_activity.test.ts +++ b/agents/src/voice/agent_activity.test.ts @@ -21,7 +21,7 @@ import { LLM, type LLMStream } from '../llm/llm.js'; import { type Tool, ToolContext, ToolFlag, Toolset, tool } from '../llm/tool_context.js'; import { Future, Task } from '../utils.js'; import { _getActivityTaskInfo } from './agent.js'; -import { AgentActivity } from './agent_activity.js'; +import { AgentActivity, onEnterStorage } from './agent_activity.js'; import type { PreemptiveGenerationInfo } from './audio_recognition.js'; import type { AgentSessionEventTypes, UserInputTranscribedEvent } from './events.js'; import { SpeechHandle } from './speech_handle.js'; @@ -582,6 +582,77 @@ describe('AgentActivity - onPreemptiveGeneration guards', () => { }); }); +describe('AgentActivity - onEnter ignored tools', () => { + const makeFn = (name: string, flags = ToolFlag.NONE) => + tool({ name, description: `${name} tool`, flags, execute: async () => name }); + + function buildIgnoredToolsActivity() { + const activity = Object.create(AgentActivity.prototype) as AgentActivity; + Object.assign(activity, { + agent: {}, + agentSession: {}, + }); + return activity; + } + + it('returns bare and toolset-nested IGNORE_ON_ENTER tools only inside this onEnter', () => { + const endCall = makeFn('end_call', ToolFlag.IGNORE_ON_ENTER); + const keep = makeFn('keep'); + const bareIgnored = makeFn('bare_ignored', ToolFlag.IGNORE_ON_ENTER); + const bareKeep = makeFn('bare_keep'); + const toolset = new Toolset({ id: 'ts', tools: [endCall, keep] }); + const toolCtx = new ToolContext([toolset, bareIgnored, bareKeep]); + const activity = buildIgnoredToolsActivity(); + + expect(activity._onEnterIgnoredTools(toolCtx)).toEqual([]); + + onEnterStorage.run({ session: activity.agentSession, agent: activity.agent }, () => { + expect( + activity + ._onEnterIgnoredTools(toolCtx) + .map((t) => t.id) + .sort(), + ).toEqual(['bare_ignored', 'end_call']); + }); + + onEnterStorage.run({ session: activity.agentSession, agent: {} as never }, () => { + expect(activity._onEnterIgnoredTools(toolCtx)).toEqual([]); + }); + }); + + it('hides ignored bare and toolset tools while preserving normal tools for nested replies', async () => { + const endCall = makeFn('end_call', ToolFlag.IGNORE_ON_ENTER); + const keep = makeFn('keep'); + const bareIgnored = makeFn('bare_ignored', ToolFlag.IGNORE_ON_ENTER); + const toolset = new Toolset({ id: 'ts', tools: [endCall, keep] }); + const activity = buildIgnoredToolsActivity(); + + await onEnterStorage.run( + { session: activity.agentSession, agent: activity.agent }, + async () => { + const greetingTools = new ToolContext([toolset, bareIgnored]); + greetingTools._exclude(activity._onEnterIgnoredTools(greetingTools)); + expect(greetingTools.flatten().map((t) => t.id)).toEqual(['keep']); + expect(greetingTools.toolsets).toEqual([toolset]); + + await Promise.resolve(); + + const toolReplyTools = new ToolContext([toolset, bareIgnored]); + toolReplyTools._exclude(activity._onEnterIgnoredTools(toolReplyTools)); + expect(toolReplyTools.flatten().map((t) => t.id)).toEqual(['keep']); + }, + ); + + const restoredTools = new ToolContext([toolset, bareIgnored]); + expect( + restoredTools + .flatten() + .map((t) => t.id) + .sort(), + ).toEqual(['bare_ignored', 'end_call', 'keep']); + }); +}); + /** * Regression test for the dynamic-toolset push path. * diff --git a/agents/src/voice/agent_activity.ts b/agents/src/voice/agent_activity.ts index 9fd906783..abaa0a2c3 100644 --- a/agents/src/voice/agent_activity.ts +++ b/agents/src/voice/agent_activity.ts @@ -41,13 +41,13 @@ import { RealtimeModel, type RealtimeModelError, type RealtimeSession, + type Tool, type ToolChoice, ToolContext, type ToolContextEntry, type ToolContextLike, ToolFlag, isFunctionTool, - isToolset, toToolContext, } from '../llm/index.js'; import type { LLMError } from '../llm/llm.js'; @@ -2136,19 +2136,8 @@ export class AgentActivity implements RecognitionHooks { instructions = concatInstructions(this.agent.instructions, '\n', instructions); } - // Filter out tools with IGNORE_ON_ENTER flag when generateReply is called inside onEnter - const onEnterData = onEnterStorage.getStore(); - const shouldFilterTools = - onEnterData?.agent === this.agent && onEnterData?.session === this.agentSession; - - const tools: ToolContext = shouldFilterTools - ? new ToolContext( - this.tools.tools.filter((t): boolean => { - if (isToolset(t) || !isFunctionTool(t)) return true; - return !(t.flags & ToolFlag.IGNORE_ON_ENTER); - }), - ) - : this.tools; + const tools = this.tools; + tools._exclude(this._onEnterIgnoredTools(tools)); const task = this.createSpeechTask({ taskFn: (abortController: AbortController) => @@ -2234,6 +2223,19 @@ export class AgentActivity implements RecognitionHooks { } } + _onEnterIgnoredTools(toolCtx: ToolContext): Tool[] { + const onEnterData = onEnterStorage.getStore(); + if (onEnterData?.agent !== this.agent || onEnterData.session !== this.agentSession) { + return []; + } + + return toolCtx + .flatten() + .filter( + (tool): tool is Tool => isFunctionTool(tool) && !!(tool.flags & ToolFlag.IGNORE_ON_ENTER), + ); + } + /** * Commit a user turn whose reply is being skipped: append the transcript to the * agent chat context (when non-empty) without triggering reply generation. @@ -3910,6 +3912,15 @@ export class AgentActivity implements RecognitionHooks { } const originalToolChoice = this.toolChoice; + let originalTools: ToolContext | undefined; + const turnTools = this.realtimeSession.tools; + const onEnterIgnoredTools = this._onEnterIgnoredTools(turnTools); + if (onEnterIgnoredTools.length > 0) { + originalTools = this.realtimeSession.tools; + turnTools._exclude(onEnterIgnoredTools); + await this.realtimeSession.updateTools(turnTools); + } + if (toolChoice !== undefined) { this.realtimeSession.updateOptions({ toolChoice }); } @@ -3942,6 +3953,9 @@ export class AgentActivity implements RecognitionHooks { if (toolChoice !== undefined && toolChoice !== originalToolChoice) { this.realtimeSession.updateOptions({ toolChoice: originalToolChoice }); } + if (originalTools !== undefined) { + await this.realtimeSession.updateTools(originalTools); + } } }