Allowing the evaluation to not call commit_tool (volcengine#2041)

* 允许评估时不调用commit_tool

* prompt回滚

Author: fujiajie666

bot/vikingbot/agent/loop.py

@@ -197,11 +197,16 @@ async def _publish_thinking_event(
             progress to users during long-running operations.
 
         Example:
-            >>> await self._publish_thinking_event(
-            ...     session_key=SessionKey(channel="telegram", chat_id="123"),
-            ...     event_type=OutboundEventType.TOOL_START,
-            ...     content="Executing web search..."
-            ... )
+            async def notify_tool_call() -> None:
+                await self._publish_thinking_event(
+                    session_key=SessionKey(
+                        type="telegram",
+                        channel_id="default",
+                        chat_id="123",
+                    ),
+                    event_type=OutboundEventType.TOOL_CALL,
+                    content="Executing web search...",
+                )
         """
         await self.bus.publish_outbound(
             OutboundMessage(
@@ -267,6 +272,7 @@ async def _run_agent_loop(
         sender_id: str | None = None,
         ov_tools_enable: bool = True,
         memory_user_ids: list[str] | None = None,
+        disabled_tools: list[str] | None = None,
     ) -> tuple[str | None, str | None, list[dict], dict[str, int], int]:
         """
         Run the core agent loop: call LLM, execute tools, repeat until done.
@@ -277,6 +283,7 @@ async def _run_agent_loop(
             publish_events: Whether to publish ITERATION/REASONING/TOOL_CALL events to the bus
             ov_tools_enable: Whether to enable OpenViking tools for this session
             memory_user_ids: List of user IDs for memory retrieval
+            disabled_tools: Tool names to hide from the model for this request
 
         Returns:
             tuple of (final_content, final_reasoning_content, tools_used, token_usage, iteration)
@@ -306,7 +313,10 @@ async def _run_agent_loop(
 
             response = await self.provider.chat(
                 messages=messages,
-                tools=self.tools.get_definitions(ov_tools_enable=ov_tools_enable),
+                tools=self.tools.get_definitions(
+                    ov_tools_enable=ov_tools_enable,
+                    disabled_tools=disabled_tools,
+                ),
                 model=self.model,
                 session_id=session_key.safe_name(),
             )
@@ -516,6 +526,9 @@ async def check_long_running():
             session = self.sessions.get_or_create(session_key, skip_heartbeat=skip_heartbeat)
 
             ov_tools_enable = self._get_ov_tools_enable(session_key)
+            disabled_tools = msg.metadata.get("disabled_tools", []) if msg.metadata else []
+            if not isinstance(disabled_tools, list):
+                disabled_tools = []
             # Get profile_user_list from channel config
             profile_user_list = []
             # Try to get memory_users from message metadata first (CLI mode), then from channel config
@@ -668,6 +681,7 @@ async def check_long_running():
                     sender_id=msg.sender_id,
                     ov_tools_enable=ov_tools_enable,
                     memory_user_ids=memory_user,
+                    disabled_tools=disabled_tools,
                 )
 
             # Log response preview
@@ -1046,4 +1060,4 @@ async def process_direct(
         )
 
         response = await self._process_message(msg)
-        return response.content if response else ""
+        return response.content if response else ""

bot/vikingbot/agent/tools/registry.py

@@ -100,7 +100,11 @@ def has(self, name: str) -> bool:
         """
         return name in self._tools
 
-    def get_definitions(self, ov_tools_enable: bool = True) -> list[dict[str, Any]]:
+    def get_definitions(
+        self,
+        ov_tools_enable: bool = True,
+        disabled_tools: list[str] | None = None,
+    ) -> list[dict[str, Any]]:
         """
         Get all tool definitions in OpenAI format.
 
@@ -110,6 +114,7 @@ def get_definitions(self, ov_tools_enable: bool = True) -> list[dict[str, Any]]:
         Args:
             ov_tools_enable: Whether to include OpenViking tools. If False,
                 tools with names starting with "openviking_" will be excluded.
+            disabled_tools: Tool names to hide from the model for this request.
 
         Returns:
             List of tool schemas in OpenAI format, where each schema contains
@@ -123,6 +128,9 @@ def get_definitions(self, ov_tools_enable: bool = True) -> list[dict[str, Any]]:
         tools = self._tools.values()
         if not ov_tools_enable:
             tools = [tool for tool in tools if not tool.name.startswith("openviking_")]
+        if disabled_tools:
+            disabled = set(disabled_tools)
+            tools = [tool for tool in tools if tool.name not in disabled]
         return [tool.to_schema() for tool in tools]
 
     async def execute(
@@ -234,4 +242,4 @@ def __len__(self) -> int:
         return len(self._tools)
 
     def __contains__(self, name: str) -> bool:
-        return name in self._tools
+        return name in self._tools

bot/vikingbot/channels/openapi.py

@@ -48,6 +48,7 @@ def __init__(self):
         self.final_content: Optional[str] = None
         self.response_id: Optional[str] = None
         self.relevant_memories: Optional[str] = None
+        self.token_usage: Dict[str, int] = {}
         self.event = asyncio.Event()
         self.stream_queue: asyncio.Queue[Optional[ChatStreamEvent]] = asyncio.Queue()
 
@@ -202,6 +203,7 @@ async def send(self, msg: OutboundMessage) -> None:
             ):
                 pending.set_response_id(msg.response_id)
                 pending.relevant_memories = (msg.metadata or {}).get("relevant_memories")
+                pending.token_usage = msg.token_usage
                 await pending.add_event(
                     "response",
                     {"content": msg.content or "", "response_id": msg.response_id},
@@ -227,6 +229,7 @@ async def send(self, msg: OutboundMessage) -> None:
         if msg.event_type == OutboundEventType.RESPONSE:
             # Final response - add to stream first
             pending.set_response_id(msg.response_id)
+            pending.token_usage = msg.token_usage
             pending.relevant_memories = (msg.metadata or {}).get("relevant_memories")
             await pending.add_event(
                 "response",
@@ -479,6 +482,7 @@ async def _handle_chat(self, request: ChatRequest) -> ChatResponse:
                 session_key=session_key,
                 sender_id=user_id,
                 content=content,
+                metadata={"disabled_tools": request.disabled_tools},
             )
 
             await self.bus.publish_inbound(msg)
@@ -498,6 +502,7 @@ async def _handle_chat(self, request: ChatRequest) -> ChatResponse:
                 message=response_content,
                 events=pending.events if pending.events else None,
                 relevant_memories=pending.relevant_memories,
+                token_usage=pending.token_usage,
             )
 
         except HTTPException:
@@ -543,6 +548,7 @@ async def event_generator():
                     session_key=session_key,
                     sender_id=user_id,
                     content=request.message,
+                    metadata={"disabled_tools": request.disabled_tools},
                 )
 
                 await self.bus.publish_inbound(msg)
@@ -622,6 +628,7 @@ async def _handle_bot_chat(self, channel_id: str, request: ChatRequest) -> ChatR
                 sender_id=user_id,
                 content=content,
                 need_reply=request.need_reply,
+                metadata={"disabled_tools": request.disabled_tools},
             )
 
             await self.bus.publish_inbound(msg)
@@ -641,6 +648,7 @@ async def _handle_bot_chat(self, channel_id: str, request: ChatRequest) -> ChatR
                 message=response_content,
                 events=pending.events if pending.events else None,
                 relevant_memories=pending.relevant_memories,
+                token_usage=pending.token_usage,
             )
 
         except HTTPException:
@@ -693,6 +701,7 @@ async def event_generator():
                     session_key=session_key,
                     sender_id=user_id,
                     content=request.message,
+                    metadata={"disabled_tools": request.disabled_tools},
                 )
 
                 await self.bus.publish_inbound(msg)
@@ -895,4 +904,4 @@ def get_openapi_router(bus: MessageBus, config: Config) -> APIRouter:
         )
         logger.info(f"Subscribed to bot_api channel: {channel_id}")
 
-    return channel.get_router()
+    return channel.get_router()

bot/vikingbot/channels/openapi_models.py

@@ -60,6 +60,10 @@ class ChatRequest(BaseModel):
     channel_id: Optional[str] = Field(
         default=None, description="Channel ID for multi-channel routing (optional)"
     )
+    disabled_tools: List[str] = Field(
+        default_factory=list,
+        description="Tool names to hide for this request",
+    )
 
 
 class ChatResponse(BaseModel):
@@ -75,6 +79,10 @@ class ChatResponse(BaseModel):
         default=None,
         description="OpenViking memories assembled during _process_message",
     )
+    token_usage: Dict[str, int] = Field(
+        default_factory=dict,
+        description="Token usage statistics (prompt_tokens, completion_tokens, total_tokens)",
+    )
     timestamp: datetime = Field(default_factory=datetime.now, description="Response timestamp")
 
 
@@ -168,4 +176,4 @@ class ErrorResponse(BaseModel):
 
     error: str = Field(..., description="Error message")
     code: Optional[str] = Field(default=None, description="Error code")
-    detail: Optional[str] = Field(default=None, description="Detailed error information")
+    detail: Optional[str] = Field(default=None, description="Detailed error information")