From 7129d54b07e63373bf1eb05bcd14374a9dbfc305 Mon Sep 17 00:00:00 2001 From: Juan Jose Soliz Priore Date: Sun, 15 Mar 2026 01:53:19 -0300 Subject: [PATCH 1/2] Document custom LLM context messages for tools Added details on custom LLM context messages for tools, including how to override the default message with specific information about the generated embed. --- .../plugin/development/rich-ui.mdx | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/docs/features/extensibility/plugin/development/rich-ui.mdx b/docs/features/extensibility/plugin/development/rich-ui.mdx index 704755e57..ee3a36d26 100644 --- a/docs/features/extensibility/plugin/development/rich-ui.mdx +++ b/docs/features/extensibility/plugin/development/rich-ui.mdx @@ -46,6 +46,30 @@ def create_visualization_tool(self, data: str) -> HTMLResponse: return HTMLResponse(content=html_content, headers=headers) ``` +### Custom LLM Context Messages (Tools) + +By default, when a Tool successfully returns an HTMLResponse embed, the LLM is sent a generic system message: "{tool_function_name}: Embedded UI result is active and visible to the user." + +While this prevents the LLM from trying to read raw HTML, it also hides all the context of what actually happened. If you want the LLM to know specific details about the generated embed—like the parameters used, the status of the action, or what the user is seeing—you can override this generic message using the optional Tool-Result-Message header. + +```python +from fastapi.responses import HTMLResponse +def generate_music_tool(self, genre: str, duration: int) -> HTMLResponse: + """ + Generates music and embeds a player in the chat. + """ + # Your HTML player generation logic + html_content = f"" + headers = { + "Content-Disposition": "inline", + # This message is sent to the LLM instead of the generic default + "Tool-Result-Message": f"Audio player embedded successfully. A {duration}-second {genre} track was generated. The user can play or download it from the player above." + } + + return HTMLResponse(content=html_content, headers=headers) +``` + +This ensures the LLM receives actionable, semantic feedback about the visualization or embedded UI, allowing it to provide relevant follow-up responses to the user. ## Action Usage Actions work exactly the same way. The rich UI embed is delivered to the chat via the event emitter: From 60bd5387a9ed23a720e41ce4b69ce859a20d7c88 Mon Sep 17 00:00:00 2001 From: Haervwe Date: Sun, 15 Mar 2026 13:58:24 -0300 Subject: [PATCH 2/2] docs: enhance LLM context messages for HTMLResponse embeds with detailed feedback --- .../extensibility/plugin/development/rich-ui.mdx | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/docs/features/extensibility/plugin/development/rich-ui.mdx b/docs/features/extensibility/plugin/development/rich-ui.mdx index ee3a36d26..156abacd2 100644 --- a/docs/features/extensibility/plugin/development/rich-ui.mdx +++ b/docs/features/extensibility/plugin/development/rich-ui.mdx @@ -50,7 +50,8 @@ def create_visualization_tool(self, data: str) -> HTMLResponse: By default, when a Tool successfully returns an HTMLResponse embed, the LLM is sent a generic system message: "{tool_function_name}: Embedded UI result is active and visible to the user." -While this prevents the LLM from trying to read raw HTML, it also hides all the context of what actually happened. If you want the LLM to know specific details about the generated embed—like the parameters used, the status of the action, or what the user is seeing—you can override this generic message using the optional Tool-Result-Message header. +While this prevents the LLM from trying to read raw HTML, it also hides all the context of what actually happened. If you want the LLM to know specific details about the generated embed—like the parameters used, the status of the action, or what the user is seeing—you can override this generic message returning +a tuple of `(HTMLResponse, llm_message)` instead of just the `HTMLResponse`. The `llm_message` string or object will be sent to the LLM as context instead of the generic message, allowing you to provide rich semantic information about the embed: ```python from fastapi.responses import HTMLResponse @@ -62,14 +63,17 @@ def generate_music_tool(self, genre: str, duration: int) -> HTMLResponse: html_content = f"" headers = { "Content-Disposition": "inline", - # This message is sent to the LLM instead of the generic default - "Tool-Result-Message": f"Audio player embedded successfully. A {duration}-second {genre} track was generated. The user can play or download it from the player above." + } - - return HTMLResponse(content=html_content, headers=headers) + + llm_message = f"Audio player embedded successfully. A {duration}-second {genre} track was generated. The user can play or download it from the player above. {download_link}" + + return HTMLResponse(content=html_content, headers=headers), llm_message ``` This ensures the LLM receives actionable, semantic feedback about the visualization or embedded UI, allowing it to provide relevant follow-up responses to the user. + + ## Action Usage Actions work exactly the same way. The rich UI embed is delivered to the chat via the event emitter: