feat(agentkit): update MLLM and LLM vendor wrappers for v2.7

digitallysavvy · digitallysavvy · commit 9df782b46d87 · 2026-05-20T20:57:54.000-04:00
Adds xAI Grok Realtime and Vertex AI MLLM wrappers, and aligns MLLM
config serialization with the generated core types. LLM vendors now
accept typed greeting_configs and serialize them through the generated
model shape, including interruptable.
diff --git a/src/agora_agent/agentkit/vendors/llm.py b/src/agora_agent/agentkit/vendors/llm.py
@@ -1,9 +1,14 @@
-from typing import Any, Dict, List, Optional
+from typing import Any, Dict, List, Optional, Union
 
 from pydantic import BaseModel, ConfigDict, Field
 
+from ...agents.types.start_agents_request_properties_llm_greeting_configs import (
+    StartAgentsRequestPropertiesLlmGreetingConfigs,
+)
 from .base import BaseLLM
 
+LlmGreetingConfigs = Union[StartAgentsRequestPropertiesLlmGreetingConfigs, Dict[str, Any]]
+
 
 def _ensure_mcp_transport(servers: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
     """Ensure each MCP server has transport set (API requires it). Default to streamable_http."""
@@ -15,6 +20,14 @@ def _ensure_mcp_transport(servers: List[Dict[str, Any]]) -> List[Dict[str, Any]]
         result.append(item)
     return result
 
+
+def _dump_optional_model(value: Any) -> Any:
+    if hasattr(value, "model_dump"):
+        return value.model_dump(exclude_none=True)
+    if hasattr(value, "dict"):
+        return value.dict(exclude_none=True)
+    return value
+
 class OpenAIOptions(BaseModel):
     model_config = ConfigDict(extra="forbid")
 
@@ -31,7 +44,7 @@ class OpenAIOptions(BaseModel):
     params: Optional[Dict[str, Any]] = Field(default=None)
     headers: Optional[Dict[str, str]] = Field(default=None)
     output_modalities: Optional[List[str]] = Field(default=None)
-    greeting_configs: Optional[Dict[str, Any]] = Field(default=None)
+    greeting_configs: Optional[LlmGreetingConfigs] = Field(default=None)
     template_variables: Optional[Dict[str, str]] = Field(default=None)
     vendor: Optional[str] = Field(default=None)
     mcp_servers: Optional[List[Dict[str, Any]]] = Field(default=None)
@@ -74,7 +87,7 @@ def to_config(self) -> Dict[str, Any]:
         if self.options.output_modalities is not None:
             config["output_modalities"] = self.options.output_modalities
         if self.options.greeting_configs is not None:
-            config["greeting_configs"] = self.options.greeting_configs
+            config["greeting_configs"] = _dump_optional_model(self.options.greeting_configs)
         if self.options.template_variables is not None:
             config["template_variables"] = self.options.template_variables
         if self.options.vendor is not None:
@@ -104,7 +117,7 @@ class AzureOpenAIOptions(BaseModel):
     params: Optional[Dict[str, Any]] = Field(default=None)
     headers: Optional[Dict[str, str]] = Field(default=None)
     output_modalities: Optional[List[str]] = Field(default=None)
-    greeting_configs: Optional[Dict[str, Any]] = Field(default=None)
+    greeting_configs: Optional[LlmGreetingConfigs] = Field(default=None)
     template_variables: Optional[Dict[str, str]] = Field(default=None)
     vendor: Optional[str] = Field(default=None)
     mcp_servers: Optional[List[Dict[str, Any]]] = Field(default=None)
@@ -150,7 +163,7 @@ def to_config(self) -> Dict[str, Any]:
         if self.options.output_modalities is not None:
             config["output_modalities"] = self.options.output_modalities
         if self.options.greeting_configs is not None:
-            config["greeting_configs"] = self.options.greeting_configs
+            config["greeting_configs"] = _dump_optional_model(self.options.greeting_configs)
         if self.options.template_variables is not None:
             config["template_variables"] = self.options.template_variables
         if self.options.mcp_servers is not None:
@@ -177,7 +190,7 @@ class AnthropicOptions(BaseModel):
     params: Optional[Dict[str, Any]] = Field(default=None)
     headers: Optional[Dict[str, str]] = Field(default=None)
     output_modalities: Optional[List[str]] = Field(default=None)
-    greeting_configs: Optional[Dict[str, Any]] = Field(default=None)
+    greeting_configs: Optional[LlmGreetingConfigs] = Field(default=None)
     template_variables: Optional[Dict[str, str]] = Field(default=None)
     vendor: Optional[str] = Field(default=None)
     mcp_servers: Optional[List[Dict[str, Any]]] = Field(default=None)
@@ -216,7 +229,7 @@ def to_config(self) -> Dict[str, Any]:
         if self.options.output_modalities is not None:
             config["output_modalities"] = self.options.output_modalities
         if self.options.greeting_configs is not None:
-            config["greeting_configs"] = self.options.greeting_configs
+            config["greeting_configs"] = _dump_optional_model(self.options.greeting_configs)
         if self.options.template_variables is not None:
             config["template_variables"] = self.options.template_variables
         if self.options.vendor is not None:
@@ -246,7 +259,7 @@ class GeminiOptions(BaseModel):
     params: Optional[Dict[str, Any]] = Field(default=None)
     headers: Optional[Dict[str, str]] = Field(default=None)
     output_modalities: Optional[List[str]] = Field(default=None)
-    greeting_configs: Optional[Dict[str, Any]] = Field(default=None)
+    greeting_configs: Optional[LlmGreetingConfigs] = Field(default=None)
     template_variables: Optional[Dict[str, str]] = Field(default=None)
     vendor: Optional[str] = Field(default=None)
     mcp_servers: Optional[List[Dict[str, Any]]] = Field(default=None)
@@ -287,7 +300,7 @@ def to_config(self) -> Dict[str, Any]:
         if self.options.output_modalities is not None:
             config["output_modalities"] = self.options.output_modalities
         if self.options.greeting_configs is not None:
-            config["greeting_configs"] = self.options.greeting_configs
+            config["greeting_configs"] = _dump_optional_model(self.options.greeting_configs)
         if self.options.template_variables is not None:
             config["template_variables"] = self.options.template_variables
         if self.options.vendor is not None:
diff --git a/src/agora_agent/agentkit/vendors/mllm.py b/src/agora_agent/agentkit/vendors/mllm.py
@@ -1,3 +1,4 @@
+import warnings
 from typing import Any, Dict, List, Optional
 
 from pydantic import BaseModel, ConfigDict, Field
@@ -22,9 +23,7 @@ class OpenAIRealtimeOptions(BaseModel):
     messages: Optional[List[Dict[str, Any]]] = Field(default=None, description="Conversation messages")
     params: Optional[Dict[str, Any]] = Field(default=None, description="Additional parameters")
     turn_detection: Optional[MllmTurnDetectionConfig] = Field(default=None, description="MLLM turn detection configuration")
-    predefined_tools: Optional[List[str]] = Field(default=None, description="Predefined tools")
     failure_message: Optional[str] = Field(default=None, description="Message played on failure")
-    max_history: Optional[int] = Field(default=None, description="Maximum conversation history length")
 
 class OpenAIRealtime(BaseMLLM):
     def __init__(self, **kwargs: Any):
@@ -53,18 +52,97 @@ def to_config(self) -> Dict[str, Any]:
             config["output_modalities"] = self.options.output_modalities
         if self.options.messages is not None:
             config["messages"] = self.options.messages
-        if self.options.predefined_tools is not None:
-            config["predefined_tools"] = self.options.predefined_tools
         if self.options.failure_message is not None:
             config["failure_message"] = self.options.failure_message
-        if self.options.max_history is not None:
-            config["max_history"] = self.options.max_history
         if self.options.turn_detection is not None:
             config["turn_detection"] = self.options.turn_detection
 
         return config
 
 
+# xAI MLLM: use XaiGrok (product name, mllm.vendor "xai"). Do not use XaiRealtime—that name
+# is deprecated and reserved naming for future XaiSTT / XaiTTS cascading vendors.
+
+
+class XaiGrokOptions(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    api_key: str = Field(..., description="xAI API key")
+    url: str = Field(default="wss://api.x.ai/v1/realtime", description="xAI Realtime WebSocket URL")
+    voice: Optional[str] = Field(default=None, description="Voice identifier (e.g., eve or rex)")
+    language: Optional[str] = Field(default=None, description="Language code (e.g., en)")
+    sample_rate: Optional[int] = Field(default=None, description="Audio sample rate in Hz")
+    greeting_message: Optional[str] = Field(default=None, description="Agent greeting message")
+    input_modalities: Optional[List[str]] = Field(default=None, description="Input modalities")
+    output_modalities: Optional[List[str]] = Field(default=None, description="Output modalities")
+    messages: Optional[List[Dict[str, Any]]] = Field(default=None, description="Conversation messages")
+    params: Optional[Dict[str, Any]] = Field(default=None, description="Additional xAI parameters")
+    turn_detection: Optional[MllmTurnDetectionConfig] = Field(default=None, description="MLLM turn detection configuration")
+    failure_message: Optional[str] = Field(default=None, description="Message played on failure")
+
+
+class XaiGrok(BaseMLLM):
+    """xAI Grok MLLM vendor (`mllm.vendor`: ``xai``)."""
+
+    def __init__(self, **kwargs: Any):
+        self.options = XaiGrokOptions(**kwargs)
+
+    def to_config(self) -> Dict[str, Any]:
+        params: Dict[str, Any] = dict(self.options.params or {})
+        if self.options.voice is not None:
+            params["voice"] = self.options.voice
+        if self.options.language is not None:
+            params["language"] = self.options.language
+        if self.options.sample_rate is not None:
+            params["sample_rate"] = self.options.sample_rate
+
+        config: Dict[str, Any] = {
+            "vendor": "xai",
+            "api_key": self.options.api_key,
+            "url": self.options.url,
+            "params": params,
+        }
+
+        if self.options.greeting_message is not None:
+            config["greeting_message"] = self.options.greeting_message
+        if self.options.input_modalities is not None:
+            config["input_modalities"] = self.options.input_modalities
+        if self.options.output_modalities is not None:
+            config["output_modalities"] = self.options.output_modalities
+        if self.options.messages is not None:
+            config["messages"] = self.options.messages
+        if self.options.failure_message is not None:
+            config["failure_message"] = self.options.failure_message
+        if self.options.turn_detection is not None:
+            config["turn_detection"] = self.options.turn_detection
+
+        return config
+
+
+class XaiRealtimeOptions(XaiGrokOptions):
+    """Deprecated: use :class:`XaiGrokOptions` instead."""
+
+    def __init__(self, **data: Any):
+        warnings.warn(
+            "XaiRealtimeOptions is deprecated; use XaiGrokOptions instead.",
+            DeprecationWarning,
+            stacklevel=2,
+        )
+        super().__init__(**data)
+
+
+class XaiRealtime(XaiGrok):
+    """Deprecated: use :class:`XaiGrok` instead."""
+
+    def __init__(self, **kwargs: Any):
+        warnings.warn(
+            "XaiRealtime is deprecated; use XaiGrok instead.",
+            DeprecationWarning,
+            stacklevel=2,
+        )
+        super().__init__(**kwargs)
+
+
 class VertexAIOptions(BaseModel):
     model_config = ConfigDict(extra="forbid")
 
@@ -81,28 +159,24 @@ class VertexAIOptions(BaseModel):
     messages: Optional[List[Dict[str, Any]]] = Field(default=None, description="Conversation messages")
     additional_params: Optional[Dict[str, Any]] = Field(default=None, description="Additional parameters")
     turn_detection: Optional[MllmTurnDetectionConfig] = Field(default=None, description="MLLM turn detection configuration")
-    predefined_tools: Optional[List[str]] = Field(default=None, description="Predefined tools")
     failure_message: Optional[str] = Field(default=None, description="Message played on failure")
-    max_history: Optional[int] = Field(default=None, description="Maximum conversation history length")
 
 class VertexAI(BaseMLLM):
     def __init__(self, **kwargs: Any):
         self.options = VertexAIOptions(**kwargs)
 
     def to_config(self) -> Dict[str, Any]:
-        params: Dict[str, Any] = {
-            "model": self.options.model,
-            "project_id": self.options.project_id,
-            "location": self.options.location,
-            "adc_credentials_string": self.options.adc_credentials_string,
-        }
-
+        # additional_params spread first so that explicit fields always win,
+        # matching the TypeScript SDK.
+        params: Dict[str, Any] = dict(self.options.additional_params or {})
+        params["model"] = self.options.model
+        params["project_id"] = self.options.project_id
+        params["location"] = self.options.location
+        params["adc_credentials_string"] = self.options.adc_credentials_string
         if self.options.instructions is not None:
             params["instructions"] = self.options.instructions
         if self.options.voice is not None:
             params["voice"] = self.options.voice
-        if self.options.additional_params is not None:
-            params.update(self.options.additional_params)
 
         config: Dict[str, Any] = {
             "vendor": "vertexai",
@@ -119,12 +193,8 @@ def to_config(self) -> Dict[str, Any]:
             config["output_modalities"] = self.options.output_modalities
         if self.options.messages is not None:
             config["messages"] = self.options.messages
-        if self.options.predefined_tools is not None:
-            config["predefined_tools"] = self.options.predefined_tools
         if self.options.failure_message is not None:
             config["failure_message"] = self.options.failure_message
-        if self.options.max_history is not None:
-            config["max_history"] = self.options.max_history
         if self.options.turn_detection is not None:
             config["turn_detection"] = self.options.turn_detection
 
@@ -145,9 +215,7 @@ class GeminiLiveOptions(BaseModel):
     messages: Optional[List[Dict[str, Any]]] = Field(default=None, description="Conversation messages")
     additional_params: Optional[Dict[str, Any]] = Field(default=None, description="Additional parameters")
     turn_detection: Optional[MllmTurnDetectionConfig] = Field(default=None, description="MLLM turn detection configuration")
-    predefined_tools: Optional[List[str]] = Field(default=None, description="Predefined tools")
     failure_message: Optional[str] = Field(default=None, description="Message played on failure")
-    max_history: Optional[int] = Field(default=None, description="Maximum conversation history length")
 
 class GeminiLive(BaseMLLM):
     def __init__(self, **kwargs: Any):
@@ -179,12 +247,8 @@ def to_config(self) -> Dict[str, Any]:
             config["output_modalities"] = self.options.output_modalities
         if self.options.messages is not None:
             config["messages"] = self.options.messages
-        if self.options.predefined_tools is not None:
-            config["predefined_tools"] = self.options.predefined_tools
         if self.options.failure_message is not None:
             config["failure_message"] = self.options.failure_message
-        if self.options.max_history is not None:
-            config["max_history"] = self.options.max_history
         if self.options.turn_detection is not None:
             config["turn_detection"] = self.options.turn_detection
 
