refine transcription

Author: Jicheng Lu

src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs

@@ -1,6 +1,4 @@
 using BotSharp.Abstraction.Realtime.Models;
-using System;
-using static System.Runtime.InteropServices.JavaScript.JSType;
 
 namespace BotSharp.Abstraction.MLTasks;
 
@@ -20,6 +18,7 @@ Task Connect(
         Func<string, Task> onConversationItemCreated,
         Func<RoleDialogModel, Task> onInputAudioTranscriptionDone,
         Func<Task> onInterruptionDetected);
+
     Task AppenAudioBuffer(string message);
     Task AppenAudioBuffer(ArraySegment<byte> data, int length);
 
@@ -31,6 +30,4 @@ Task Connect(
     Task RemoveConversationItem(string itemId);
     Task TriggerModelInference(string? instructions = null);
     Task CancelModelResponse();
-    Task<List<RoleDialogModel>> OnResponsedDone(RealtimeHubConnection conn, string response);
-    Task<RoleDialogModel> OnConversationItemCreated(RealtimeHubConnection conn, string response);
 }

src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeModelSettings.cs

@@ -12,7 +12,12 @@ public class RealtimeModelSettings
     public string Voice { get; set; } = "alloy";
     public float Temperature { get; set; } = 0.8f;
     public int MaxResponseOutputTokens { get; set; } = 512;
-    public int ModelResponseTimeout { get; set; } = 30;
+    public int ModelResponseTimeoutSeconds { get; set; } = 30;
+
+    /// <summary>
+    /// Whether the target event arrives after ModelResponseTimeoutSeconds, e.g., "response.done"
+    /// </summary>
+    public string? ModelResponseTimeoutEndEvent { get; set; }
     public AudioTranscription InputAudioTranscription { get; set; } = new();
     public ModelTurnDetection TurnDetection { get; set; } = new();
 }

src/Infrastructure/BotSharp.Core.Realtime/Hooks/RealtimeConversationHook.cs

@@ -42,6 +42,7 @@ public async Task OnFunctionExecuted(RoleDialogModel message)
         var routing = _services.GetRequiredService<IRoutingService>();
 
         message.Role = AgentRole.Function;
+        //message.Role = AgentRole.Assistant;
 
         if (message.FunctionName == "route_to_agent")
         {

src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj

@@ -211,9 +211,6 @@
     <Content Include="data\agents\01e2fc5c-2c89-4ec7-8470-7688608b496c\functions\get_weather.json">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </Content>
-    <Content Include="data\agents\01e2fc5c-2c89-4ec7-8470-7688608b496c\functions\get_location.json">
-      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
-    </Content>
   </ItemGroup>
 
   <ItemGroup>

src/Infrastructure/BotSharp.Core/Functions/GetWeatherFn.cs

@@ -20,14 +20,14 @@ public GetWeatherFn(IServiceProvider services)
 
     public async Task<bool> Execute(RoleDialogModel message)
     {
-        var args = JsonSerializer.Deserialize<Location>(message.FunctionArgs, BotSharpOptions.defaultJsonOptions);
+        //var args = JsonSerializer.Deserialize<Location>(message.FunctionArgs, BotSharpOptions.defaultJsonOptions);
 
-        var sidecar = _services.GetService<IConversationSideCar>();
-        var states = GetSideCarStates();
+        //var sidecar = _services.GetService<IConversationSideCar>();
+        //var states = GetSideCarStates();
 
-        var userMessage = $"Please find the information at location {args.City}, {args.State}";
-        var response = await sidecar.SendMessage(BuiltInAgentId.Chatbot, userMessage, states: states);
-        message.Content = $"It is a sunny day {response.Content}.";
+        //var userMessage = $"Please find the information at location {args.City}, {args.State}";
+        //var response = await sidecar.SendMessage(BuiltInAgentId.Chatbot, userMessage, states: states);
+        message.Content = $"It is a sunny day.";
         return true;
     }
 

src/Infrastructure/BotSharp.Core/Infrastructures/Websocket/AsyncWebsocketDataResultEnumerator.cs

@@ -1,4 +1,3 @@
-using BotSharp.Abstraction.Realtime.Models.Session;
 using System.Buffers;
 using System.ClientModel;
 using System.Net.WebSockets;

src/Infrastructure/BotSharp.Core/Session/BotSharpRealtimeSession.cs

@@ -55,7 +55,7 @@ private ChatSessionUpdate HandleSessionResult(ClientResult result)
         };
     }
 
-    public async Task SendEvent(string message)
+    public async Task SendEventAsync(string message)
     {
         if (_websocket.State == WebSocketState.Open)
         {
@@ -64,7 +64,7 @@ public async Task SendEvent(string message)
         }
     }
 
-    public async Task Disconnect()
+    public async Task DisconnectAsync()
     {
         if (_websocket.State == WebSocketState.Open)
         {

src/Infrastructure/BotSharp.Core/Session/LlmRealtimeSession.cs

@@ -71,7 +71,7 @@ private ChatSessionUpdate HandleSessionResult(ClientResult result)
         };
     }
 
-    public async Task SendEventToModel(object message)
+    public async Task SendEventToModelAsync(object message)
     {
         if (_webSocket.State != WebSocketState.Open)
         {
@@ -96,7 +96,7 @@ public async Task SendEventToModel(object message)
         }
     }
 
-    public async Task Disconnect()
+    public async Task DisconnectAsync()
     {
         if (_webSocket.State == WebSocketState.Open)
         {

src/Infrastructure/BotSharp.Core/data/agents/01e2fc5c-2c89-4ec7-8470-7688608b496c/functions/get_location.json

@@ -1,19 +1,14 @@
 {
   "name": "get_weather",
   "description": "Get weather information for user.",
-  "visibility_expression": "{% if states.channel != 'email' %}visible{% endif %}",
   "parameters": {
     "type": "object",
     "properties": {
       "city": {
         "type": "string",
         "description": "The city where the user wants to get weather information."
-      },
-      "state": {
-        "type": "string",
-        "description": "The state where the user wants to get weather information."
       }
     },
-    "required": [ "city", "state" ]
+    "required": [ "city" ]
   }
 }