tool call done; to do session restart

iceljc · iceljc · commit bc40deed5405 · 2025-05-14T23:49:36.000-05:00
diff --git a/src/Infrastructure/BotSharp.Core.Realtime/Hooks/RealtimeConversationHook.cs b/src/Infrastructure/BotSharp.Core.Realtime/Hooks/RealtimeConversationHook.cs
@@ -42,7 +42,6 @@ public async Task OnFunctionExecuted(RoleDialogModel message)
         var routing = _services.GetRequiredService<IRoutingService>();
 
         message.Role = AgentRole.Function;
-        //message.Role = AgentRole.Assistant;
 
         if (message.FunctionName == "route_to_agent")
         {
@@ -66,21 +65,24 @@ public async Task OnFunctionExecuted(RoleDialogModel message)
         else
         {
             // Update session for changed states
-            var instruction = await hub.Completer.UpdateSession(hub.HubConn);
+
+            // TO DO
+            //var instruction = await hub.Completer.UpdateSession(hub.HubConn);
             await hub.Completer.InsertConversationItem(message);
 
             if (string.IsNullOrEmpty(message.Content))
             {
                 return;
             }
-            else if (message.StopCompletion)
-            {
-                await hub.Completer.TriggerModelInference($"Say to user: \"{message.Content}\"");
-            }
-            else
-            {
-                await hub.Completer.TriggerModelInference(instruction);
-            }
+
+            //if (message.StopCompletion)
+            //{
+            //    await hub.Completer.TriggerModelInference($"Say to user: \"{message.Content}\"");
+            //}
+            //else
+            //{
+            //    await hub.Completer.TriggerModelInference();
+            //}
         }
     }
 }
diff --git a/src/Infrastructure/BotSharp.Core/Conversations/ConversationPlugin.cs b/src/Infrastructure/BotSharp.Core/Conversations/ConversationPlugin.cs
@@ -41,6 +41,7 @@ public void RegisterDI(IServiceCollection services, IConfiguration config)
             return settingService.Bind<GoogleApiSettings>("GoogleApi");
         });
 
+        services.AddScoped<ConversationHookProvider>();
         services.AddScoped<IConversationStorage, ConversationStorage>();
         services.AddScoped<IConversationService, ConversationService>();
         services.AddScoped<IConversationProgressService, ConversationProgressService>();
diff --git a/src/Infrastructure/BotSharp.Core/Session/LlmRealtimeSession.cs b/src/Infrastructure/BotSharp.Core/Session/LlmRealtimeSession.cs
@@ -106,6 +106,7 @@ public async Task DisconnectAsync()
 
     public void Dispose()
     {
+        _clientEventSemaphore?.Dispose();
         _webSocket?.Dispose();
     }
 }
diff --git a/src/Infrastructure/BotSharp.Core/Statistics/Services/BotSharpStatsService.cs b/src/Infrastructure/BotSharp.Core/Statistics/Services/BotSharpStatsService.cs
@@ -1,4 +1,3 @@
-using BotSharp.Abstraction.Infrastructures;
 using BotSharp.Abstraction.Statistics.Settings;
 
 namespace BotSharp.Core.Statistics.Services;
diff --git a/src/Plugins/BotSharp.Plugin.ChatHub/ChatHubPlugin.cs b/src/Plugins/BotSharp.Plugin.ChatHub/ChatHubPlugin.cs
@@ -24,7 +24,6 @@ public void RegisterDI(IServiceCollection services, IConfiguration config)
         services.AddScoped<IConversationHook, ChatHubConversationHook>();
         services.AddScoped<IConversationHook, StreamingLogHook>();
         services.AddScoped<IConversationHook, WelcomeHook>();
-        services.AddScoped<ConversationHookProvider>();
         services.AddScoped<IRoutingHook, StreamingLogHook>();
         services.AddScoped<IContentGeneratingHook, StreamingLogHook>();
         services.AddScoped<ICrontabHook, ChatHubCrontabHook>();
diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/Models/Realtime/RealtimeGenerateContentSetup.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/Models/Realtime/RealtimeGenerateContentSetup.cs
@@ -21,6 +21,15 @@ internal class RealtimeGenerateContentSetup
 
     [JsonPropertyName("outputAudioTranscription")]
     public AudioTranscriptionConfig? OutputAudioTranscription { get; set; }
+
+    [JsonPropertyName("sessionResumption")]
+    public SessionResumptionConfig? SessionResumption { get; set; }
 }
 
-internal class AudioTranscriptionConfig { }
+internal class AudioTranscriptionConfig { }
+
+internal class SessionResumptionConfig
+{
+    [JsonPropertyName("handle")]
+    public string? Handle { get; set; }
+}
diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/Models/Realtime/RealtimeServerResponse.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/Models/Realtime/RealtimeServerResponse.cs
@@ -15,6 +15,9 @@ internal class RealtimeServerResponse
 
     [JsonPropertyName("toolCall")]
     public RealtimeToolCall? ToolCall { get; set; }
+
+    [JsonPropertyName("sessionResumptionUpdate")]
+    public RealtimeSessionResumptionUpdate? SessionResumptionUpdate { get; set; }
 }
 
 
@@ -91,4 +94,13 @@ internal class RealtimeFunctionCall
 
     [JsonPropertyName("args")]
     public JsonNode? Args { get; set; }
+}
+
+internal class RealtimeSessionResumptionUpdate
+{
+    [JsonPropertyName("newHandle")]
+    public string? NewHandle { get; set; }
+
+    [JsonPropertyName("resumable")]
+    public bool? Resumable { get; set; }
 }
diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/Models/Realtime/RealtimeTranscriptionResponse.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/Models/Realtime/RealtimeTranscriptionResponse.cs
@@ -28,7 +28,7 @@ public void Collect(string text)
         _contentStream.Position = 0;
     }
 
-    public string GetString()
+    public string GetText()
     {
         if (_contentStream.Length == 0)
         {
diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs
@@ -113,7 +113,6 @@ private async Task ReceiveMessage(
                 continue;
             }
 
-            Console.WriteLine($"Received text: {receivedText}");
             try
             {
                 var response = JsonSerializer.Deserialize<RealtimeServerResponse>(receivedText, _jsonOptions);
@@ -126,10 +125,15 @@ private async Task ReceiveMessage(
                 {
                     _logger.LogInformation($"Session setup completed.");
                 }
+                else if (response.SessionResumptionUpdate != null)
+                {
+                    _logger.LogInformation($"Session resumption update => New handle: {response.SessionResumptionUpdate.NewHandle}, Resumable: {response.SessionResumptionUpdate.Resumable}");
+                }
                 else if (response.ToolCall != null && !response.ToolCall.FunctionCalls.IsNullOrEmpty())
                 {
-                    var functionCall = response.ToolCall.FunctionCalls.First();
-                    _logger.LogInformation($"Tool call received {functionCall.Name}({functionCall.Args?.ToJsonString(_jsonOptions) ?? string.Empty}).");
+                    var functionCall = response.ToolCall.FunctionCalls!.First();
+
+                    _logger.LogInformation($"Tool call received: {functionCall.Name}({functionCall.Args?.ToJsonString(_jsonOptions) ?? string.Empty}).");
 
                     if (functionCall != null)
                     {
@@ -154,7 +158,7 @@ private async Task ReceiveMessage(
                         _logger.LogInformation($"Model audio delta received.");
 
                         // Handle input transcription
-                        var inputTranscription = inputStream.GetString();
+                        var inputTranscription = inputStream.GetText();
                         if (!string.IsNullOrEmpty(inputTranscription))
                         {
                             var message = OnUserAudioTranscriptionCompleted(conn, inputTranscription);
@@ -182,7 +186,8 @@ private async Task ReceiveMessage(
                     {
                         _logger.LogInformation($"Model turn completed.");
 
-                        var outputTranscription = outputStream.GetString();
+                        // Handle output transcription
+                        var outputTranscription = outputStream.GetText();
                         if (!string.IsNullOrEmpty(outputTranscription))
                         {
                             var messages = await OnResponseDone(conn, outputTranscription, response.UsageMetaData);
@@ -237,13 +242,13 @@ await SendEventToModel(new BidiClientPayload
 
     public async Task TriggerModelInference(string? instructions = null)
     {
-        var content = new Content("Please respond to me.", AgentRole.User);
+        var content = new Content(instructions ?? "Please respond to user.", AgentRole.User);
 
         await SendEventToModel(new BidiClientPayload
         {
             ClientContent = new()
             {
-                Turns = null,
+                Turns = [content],
                 TurnComplete = true
             }
         });
@@ -269,9 +274,10 @@ public async Task SendEventToModel(object message)
     public async Task<string> UpdateSession(RealtimeHubConnection conn, bool isInit = false)
     {
         var convService = _services.GetRequiredService<IConversationService>();
-        var conv = await convService.GetConversation(conn.ConversationId);
-
         var agentService = _services.GetRequiredService<IAgentService>();
+        var realtimeSetting = _services.GetRequiredService<RealtimeModelSettings>();
+
+        var conv = await convService.GetConversation(conn.ConversationId);
         var agent = await agentService.LoadAgent(conn.CurrentAgentId);
 
         var (prompt, request) = PrepareOptions(agent, []);
@@ -285,10 +291,8 @@ public async Task<string> UpdateSession(RealtimeHubConnection conn, bool isInit
             var words = new List<string>();
             HookEmitter.Emit<IRealtimeHook>(_services, hook => words.AddRange(hook.OnModelTranscriptPrompt(agent)));
 
-            var realtimeModelSettings = _services.GetRequiredService<RealtimeModelSettings>();
-
-            config.Temperature = Math.Max(realtimeModelSettings.Temperature, 0.6f);
-            config.MaxOutputTokens = realtimeModelSettings.MaxResponseOutputTokens;
+            config.Temperature = Math.Max(realtimeSetting.Temperature, 0.6f);
+            config.MaxOutputTokens = realtimeSetting.MaxResponseOutputTokens;
         }
         
         var functions = request.Tools?.SelectMany(s => s.FunctionDeclarations).Select(x =>
@@ -316,7 +320,6 @@ await HookEmitter.Emit<IContentGeneratingHook>(_services,
             });
         }
 
-        var realtimeSetting = _services.GetRequiredService<RealtimeModelSettings>();
         await SendEventToModel(new RealtimeClientPayload
         {
             Setup = new RealtimeGenerateContentSetup()
@@ -326,7 +329,8 @@ await SendEventToModel(new RealtimeClientPayload
                 SystemInstruction = request.SystemInstruction,
                 Tools = request.Tools?.ToArray(),
                 InputAudioTranscription = realtimeSetting.InputAudioTranscribe ? new() : null,
-                OutputAudioTranscription = realtimeSetting.InputAudioTranscribe ? new() : null
+                OutputAudioTranscription = realtimeSetting.InputAudioTranscribe ? new() : null,
+                SessionResumption = new()
             }
         });
 
@@ -339,6 +343,7 @@ public async Task InsertConversationItem(RoleDialogModel message)
         {
             var function = new FunctionResponse()
             {
+                Id = message.ToolCallId,
                 Name = message.FunctionName ?? string.Empty,
                 Response = new JsonObject()
                 {
diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs
@@ -65,7 +65,7 @@ await _session.ConnectAsync(
             cancellationToken: CancellationToken.None);
 
         _ = ReceiveMessage(
-                _services,
+                realtimeSettings,
                 conn,
                 onModelReady,
                 onModelAudioDeltaReceived,
@@ -144,7 +144,7 @@ await SendEventToModel(new
     }
 
     private async Task ReceiveMessage(
-        IServiceProvider services,
+        RealtimeModelSettings realtimeSettings,
         RealtimeHubConnection conn,
         Func<Task> onModelReady,
         Func<string, string, Task> onModelAudioDeltaReceived,
@@ -156,7 +156,6 @@ private async Task ReceiveMessage(
         Func<Task> onInterruptionDetected)
     {
         DateTime? startTime = null;
-        var realtimeSettings = _services.GetRequiredService<RealtimeModelSettings>();
 
         await foreach (ChatSessionUpdate update in _session.ReceiveUpdatesAsync(CancellationToken.None))
         {
diff --git a/tests/BotSharp.Test.RealtimeVoice/appsettings.json b/tests/BotSharp.Test.RealtimeVoice/appsettings.json
@@ -16,7 +16,6 @@
           "Version": "2024-12-17",
           "ApiKey": "",
           "Type": "realtime",
-          "RealTime": true,
           "Cost": {
             "TextInputCost": 0.0006,
             "CachedTextInputCost": 0.0003,
@@ -37,7 +36,6 @@
           "Version": "20240620",
           "ApiKey": "",
           "Type": "realtime",
-          "RealTime": true,
           "Cost": {
             "TextInputCost": 0.0006,
             "CachedTextInputCost": 0.0003,

Original file line number	Diff line number	Diff line change
`@@ -42,7 +42,6 @@ public async Task OnFunctionExecuted(RoleDialogModel message)`
`42`	`42`	`var routing = _services.GetRequiredService<IRoutingService>();`
`43`	`43`
`44`	`44`	`message.Role = AgentRole.Function;`
`45`		`- //message.Role = AgentRole.Assistant;`
`46`	`45`
`47`	`46`	`if (message.FunctionName == "route_to_agent")`
`48`	`47`	`{`
`@@ -66,21 +65,24 @@ public async Task OnFunctionExecuted(RoleDialogModel message)`
`66`	`65`	`else`
`67`	`66`	`{`
`68`	`67`	`// Update session for changed states`
`69`		`- var instruction = await hub.Completer.UpdateSession(hub.HubConn);`
	`68`	`+`
	`69`	`+ // TO DO`
	`70`	`+ //var instruction = await hub.Completer.UpdateSession(hub.HubConn);`
`70`	`71`	`await hub.Completer.InsertConversationItem(message);`
`71`	`72`
`72`	`73`	`if (string.IsNullOrEmpty(message.Content))`
`73`	`74`	`{`
`74`	`75`	`return;`
`75`	`76`	`}`
`76`		`- else if (message.StopCompletion)`
`77`		`- {`
`78`		`- await hub.Completer.TriggerModelInference($"Say to user: \"{message.Content}\"");`
`79`		`- }`
`80`		`- else`
`81`		`- {`
`82`		`- await hub.Completer.TriggerModelInference(instruction);`
`83`		`- }`
	`77`	`+`
	`78`	`+ //if (message.StopCompletion)`
	`79`	`+ //{`
	`80`	`+ // await hub.Completer.TriggerModelInference($"Say to user: \"{message.Content}\"");`
	`81`	`+ //}`
	`82`	`+ //else`
	`83`	`+ //{`
	`84`	`+ // await hub.Completer.TriggerModelInference();`
	`85`	`+ //}`
`84`	`86`	`}`
`85`	`87`	`}`
`86`	`88`	`}`
Original file line number	Diff line number	Diff line change
`@@ -106,6 +106,7 @@ public async Task DisconnectAsync()`
`106`	`106`
`107`	`107`	`public void Dispose()`
`108`	`108`	`{`
	`109`	`+ _clientEventSemaphore?.Dispose();`
`109`	`110`	`_webSocket?.Dispose();`
`110`	`111`	`}`
`111`	`112`	`}`
Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,3 @@`
`1`		`-using BotSharp.Abstraction.Infrastructures;`
`2`	`1`	`using BotSharp.Abstraction.Statistics.Settings;`
`3`	`2`
`4`	`3`	`namespace BotSharp.Core.Statistics.Services;`
Original file line number	Diff line number	Diff line change
`@@ -28,7 +28,7 @@ public void Collect(string text)`
`28`	`28`	`_contentStream.Position = 0;`
`29`	`29`	`}`
`30`	`30`
`31`		`- public string GetString()`
	`31`	`+ public string GetText()`
`32`	`32`	`{`
`33`	`33`	`if (_contentStream.Length == 0)`
`34`	`34`	`{`
Original file line number	Diff line number	Diff line change
`@@ -113,7 +113,6 @@ private async Task ReceiveMessage(`
`113`	`113`	`continue;`
`114`	`114`	`}`
`115`	`115`
`116`		`- Console.WriteLine($"Received text: {receivedText}");`
`117`	`116`	`try`
`118`	`117`	`{`
`119`	`118`	`var response = JsonSerializer.Deserialize<RealtimeServerResponse>(receivedText, _jsonOptions);`
`@@ -126,10 +125,15 @@ private async Task ReceiveMessage(`
`126`	`125`	`{`
`127`	`126`	`_logger.LogInformation($"Session setup completed.");`
`128`	`127`	`}`
	`128`	`+ else if (response.SessionResumptionUpdate != null)`
	`129`	`+ {`
	`130`	`+ _logger.LogInformation($"Session resumption update => New handle: {response.SessionResumptionUpdate.NewHandle}, Resumable: {response.SessionResumptionUpdate.Resumable}");`
	`131`	`+ }`
`129`	`132`	`else if (response.ToolCall != null && !response.ToolCall.FunctionCalls.IsNullOrEmpty())`
`130`	`133`	`{`
`131`		`- var functionCall = response.ToolCall.FunctionCalls.First();`
`132`		`- _logger.LogInformation($"Tool call received {functionCall.Name}({functionCall.Args?.ToJsonString(_jsonOptions) ?? string.Empty}).");`
	`134`	`+ var functionCall = response.ToolCall.FunctionCalls!.First();`
	`135`	`+`
	`136`	`+ _logger.LogInformation($"Tool call received: {functionCall.Name}({functionCall.Args?.ToJsonString(_jsonOptions) ?? string.Empty}).");`
`133`	`137`
`134`	`138`	`if (functionCall != null)`
`135`	`139`	`{`
`@@ -154,7 +158,7 @@ private async Task ReceiveMessage(`
`154`	`158`	`_logger.LogInformation($"Model audio delta received.");`
`155`	`159`
`156`	`160`	`// Handle input transcription`
`157`		`- var inputTranscription = inputStream.GetString();`
	`161`	`+ var inputTranscription = inputStream.GetText();`
`158`	`162`	`if (!string.IsNullOrEmpty(inputTranscription))`
`159`	`163`	`{`
`160`	`164`	`var message = OnUserAudioTranscriptionCompleted(conn, inputTranscription);`
`@@ -182,7 +186,8 @@ private async Task ReceiveMessage(`
`182`	`186`	`{`
`183`	`187`	`_logger.LogInformation($"Model turn completed.");`
`184`	`188`
`185`		`- var outputTranscription = outputStream.GetString();`
	`189`	`+ // Handle output transcription`
	`190`	`+ var outputTranscription = outputStream.GetText();`
`186`	`191`	`if (!string.IsNullOrEmpty(outputTranscription))`
`187`	`192`	`{`
`188`	`193`	`var messages = await OnResponseDone(conn, outputTranscription, response.UsageMetaData);`
`@@ -237,13 +242,13 @@ await SendEventToModel(new BidiClientPayload`
`237`	`242`
`238`	`243`	`public async Task TriggerModelInference(string? instructions = null)`
`239`	`244`	`{`
`240`		`- var content = new Content("Please respond to me.", AgentRole.User);`
	`245`	`+ var content = new Content(instructions ?? "Please respond to user.", AgentRole.User);`
`241`	`246`
`242`	`247`	`await SendEventToModel(new BidiClientPayload`
`243`	`248`	`{`
`244`	`249`	`ClientContent = new()`
`245`	`250`	`{`
`246`		`- Turns = null,`
	`251`	`+ Turns = [content],`
`247`	`252`	`TurnComplete = true`
`248`	`253`	`}`
`249`	`254`	`});`
`@@ -269,9 +274,10 @@ public async Task SendEventToModel(object message)`
`269`	`274`	`public async Task<string> UpdateSession(RealtimeHubConnection conn, bool isInit = false)`
`270`	`275`	`{`
`271`	`276`	`var convService = _services.GetRequiredService<IConversationService>();`
`272`		`- var conv = await convService.GetConversation(conn.ConversationId);`
`273`		`-`
`274`	`277`	`var agentService = _services.GetRequiredService<IAgentService>();`
	`278`	`+ var realtimeSetting = _services.GetRequiredService<RealtimeModelSettings>();`
	`279`	`+`
	`280`	`+ var conv = await convService.GetConversation(conn.ConversationId);`
`275`	`281`	`var agent = await agentService.LoadAgent(conn.CurrentAgentId);`
`276`	`282`
`277`	`283`	`var (prompt, request) = PrepareOptions(agent, []);`
`@@ -285,10 +291,8 @@ public async Task<string> UpdateSession(RealtimeHubConnection conn, bool isInit`
`285`	`291`	`var words = new List<string>();`
`286`	`292`	`HookEmitter.Emit<IRealtimeHook>(_services, hook => words.AddRange(hook.OnModelTranscriptPrompt(agent)));`
`287`	`293`
`288`		`- var realtimeModelSettings = _services.GetRequiredService<RealtimeModelSettings>();`
`289`		`-`
`290`		`- config.Temperature = Math.Max(realtimeModelSettings.Temperature, 0.6f);`
`291`		`- config.MaxOutputTokens = realtimeModelSettings.MaxResponseOutputTokens;`
	`294`	`+ config.Temperature = Math.Max(realtimeSetting.Temperature, 0.6f);`
	`295`	`+ config.MaxOutputTokens = realtimeSetting.MaxResponseOutputTokens;`
`292`	`296`	`}`
`293`	`297`
`294`	`298`	`var functions = request.Tools?.SelectMany(s => s.FunctionDeclarations).Select(x =>`
`@@ -316,7 +320,6 @@ await HookEmitter.Emit<IContentGeneratingHook>(_services,`
`316`	`320`	`});`
`317`	`321`	`}`
`318`	`322`
`319`		`- var realtimeSetting = _services.GetRequiredService<RealtimeModelSettings>();`
`320`	`323`	`await SendEventToModel(new RealtimeClientPayload`
`321`	`324`	`{`
`322`	`325`	`Setup = new RealtimeGenerateContentSetup()`
`@@ -326,7 +329,8 @@ await SendEventToModel(new RealtimeClientPayload`
`326`	`329`	`SystemInstruction = request.SystemInstruction,`
`327`	`330`	`Tools = request.Tools?.ToArray(),`
`328`	`331`	`InputAudioTranscription = realtimeSetting.InputAudioTranscribe ? new() : null,`
`329`		`- OutputAudioTranscription = realtimeSetting.InputAudioTranscribe ? new() : null`
	`332`	`+ OutputAudioTranscription = realtimeSetting.InputAudioTranscribe ? new() : null,`
	`333`	`+ SessionResumption = new()`
`330`	`334`	`}`
`331`	`335`	`});`
`332`	`336`
`@@ -339,6 +343,7 @@ public async Task InsertConversationItem(RoleDialogModel message)`
`339`	`343`	`{`
`340`	`344`	`var function = new FunctionResponse()`
`341`	`345`	`{`
	`346`	`+ Id = message.ToolCallId,`
`342`	`347`	`Name = message.FunctionName ?? string.Empty,`
`343`	`348`	`Response = new JsonObject()`
`344`	`349`	`{`