temp save

Author: Jicheng Lu

src/Plugins/BotSharp.Plugin.GoogleAI/Models/Realtime/RealtimeClientPayload.cs

@@ -0,0 +1,18 @@
+using GenerativeAI.Types;
+
+namespace BotSharp.Plugin.GoogleAI.Models.Realtime;
+
+internal class RealtimeClientPayload
+{
+    [JsonPropertyName("setup")]
+    public RealtimeGenerateContentSetup? Setup { get; set; }
+
+    [JsonPropertyName("clientContent")]
+    public BidiGenerateContentClientContent? ClientContent { get; set; }
+
+    [JsonPropertyName("realtimeInput")]
+    public BidiGenerateContentRealtimeInput? RealtimeInput { get; set; }
+
+    [JsonPropertyName("toolResponse")]
+    public BidiGenerateContentToolResponse? ToolResponse { get; set; }
+}

src/Plugins/BotSharp.Plugin.GoogleAI/Models/Realtime/RealtimeGenerateContentSetup.cs

@@ -0,0 +1,26 @@
+using GenerativeAI.Types;
+
+namespace BotSharp.Plugin.GoogleAI.Models.Realtime;
+
+internal class RealtimeGenerateContentSetup
+{
+    [JsonPropertyName("model")]
+    public string? Model { get; set; }
+
+    [JsonPropertyName("generationConfig")]
+    public GenerationConfig? GenerationConfig { get; set; }
+
+    [JsonPropertyName("systemInstruction")]
+    public Content? SystemInstruction { get; set; }
+
+    [JsonPropertyName("tools")]
+    public Tool[]? Tools { get; set; }
+
+    [JsonPropertyName("inputAudioTranscription")]
+    public AudioTranscriptionConfig? InputAudioTranscription { get; set; }
+
+    [JsonPropertyName("outputAudioTranscription")]
+    public AudioTranscriptionConfig? OutputAudioTranscription { get; set; }
+}
+
+internal class AudioTranscriptionConfig { }

src/Plugins/BotSharp.Plugin.GoogleAI/Models/Realtime/RealtimeServerResponse.cs

@@ -30,6 +30,12 @@ internal class RealtimeGenerateContentServerContent
 
     [JsonPropertyName("modelTurn")]
     public Content? ModelTurn { get; set; }
+
+    [JsonPropertyName("inputTranscription")]
+    public RealtimeGenerateContentTranscription? InputTranscription { get; set; }
+
+    [JsonPropertyName("outputTranscription")]
+    public RealtimeGenerateContentTranscription? OutputTranscription { get; set; }
 }
 
 internal class RealtimeUsageMetaData
@@ -58,4 +64,10 @@ internal class RealtimeTokenDetail
 
     [JsonPropertyName("tokenCount")]
     public int? TokenCount { get; set; }
+}
+
+internal class RealtimeGenerateContentTranscription
+{
+    [JsonPropertyName("text")]
+    public string? Text { get; set; }
 }

src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs

@@ -107,8 +107,9 @@ public async Task Connect(
             JsonOptions = _jsonOptions
         });
 
+        var uri = BuildWebsocketUri(modelSettings.ApiKey, "v1beta");
         await _session.ConnectAsync(
-            uri: new Uri($"wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContent?key={modelSettings.ApiKey}"),
+            uri: uri,
             cancellationToken: CancellationToken.None);
 
         await onModelReady();
@@ -148,9 +149,12 @@ private async Task ReceiveMessage(
         Func<string, Task> onModelAudioTranscriptDone,
         Func<List<RoleDialogModel>, Task> onModelResponseDone,
         Func<string, Task> onConversationItemCreated,
-        Func<RoleDialogModel, Task> onInputAudioTranscriptionCompleted,
+        Func<RoleDialogModel, Task> onInputAudioTranscriptionDone,
         Func<Task> onInterruptionDetected)
     {
+        var inputTranscription = string.Empty;
+        var outputTranscription = string.Empty;
+
         await foreach (ChatSessionUpdate update in _session.ReceiveUpdatesAsync(CancellationToken.None))
         {
             var receivedText = update?.RawResponse;
@@ -163,7 +167,6 @@ private async Task ReceiveMessage(
             try
             {
                 var response = JsonSerializer.Deserialize<RealtimeServerResponse>(receivedText, _jsonOptions);
-
                 if (response == null)
                 {
                     continue;
@@ -175,10 +178,29 @@ private async Task ReceiveMessage(
                 }
                 else if (response.ServerContent != null)
                 {
+                    if (response.ServerContent.InputTranscription?.Text != null)
+                    {
+                        outputTranscription = string.Empty;
+                        inputTranscription += response.ServerContent.InputTranscription.Text;
+                    }
+
+                    if (response.ServerContent.OutputTranscription?.Text != null)
+                    {
+                        outputTranscription += response.ServerContent.OutputTranscription.Text;
+                    }
+
                     if (response.ServerContent.ModelTurn != null)
                     {
                         _logger.LogInformation($"Model audio delta received.");
                         var parts = response.ServerContent.ModelTurn.Parts;
+
+                        if (!string.IsNullOrEmpty(inputTranscription))
+                        {
+                            var message = await OnUserAudioTranscriptionCompleted(conn, inputTranscription);
+                            await onInputAudioTranscriptionDone(message);
+                            inputTranscription = string.Empty;
+                        }
+
                         if (!parts.IsNullOrEmpty())
                         {
                             foreach (var part in parts)
@@ -197,13 +219,23 @@ private async Task ReceiveMessage(
                     else if (response.ServerContent.TurnComplete == true)
                     {
                         _logger.LogInformation($"Model turn completed.");
+
+                        if (!string.IsNullOrEmpty(outputTranscription))
+                        {
+                            var messages = await OnResponseDone(conn, outputTranscription, response.UsageMetaData);
+                            await onModelResponseDone(messages);
+
+                            // Reset input/output transcription
+                            inputTranscription = string.Empty;
+                            outputTranscription = string.Empty;
+                        }
                     }
                 }
             }
             catch (Exception ex)
             {
-                _logger.LogError(ex, $"Error when deserializing server response.");
-                continue;
+                _logger.LogError(ex, $"Error when deserializing server response. {ex.Message}");
+                break;
             }
         }
 
@@ -288,7 +320,7 @@ private Task AttachEvents(MultiModalLiveClient client)
         client.Connected += (sender, e) =>
         {
             _logger.LogInformation("Google Realtime Client connected.");
-            _onModelReady();
+            _onModelReady().ConfigureAwait(false).GetAwaiter().GetResult();
         };
 
         client.Disconnected += (sender, e) =>
@@ -301,39 +333,39 @@ private Task AttachEvents(MultiModalLiveClient client)
             _logger.LogInformation("User message received.");
             if (e.Payload.SetupComplete != null)
             {
-                _onConversationItemCreated(_client.ConnectionId.ToString());
+                _onConversationItemCreated(_client.ConnectionId.ToString()).ConfigureAwait(false).GetAwaiter().GetResult();
             }
 
             if (e.Payload.ServerContent != null)
             {
                 if (e.Payload.ServerContent.TurnComplete == true)
                 {
                     var responseDone = await ResponseDone(_conn, e.Payload.ServerContent);
-                    _onModelResponseDone(responseDone);
+                    _onModelResponseDone(responseDone).ConfigureAwait(false).GetAwaiter().GetResult();
                 }
             }
         };
 
         client.AudioChunkReceived += (sender, e) =>
         {
-            _onModelAudioDeltaReceived(Convert.ToBase64String(e.Buffer), Guid.NewGuid().ToString());
+            _onModelAudioDeltaReceived(Convert.ToBase64String(e.Buffer), Guid.NewGuid().ToString()).ConfigureAwait(false).GetAwaiter().GetResult();
         };
 
         client.TextChunkReceived += (sender, e) =>
         {
-            _onInputAudioTranscriptionDone(new RoleDialogModel(AgentRole.Assistant, e.Text));
+            _onInputAudioTranscriptionDone(new RoleDialogModel(AgentRole.Assistant, e.Text)).ConfigureAwait(false).GetAwaiter().GetResult();
         };
 
         client.GenerationInterrupted += (sender, e) => 
         {
             _logger.LogInformation("Audio generation interrupted.");
-            _onUserInterrupted(); 
+            _onUserInterrupted().ConfigureAwait(false).GetAwaiter().GetResult(); 
         };
 
         client.AudioReceiveCompleted += (sender, e) => 
         {
             _logger.LogInformation("Audio receive completed.");
-            _onModelAudioResponseDone(); 
+            _onModelAudioResponseDone().ConfigureAwait(false).GetAwaiter().GetResult(); 
         };
 
         client.ErrorOccurred += (sender, e) =>
@@ -345,6 +377,43 @@ private Task AttachEvents(MultiModalLiveClient client)
         return Task.CompletedTask;
     }
 
+    private async Task<List<RoleDialogModel>> OnResponseDone(RealtimeHubConnection conn, string text, RealtimeUsageMetaData? useage)
+    {
+        var outputs = new List<RoleDialogModel>
+        {
+            new(AgentRole.Assistant, text)
+            {
+                CurrentAgentId = conn.CurrentAgentId,
+                MessageId = Guid.NewGuid().ToString(),
+                MessageType = MessageTypeName.Plain
+            }
+        };
+
+        if (useage != null)
+        {
+            var contentHooks = _services.GetServices<IContentGeneratingHook>();
+            foreach (var hook in contentHooks)
+            {
+                await hook.AfterGenerated(new RoleDialogModel(AgentRole.Assistant, text)
+                {
+                    CurrentAgentId = conn.CurrentAgentId
+                },
+                new TokenStatsModel
+                {
+                    Provider = Provider,
+                    Model = _model,
+                    Prompt = text,
+                    TextInputTokens = useage.PromptTokensDetails?.FirstOrDefault(x => x.Modality == Modality.TEXT.ToString())?.TokenCount ?? 0,
+                    AudioInputTokens = useage.PromptTokensDetails?.FirstOrDefault(x => x.Modality == Modality.AUDIO.ToString())?.TokenCount ?? 0,
+                    TextOutputTokens = useage.ResponseTokensDetails?.FirstOrDefault(x => x.Modality == Modality.TEXT.ToString())?.TokenCount ?? 0,
+                    AudioOutputTokens = useage.ResponseTokensDetails?.FirstOrDefault(x => x.Modality == Modality.AUDIO.ToString())?.TokenCount ?? 0
+                });
+            }
+        }
+
+        return outputs;
+    }
+
     private async Task<List<RoleDialogModel>> ResponseDone(RealtimeHubConnection conn,
         BidiGenerateContentServerContent serverContent)
     {
@@ -401,8 +470,6 @@ await hook.AfterGenerated(new RoleDialogModel(AgentRole.Assistant, "response.don
 
     public async Task SendEventToModel(object message)
     {
-        //todo Send Audio Chunks to Model, Botsharp RealTime Implementation seems to be incomplete
-
         if (_session == null) return;
 
         await _session.SendEventToModel(message);
@@ -419,9 +486,9 @@ public async Task<string> UpdateSession(RealtimeHubConnection conn, bool isInit
         var (prompt, request) = PrepareOptions(agent, []);
 
         var config = request.GenerationConfig;
-        //Output Modality can either be text or audio
         if (config != null)
         {
+            //Output Modality can either be text or audio
             config.ResponseModalities = [Modality.AUDIO];
 
             var words = new List<string>();
@@ -467,14 +534,16 @@ await HookEmitter.Emit<IContentGeneratingHook>(_services,
         //    //Tools = request.Tools?.ToArray(),
         //});
 
-        await SendEventToModel(new BidiClientPayload
+        await SendEventToModel(new RealtimeClientPayload
         {
-            Setup = new BidiGenerateContentSetup()
+            Setup = new RealtimeGenerateContentSetup()
             {
                 GenerationConfig = config,
                 Model = Model.ToModelId(),
                 SystemInstruction = request.SystemInstruction,
-                Tools = []
+                Tools = [],
+                InputAudioTranscription = new(),
+                OutputAudioTranscription = new()
             }
         });
 
@@ -532,7 +601,7 @@ await SendEventToModel(new BidiClientPayload
         }
         else
         {
-            throw new NotImplementedException("");
+            throw new NotImplementedException($"Unrecognized role {message.Role}.");
         }
     }
 
@@ -542,9 +611,9 @@ public async Task<List<RoleDialogModel>> OnResponsedDone(RealtimeHubConnection c
     }
 
 
-    public async Task<RoleDialogModel> OnConversationItemCreated(RealtimeHubConnection conn, string response)
+    public async Task<RoleDialogModel> OnConversationItemCreated(RealtimeHubConnection conn, string text)
     {
-        return await Task.FromResult(new RoleDialogModel(AgentRole.User, response));
+        return await Task.FromResult(new RoleDialogModel(AgentRole.User, text));
     }
 
     private (string, GenerateContentRequest) PrepareOptions(Agent agent,
@@ -688,4 +757,18 @@ private string GetPrompt(IEnumerable<string> systemPrompts, IEnumerable<string>
 
         return prompt;
     }
+
+
+    private async Task<RoleDialogModel> OnUserAudioTranscriptionCompleted(RealtimeHubConnection conn, string text)
+    {
+        return new RoleDialogModel(AgentRole.User, text)
+        {
+            CurrentAgentId = conn.CurrentAgentId
+        };
+    }
+
+    private Uri BuildWebsocketUri(string apiKey, string version = "v1alpha")
+    {
+        return new Uri($"wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.{version}.GenerativeService.BidiGenerateContent?key={apiKey}");
+    }
 }

src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs

@@ -402,7 +402,7 @@ public async Task InsertConversationItem(RoleDialogModel message)
         }
         else
         {
-            throw new NotImplementedException("");
+            throw new NotImplementedException($"Unrecognized role {message.Role}.");
         }
     }