Test/realtime chat

Directory.Packages.props

@@ -21,6 +21,7 @@
     <PackageVersion Include="Newtonsoft.Json" Version="13.0.3" />
     <PackageVersion Include="SharpHook" Version="5.3.9" />
     <PackageVersion Include="SixLabors.ImageSharp" Version="3.1.7" />
+    <PackageVersion Include="System.ClientModel" Version="1.3.0" />
     <PackageVersion Include="System.ComponentModel.Annotations" Version="5.0.0" />
     <PackageVersion Include="System.IdentityModel.Tokens.Jwt" Version="8.0.0" />
     <PackageVersion Include="System.Memory.Data" Version="8.0.0" />

src/Infrastructure/BotSharp.Abstraction/Agents/Enums/AgentField.cs

@@ -16,6 +16,7 @@ public enum AgentField
     Instruction,
     Function,
     Template,
+    Link,
     Response,
     Sample,
     LlmConfig,

src/Infrastructure/BotSharp.Abstraction/Agents/Models/Agent.cs

@@ -46,6 +46,12 @@ public class Agent
     [JsonIgnore]
     public List<AgentTemplate> Templates { get; set; } = new();
 
+    /// <summary>
+    /// Links that can be filled into parent prompt
+    /// </summary>
+    [JsonIgnore]
+    public List<AgentLink> Links { get; set; } = new();
+
     /// <summary>
     /// Agent tasks
     /// </summary>
@@ -168,6 +174,8 @@ public static Agent Clone(Agent agent)
             Functions = agent.Functions,
             Responses = agent.Responses,
             Samples = agent.Samples,
+            Templates = agent.Templates,
+            Links = agent.Links,
             Utilities = agent.Utilities,
             McpTools = agent.McpTools,
             Knowledges = agent.Knowledges,
@@ -204,6 +212,12 @@ public Agent SetTemplates(List<AgentTemplate> templates)
         return this;
     }
 
+    public Agent SetLinks(List<AgentLink> links)
+    {
+        Links = links ?? [];
+        return this;
+    }
+
     public Agent SetTasks(List<AgentTask> tasks)
     {
         Tasks = tasks ?? [];

src/Infrastructure/BotSharp.Abstraction/Agents/Models/AgentLink.cs

@@ -0,0 +1,17 @@
+namespace BotSharp.Abstraction.Agents.Models;
+
+public class AgentLink : AgentPromptBase
+{
+    public AgentLink() : base()
+    {
+    }
+
+    public AgentLink(string name, string content) : base(name, content)
+    {
+    }
+
+    public override string ToString()
+    {
+        return base.ToString();
+    }
+}

src/Infrastructure/BotSharp.Abstraction/Agents/Models/AgentPromptBase.cs

@@ -0,0 +1,23 @@
+namespace BotSharp.Abstraction.Agents.Models;
+
+public class AgentPromptBase
+{
+    public string Name { get; set; }
+    public string Content { get; set; }
+
+    public AgentPromptBase()
+    {
+
+    }
+
+    public AgentPromptBase(string name, string content)
+    {
+        Name = name;
+        Content = content;
+    }
+
+    public override string ToString()
+    {
+        return Name;
+    }
+}

src/Infrastructure/BotSharp.Abstraction/Agents/Models/AgentTemplate.cs

@@ -1,23 +1,17 @@
 namespace BotSharp.Abstraction.Agents.Models;
 
-public class AgentTemplate
+public class AgentTemplate : AgentPromptBase
 {
-    public string Name { get; set; }
-    public string Content { get; set; }
-
-    public AgentTemplate()
+    public AgentTemplate() : base()
     {
-
     }
 
-    public AgentTemplate(string name, string content)
+    public AgentTemplate(string name, string content) : base(name, content)
     {
-        Name = name;
-        Content = content;
     }
 
     public override string ToString()
     {
-        return Name;
+        return base.ToString();
     }
 }

src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs

@@ -10,6 +10,7 @@ public interface IKnowledgeService
     Task<bool> CreateVectorCollection(string collectionName, string collectionType, int dimension, string provider, string model);
     Task<bool> DeleteVectorCollection(string collectionName);
     Task<IEnumerable<VectorCollectionConfig>> GetVectorCollections(string? type = null);
+    Task<VectorCollectionDetails?> GetVectorCollectionDetails(string collectionName);
     Task<IEnumerable<VectorSearchResult>> SearchVectorKnowledge(string query, string collectionName, VectorSearchOptions options);
     Task<StringIdPagedItems<VectorSearchResult>> GetPagedVectorCollectionData(string collectionName, VectorFilter filter);
     Task<bool> DeleteVectorCollectionData(string collectionName, string id);

src/Infrastructure/BotSharp.Abstraction/Loggers/IContentGeneratingHook.cs

@@ -47,5 +47,5 @@ public interface IContentGeneratingHook
     /// <param name="instruction"></param>
     /// <param name="functions"></param>
     /// <returns></returns>
-    Task OnSessionUpdated(Agent agent, string instruction, FunctionDef[] functions) => Task.CompletedTask;
+    Task OnSessionUpdated(Agent agent, string instruction, FunctionDef[] functions, bool isInit = false) => Task.CompletedTask;
 }

src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs

@@ -8,7 +8,8 @@ public interface IRealTimeCompletion
     string Model { get; }
     void SetModelName(string model);
 
-    Task Connect(RealtimeHubConnection conn,
+    Task Connect(
+        RealtimeHubConnection conn,
         Action onModelReady,
         Action<string, string> onModelAudioDeltaReceived,
         Action onModelAudioResponseDone,
@@ -23,7 +24,7 @@ Task Connect(RealtimeHubConnection conn,
     Task SendEventToModel(object message);
     Task Disconnect();
 
-    Task<string> UpdateSession(RealtimeHubConnection conn);
+    Task<string> UpdateSession(RealtimeHubConnection conn, bool isInit = false);
     Task InsertConversationItem(RoleDialogModel message);
     Task RemoveConversationItem(string itemId);
     Task TriggerModelInference(string? instructions = null);

src/Infrastructure/BotSharp.Abstraction/Templating/ITemplateRender.cs

@@ -3,5 +3,22 @@ namespace BotSharp.Abstraction.Templating;
 public interface ITemplateRender
 {
     string Render(string template, Dictionary<string, object> dict);
-    void Register(Type type);
+
+    /// <summary>
+    /// Register tag
+    /// </summary>
+    /// <param name="tag"></param>
+    /// <param name="content">A dictionary whose key is identifier and value is its content to render</param>
+    /// <param name="data"></param>
+    /// <returns></returns>
+    bool RegisterTag(string tag, Dictionary<string, string> content, Dictionary<string, object>? data = null);
+
+    /// <summary>
+    /// Register tags
+    /// </summary>
+    /// <param name="tags">A dictionary whose key is tag and value is its identifier and content to render</param>
+    /// <param name="data"></param>
+    /// <returns></returns>
+    bool RegisterTags(Dictionary<string, List<AgentPromptBase>> tags, Dictionary<string, object>? data = null);
+    void RegisterType(Type type);
 }

src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs

@@ -10,6 +10,8 @@ Task<bool> DoesCollectionExist(string collectionName)
         => throw new NotImplementedException();
     Task<IEnumerable<string>> GetCollections()
         => throw new NotImplementedException();
+    Task<VectorCollectionDetails?> GetCollectionDetails(string collectionName)
+        => throw new NotImplementedException();
     Task<StringIdPagedItems<VectorCollectionData>> GetPagedCollectionData(string collectionName, VectorFilter filter)
         => throw new NotImplementedException();
     Task<IEnumerable<VectorCollectionData>> GetCollectionData(string collectionName, IEnumerable<Guid> ids,

src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorCollectionDetails.cs

@@ -0,0 +1,51 @@
+namespace BotSharp.Abstraction.VectorStorage.Models;
+
+public class VectorCollectionDetails
+{
+    [JsonPropertyName("status")]
+    public string Status { get; set; }
+
+    [JsonPropertyName("optimizer_status")]
+    public string OptimizerStatus { get; set; }
+
+    [JsonPropertyName("segments_count")]
+    public ulong SegmentsCount { get; set; }
+
+    [JsonPropertyName("vectors_count")]
+    public ulong VectorsCount { get; set; }
+
+    [JsonPropertyName("indexed_vectors_count")]
+    public ulong IndexedVectorsCount { get; set; }
+
+    [JsonPropertyName("points_count")]
+    public ulong PointsCount { get; set; }
+
+    [JsonPropertyName("inner_config")]
+    public VectorCollectionDetailConfig? InnerConfig { get; set; }
+
+    [JsonPropertyName("basic_info")]
+    public VectorCollectionConfig? BasicInfo { get; set; }
+}
+
+public class VectorCollectionDetailConfig
+{
+    public VectorCollectionDetailConfigParam? Param { get; set; }
+}
+
+public class VectorCollectionDetailConfigParam
+{
+    [JsonPropertyName("shard_number")]
+    public uint? ShardNumber { get; set; }
+
+    [JsonPropertyName("sharding_method")]
+    public string? ShardingMethod { get; set; }
+
+    [JsonPropertyName("replication_factor")]
+    public uint? ReplicationFactor { get; set; }
+
+    [JsonPropertyName("write_consistency_factor")]
+    public uint? WriteConsistencyFactor { get; set; }
+
+    [JsonPropertyName("read_fan_out_factor")]
+    public uint? ReadFanOutFactor { get; set; }
+}

src/Infrastructure/BotSharp.Core.MCP/Managers/McpClientManager.cs

@@ -23,7 +23,9 @@ public async Task<IMcpClient> GetMcpClientAsync(string serverId)
             transport = new SseClientTransport(new SseClientTransportOptions
             {
                 Name = config.Name,
-                Endpoint = new Uri(config.SseConfig.EndPoint)
+                Endpoint = new Uri(config.SseConfig.EndPoint),
+                AdditionalHeaders = config.SseConfig.AdditionalHeaders,
+                ConnectionTimeout = config.SseConfig.ConnectionTimeout
             });
         }
         else if (config.StdioConfig != null)
@@ -33,7 +35,8 @@ public async Task<IMcpClient> GetMcpClientAsync(string serverId)
                 Name = config.Name,
                 Command = config.StdioConfig.Command,
                 Arguments = config.StdioConfig.Arguments,
-                EnvironmentVariables = config.StdioConfig.EnvironmentVariables
+                EnvironmentVariables = config.StdioConfig.EnvironmentVariables,
+                ShutdownTimeout = config.StdioConfig.ShutdownTimeout
             });
         }
         else

src/Infrastructure/BotSharp.Core.Realtime/BotSharp.Core.Realtime.csproj

@@ -12,6 +12,7 @@
 
   <ItemGroup>
     <PackageReference Include="NAudio" />
+    <PackageReference Include="System.ClientModel" />
   </ItemGroup>
 
   <ItemGroup>

src/Infrastructure/BotSharp.Core.Realtime/Models/Chat/ChatSessionUpdate.cs

@@ -0,0 +1,11 @@
+namespace BotSharp.Core.Realtime.Models.Chat;
+
+public class ChatSessionUpdate
+{
+    public string RawResponse { get; set; }
+
+    public ChatSessionUpdate()
+    {
+
+    }
+}

src/Infrastructure/BotSharp.Core.Realtime/Models/Options/ChatSessionOptions.cs

@@ -0,0 +1,7 @@
+namespace BotSharp.Core.Realtime.Models.Options;
+
+public class ChatSessionOptions
+{
+    public int? BufferSize { get; set; }
+    public JsonSerializerOptions? JsonOptions { get; set; }
+}

src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs

@@ -42,14 +42,15 @@ public async Task ConnectToModel(Func<string, Task>? responseToUser = null, Func
 
         _completer = _services.GetServices<IRealTimeCompletion>().First(x => x.Provider == settings.Provider);
 
-        await _completer.Connect(_conn, 
+        await _completer.Connect(
+            conn: _conn, 
             onModelReady: async () => 
             {
                 // Not TriggerModelInference, waiting for user utter.
-                var instruction = await _completer.UpdateSession(_conn);
+                var instruction = await _completer.UpdateSession(_conn, isInit: true);
                 var data = _conn.OnModelReady();
-                await (init?.Invoke(data) ?? Task.CompletedTask);
                 await HookEmitter.Emit<IRealtimeHook>(_services, async hook => await hook.OnModelReady(agent, _completer));
+                await (init?.Invoke(data) ?? Task.CompletedTask);
             },
             onModelAudioDeltaReceived: async (audioDeltaData, itemId) =>
             {

src/Infrastructure/BotSharp.Core.Realtime/Services/WaveStreamChannel.cs

@@ -39,6 +39,7 @@ public async Task ConnectAsync(string conversationId)
         var waveFormat = new WaveFormat(24000, 16, 1); // 24000 Hz, 16-bit PCM, Mono
         _bufferedWaveProvider = new BufferedWaveProvider(waveFormat);
         _bufferedWaveProvider.BufferDuration = TimeSpan.FromMinutes(10);
+        //_bufferedWaveProvider.BufferLength = 1024;
         _bufferedWaveProvider.DiscardOnBufferOverflow = true;
 
         _waveOut = new WaveOutEvent()

src/Infrastructure/BotSharp.Core.Realtime/Using.cs

@@ -15,3 +15,7 @@
 global using BotSharp.Abstraction.Routing;
 global using BotSharp.Abstraction.Agents.Enums;
 global using BotSharp.Abstraction.Conversations.Models;
+
+global using BotSharp.Core.Realtime.Models.Chat;
+global using BotSharp.Core.Realtime.Models.Options;
+global using BotSharp.Core.Realtime.Websocket.Chat;