Merge pull request #82 from wisedev-code/feature/tts

wisedev-pstach · web-flow · commit 03098b224a2d · 2025-08-25T19:29:15.000+02:00
Feature/tts
diff --git a/Examples/Examples/Chat/ChatWithTextToSpeechExample.cs b/Examples/Examples/Chat/ChatWithTextToSpeechExample.cs
@@ -0,0 +1,28 @@
+﻿using MaIN.Core.Hub;
+using MaIN.Domain.Entities;
+using MaIN.Services.Services.TTSService;
+
+namespace Examples;
+
+public class ChatWithTextToSpeechExample : IExample
+{
+    private const string VoicePath = "<your-path-to-voices>";
+    
+    public async Task Start()
+    {
+        Console.WriteLine("ChatWithTextToSpeech is running! Put on your headphones and press any key.");
+        Console.ReadKey();
+        
+        VoiceService.SetVoicesPath(VoicePath);
+        var voice = VoiceService.GetVoice("af_heart")
+            .MixWith(VoiceService.GetVoice("bf_emma"));
+        
+        await AIHub.Chat().WithModel("gemma2:2b")
+            .WithMessage("Generate a 4 sentence poem.")
+            .Speak(new TextToSpeechParams("kokoro:82m", voice, true))
+            .CompleteAsync(interactive: true);
+
+        Console.WriteLine("Done!");
+        Console.ReadKey();
+    }
+}
diff --git a/Examples/Examples/Program.cs b/Examples/Examples/Program.cs
@@ -67,6 +67,7 @@ static void RegisterExamples(IServiceCollection services)
     services.AddTransient<ChatWithImageGenGeminiExample>();
     services.AddTransient<ChatWithFilesExampleGemini>();
     services.AddTransient<ChatWithReasoningDeepSeekExample>();
+    services.AddTransient<ChatWithTextToSpeechExample>();
     services.AddTransient<ChatExampleGroqCloud>();
 }
 
@@ -156,6 +157,8 @@ public class ExampleRegistry(IServiceProvider serviceProvider)
             ("\u25a0 DeepSeek Chat with reasoning", serviceProvider.GetRequiredService<ChatWithReasoningDeepSeekExample>()),
             ("\u25a0 GroqCloud Chat", serviceProvider.GetRequiredService<ChatExampleGroqCloud>()),
             ("\u25a0 McpClient example", serviceProvider.GetRequiredService<McpExample>()),
+            ("\u25a0 McpAgent example", serviceProvider.GetRequiredService<McpAgentsExample>()),
+            ("\u25a0 Chat with TTS example", serviceProvider.GetRequiredService<ChatWithTextToSpeechExample>()),
             ("\u25a0 McpAgent example", serviceProvider.GetRequiredService<McpAgentsExample>())
         };
     }
diff --git a/Examples/Examples/appsettings.json b/Examples/Examples/appsettings.json
@@ -6,10 +6,7 @@
     }
   },
   "MaIN": {
-    "ImageGenUrl": "http://localhost:5003",
-    //    "SqliteSettings": {
-//      "ConnectionString": "Data Source=Main_test.db"
-//    }
+    "ImageGenUrl": "http://localhost:5003"
 },
   "AllowedHosts": "*"
 }
diff --git a/Releases/0.4.0.md b/Releases/0.4.0.md
@@ -0,0 +1,3 @@
+# 0.3.1 release
+
+- Added TextToSpeech functionality to using Kokoro TTS model. 
diff --git a/src/MaIN.Core/.nuspec b/src/MaIN.Core/.nuspec
@@ -2,7 +2,7 @@
 <package>
   <metadata>
     <id>MaIN.NET</id>
-    <version>0.3.1</version>
+    <version>0.4.0</version>
     <authors>Wisedev</authors>
     <owners>Wisedev</owners>
     <icon>favicon.png</icon>
diff --git a/src/MaIN.Core/Hub/Contexts/ChatContext.cs b/src/MaIN.Core/Hub/Contexts/ChatContext.cs
@@ -3,7 +3,6 @@
 using MaIN.Domain.Models;
 using MaIN.Services;
 using MaIN.Services.Constants;
-using MaIN.Services.Dtos;
 using MaIN.Services.Services.Abstract;
 using MaIN.Services.Services.Models;
 using FileInfo = MaIN.Domain.Entities.FileInfo;
@@ -61,6 +60,14 @@ public ChatContext WithCustomModel(string model, string path, string? mmProject
         return this;
     }
 
+    public ChatContext Speak(TextToSpeechParams textToSpeechParams)
+    {
+        _chat.Visual = false;
+        _chat.TextToSpeechParams = textToSpeechParams;
+        
+        return this;
+    }
+
     public ChatContext WithBackend(BackendType backendType)
     {
         _chat.Backend = backendType;
@@ -151,6 +158,7 @@ public ChatContext WithFiles(List<string> filePaths, bool preProcess = false)
     public ChatContext EnableVisual()
     {
         _chat.Visual = true;
+        _chat.TextToSpeechParams = null;
         return this;
     }
 
diff --git a/src/MaIN.Domain/Configuration/MaINSettings.cs b/src/MaIN.Domain/Configuration/MaINSettings.cs
@@ -16,6 +16,7 @@ public class MaINSettings
     public FileSystemSettings? FileSystemSettings { get; set; }
     public SqliteSettings? SqliteSettings { get; set; }
     public SqlSettings? SqlSettings { get; set; }
+    public string? VoicesPath { get; set; }
 }
 
 public enum BackendType
diff --git a/src/MaIN.Domain/Entities/Chat.cs b/src/MaIN.Domain/Entities/Chat.cs
@@ -13,6 +13,7 @@ public class Chat
     public bool Visual { get; set; }
     public InferenceParams InterferenceParams { get; set; } = new();
     public MemoryParams MemoryParams { get; set; } = new();
+    public TextToSpeechParams? TextToSpeechParams { get; set; }
     public Dictionary<string, string> Properties { get; init; } = [];
     public List<string> Memory { get; } = [];
     public BackendType? Backend { get; set; }
diff --git a/src/MaIN.Domain/Entities/Message.cs b/src/MaIN.Domain/Entities/Message.cs
@@ -20,6 +20,7 @@ public Message()
     public bool Tool { get; init; }
     public DateTime Time { get; set; }
     public byte[]? Image { get; init; }
+    public byte[]? Speech { get; set; }
     public List<FileInfo>? Files { get; set; } 
     public Dictionary<string, string> Properties { get; set; } = [];
 
diff --git a/src/MaIN.Domain/Entities/TextToSpeechParams.cs b/src/MaIN.Domain/Entities/TextToSpeechParams.cs
@@ -0,0 +1,15 @@
+﻿namespace MaIN.Domain.Entities;
+
+public class TextToSpeechParams
+{
+    public string Model { get; set; }
+    public Voice Voice { get;  set; }
+    public bool Playback { get; set; }
+
+    public TextToSpeechParams(string model, Voice voice, bool playback = false)
+    {
+        Model = model;  
+        Voice = voice;
+        Playback = playback;
+    }
+}
diff --git a/src/MaIN.Domain/Entities/Voice.cs b/src/MaIN.Domain/Entities/Voice.cs
@@ -0,0 +1,7 @@
+﻿namespace MaIN.Domain.Entities;
+
+public class Voice
+{
+    public required string Name { get; set; }
+    public required float[,,] Features { get; set; }
+}
diff --git a/src/MaIN.Domain/Models/SupportedModels.cs b/src/MaIN.Domain/Models/SupportedModels.cs
@@ -183,6 +183,13 @@ public static class KnownModels
             DownloadUrl = "https://huggingface.co/Inza124/QwQ-7b/resolve/main/qwq-7b.gguf?download=true",
             AdditionalPrompt = "- Output nothing before <think>, enclose all step-by-step reasoning (excluding the final answer) within <think>...</think>, and place the final answer immediately after the closing </think>",
             ReasonFunction = ReasoningFunctions.ProcessQwQ_QwenModToken
+        },
+        new()
+        {
+            Description = "Frontier TTS model for its size of 82 million parameters (text in/audio out).",
+            Name = KnownModelNames.Kokoro_82m,
+            FileName = "kokoro.onnx",
+            DownloadUrl = "https://github.com/taylorchu/kokoro-onnx/releases/download/v0.2.0/kokoro.onnx"
         }
     ];
 
@@ -294,4 +301,5 @@ public struct KnownModelNames
     public const string Smollm2_0_1b = "smollm2:0.1b";
     public const string Yi_6b = "yi:6b";
 
+    public const string Kokoro_82m = "kokoro:82m";
 }
diff --git a/src/MaIN.Services/Bootstrapper.cs b/src/MaIN.Services/Bootstrapper.cs
@@ -11,6 +11,7 @@
 using MaIN.Services.Services.Models.Commands;
 using MaIN.Services.Services.Steps;
 using MaIN.Services.Services.Steps.Commands;
+using MaIN.Services.Services.TTSService;
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.DependencyInjection;
 
@@ -37,13 +38,16 @@ public static IServiceCollection ConfigureMaIN(
         serviceCollection.AddSingleton<IMemoryFactory, MemoryFactory>();
         serviceCollection.AddSingleton<ILLMServiceFactory, LLMServiceFactory>();
         serviceCollection.AddSingleton<IImageGenServiceFactory, ImageGenServiceFactory>();
+        serviceCollection.AddSingleton<ITTSServiceFactory, TTSServiceFactory>();
 
 // Register all concrete implementations as transient
         serviceCollection.AddTransient<LLMService>();
         serviceCollection.AddTransient<OpenAiService>();
         serviceCollection.AddTransient<ImageGenService>();
         serviceCollection.AddTransient<OpenAiImageGenService>();
         
+        serviceCollection.AddTransient<ITextToSpeechService, TextToSpeechService>();
+        
         // Register all step handlers
         serviceCollection.AddSingleton<IStepHandler, RedirectStepHandler>();
         serviceCollection.AddSingleton<IStepHandler, FetchDataStepHandler>();
diff --git a/src/MaIN.Services/MaIN.Services.csproj b/src/MaIN.Services/MaIN.Services.csproj
@@ -23,6 +23,8 @@
       <PackageReference Include="Microsoft.SemanticKernel" Version="1.49.0" />
       <PackageReference Include="Microsoft.SemanticKernel.Connectors.Google" Version="1.33.0-alpha" />
       <PackageReference Include="ModelContextProtocol.AspNetCore" Version="0.2.0-preview.1" />
+      <PackageReference Include="NAudio" Version="2.2.1" />
+      <PackageReference Include="NumSharp" Version="0.30.0" />
       <PackageReference Include="System.Data.SqlClient" Version="4.8.6" />
       <PackageReference Include="Tesseract" Version="5.2.0" />
       <PackageReference Include="Tesseract.Data.English" Version="4.0.0" />
diff --git a/src/MaIN.Services/Services/Abstract/ITTSEnginge.cs b/src/MaIN.Services/Services/Abstract/ITTSEnginge.cs
@@ -0,0 +1,11 @@
+namespace MaIN.Services.Services.Abstract;
+
+public interface ITTSEnginge : IDisposable
+{
+    void InitializeSession(string modelPath);
+
+    Task<float[]> GenerateAudioAsync(string text, string voiceName = "af_heart", float speed = 1.0f, float pitch = 1.0f);
+
+    void SaveAudioToWav(float[] audioData, string outputPath, int sampleRate = 24000);
+    void PlayAudio(float[] audioData, int sampleRate = 24000);
+}
diff --git a/src/MaIN.Services/Services/Abstract/ITTSServiceFactory.cs b/src/MaIN.Services/Services/Abstract/ITTSServiceFactory.cs
@@ -0,0 +1,9 @@
+﻿using MaIN.Domain.Configuration;
+using MaIN.Services.Services.TTSService;
+
+namespace MaIN.Services.Services.Abstract;
+
+public interface ITTSServiceFactory
+{
+    ITextToSpeechService CreateService(BackendType backendType);
+}
diff --git a/src/MaIN.Services/Services/ChatService.cs b/src/MaIN.Services/Services/ChatService.cs
@@ -16,6 +16,7 @@ public class ChatService(
     IChatRepository chatProvider,
     ILLMServiceFactory llmServiceFactory,
     IImageGenServiceFactory imageGenServiceFactory,
+    ITTSServiceFactory ttsServiceFactory,
     MaINSettings settings) : IChatService
 {
     public async Task Create(Chat chat)
@@ -71,6 +72,15 @@ public async Task<ChatResult> Completions(
         result.Message.Time = DateTime.Now;
     }
     
+    if (!chat.Visual && chat.TextToSpeechParams != null)
+    {
+        var speechBytes = await ttsServiceFactory
+            .CreateService(chat.Backend.Value).Send(result!.Message, chat.TextToSpeechParams.Model,
+                chat.TextToSpeechParams.Voice, chat.TextToSpeechParams.Playback);
+        
+        result.Message.Speech = speechBytes;
+    }
+    
     originalMessages.Add(result!.Message);
     chat.Messages = originalMessages;
     
diff --git a/src/MaIN.Services/Services/LLMService/Factory/TTSServiceFactory.cs b/src/MaIN.Services/Services/LLMService/Factory/TTSServiceFactory.cs
@@ -0,0 +1,18 @@
+﻿using MaIN.Domain.Configuration;
+using MaIN.Services.Services.Abstract;
+using MaIN.Services.Services.TTSService;
+using Microsoft.Extensions.DependencyInjection;
+
+namespace MaIN.Services.Services.LLMService.Factory;
+
+public class TTSServiceFactory(IServiceProvider serviceProvider) : ITTSServiceFactory
+{
+    public ITextToSpeechService CreateService(BackendType backendType)
+    {
+        return backendType switch
+        {
+            BackendType.Self => new TTSService.TextToSpeechService(serviceProvider.GetRequiredService<MaINSettings>()),
+            _ => throw new ArgumentOutOfRangeException(nameof(backendType))
+        };
+    }
+}
diff --git a/src/MaIN.Services/Services/TTSService/TextToSpeechModel.cs b/src/MaIN.Services/Services/TTSService/TextToSpeechModel.cs
@@ -0,0 +1,78 @@
+﻿using Microsoft.ML.OnnxRuntime;
+using Microsoft.ML.OnnxRuntime.Tensors;
+
+namespace MaIN.Services.Services.TTSService;
+
+/// <summary>
+/// Currently working with kokoro model. WIP to work with other
+/// </summary>
+public sealed class TextToSpeechModel : IDisposable
+{
+    private readonly InferenceSession _session;
+    private readonly SessionOptions _defaultOptions = new() 
+    { 
+        EnableMemoryPattern = true, 
+        InterOpNumThreads = 8, 
+        IntraOpNumThreads = 8 
+    };
+
+    public const int MaxTokens = 510;
+
+    public TextToSpeechModel(string modelPath, SessionOptions? options = null)
+    {
+        _session = new InferenceSession(modelPath, options ?? _defaultOptions);
+    }
+
+    public float[] Infer(int[] tokens, float[,,] voiceStyle, float speed = 1)
+    {
+        var (B, T, C) = (1, tokens.Length, voiceStyle.GetLength(2));
+        
+        switch (tokens.Length)
+        {
+            case 0:
+                return [];
+            case > MaxTokens:
+                Array.Resize(ref tokens, T = MaxTokens);
+                break;
+        }
+
+        var tokenTensor = new DenseTensor<long>(new[] { B, T + 2 });
+        var styleTensor = new DenseTensor<float>(new[] { B, C });
+        var speedTensor = new DenseTensor<float>(new[] { speed }, new[] { B });
+
+        // Form Kokoro's input (<start>{text}<end>)
+        var inputTokens = new int[T + 2];
+        Array.Copy(tokens, 0, inputTokens, 1, T);
+
+        for (var j = 0; j < C; j++) 
+        { 
+            styleTensor[0, j] = voiceStyle[T - 1, 0, j]; 
+        }
+        
+        for (var i = 0; i < inputTokens.Length; i++) 
+        { 
+            tokenTensor[0, i] = inputTokens[i] >= 0 ? inputTokens[i] : 4; // [unk] --> '.'
+        }
+
+        var inputs = new List<NamedOnnxValue> 
+        { 
+            NamedOnnxValue.CreateFromTensor("tokens", tokenTensor),
+            NamedOnnxValue.CreateFromTensor("style", styleTensor),
+            NamedOnnxValue.CreateFromTensor("speed", speedTensor)
+        };
+
+        lock (_session)
+        {
+            using var results = _session.Run(inputs);
+            return results[0].AsTensor<float>().ToArray();
+        }
+    }
+
+    public void Dispose()
+    {
+        lock (_session)
+        {
+            _session.Dispose();
+        }
+    }
+}
diff --git a/src/MaIN.Services/Services/TTSService/TextToSpeechService.cs b/src/MaIN.Services/Services/TTSService/TextToSpeechService.cs
diff --git a/src/MaIN.Services/Services/TTSService/Tokenizer.cs b/src/MaIN.Services/Services/TTSService/Tokenizer.cs
diff --git a/src/MaIN.Services/Services/TTSService/VoiceService.cs b/src/MaIN.Services/Services/TTSService/VoiceService.cs

Original file line number	Diff line number	Diff line change
`@@ -6,10 +6,7 @@`
`6`	`6`	`}`
`7`	`7`	`},`
`8`	`8`	`"MaIN": {`
`9`		`- "ImageGenUrl": "http://localhost:5003",`
`10`		`- // "SqliteSettings": {`
`11`		`-// "ConnectionString": "Data Source=Main_test.db"`
`12`		`-// }`
	`9`	`+ "ImageGenUrl": "http://localhost:5003"`
`13`	`10`	`},`
`14`	`11`	`"AllowedHosts": "*"`
`15`	`12`	`}`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+# 0.3.1 release`
	`2`	`+`
	`3`	`+- Added TextToSpeech functionality to using Kokoro TTS model.`
Original file line number	Diff line number	Diff line change
`@@ -16,6 +16,7 @@ public class MaINSettings`
`16`	`16`	`public FileSystemSettings? FileSystemSettings { get; set; }`
`17`	`17`	`public SqliteSettings? SqliteSettings { get; set; }`
`18`	`18`	`public SqlSettings? SqlSettings { get; set; }`
	`19`	`+ public string? VoicesPath { get; set; }`
`19`	`20`	`}`
`20`	`21`
`21`	`22`	`public enum BackendType`