feat: add C# implementation of image generation as a service

Author: srebrek

ImageGen/MaIN.ImageGen/appsettings.json

@@ -10,7 +10,7 @@
     "NumInferenceSteps": 8,
     "GuidanceScale": 7.5,
     "ExecutionProviderTarget": "Cpu", // Options: "Cpu", "DirectML"
-    "ModelPath": "C:\\Users\\examplePath\\CompVis"
+    "ModelPath": "C:\\Users\\zlote\\Desktop\\MAIN\\ImageGeneration\\ImageGeneration\\OnnxModels\\CompVis"
     // Folder hierarchy should look like:
     // CompVis
     // ├── text_encoder

MaIN.sln

@@ -1,7 +1,7 @@
 
 Microsoft Visual Studio Solution File, Format Version 12.00
 # Visual Studio Version 17
-VisualStudioVersion = 17.14.36429.23 d17.14
+VisualStudioVersion = 17.14.36429.23
 MinimumVisualStudioVersion = 10.0.40219.1
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "MaIN", "src\MaIN\MaIN.csproj", "{7E8DD6E0-F9A8-4BD0-8733-49F0F2F4FCA0}"
 EndProject
@@ -29,8 +29,6 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "MaIN.InferPage", "src\MaIN.
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "MaIN.Core.IntegrationTests", "MaIN.Core.IntegrationTests\MaIN.Core.IntegrationTests.csproj", "{2C15062A-E9F6-47FC-A4CD-1190A49E3FE3}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "MaIN.ImageGen", "ImageGen\MaIN.ImageGen\MaIN.ImageGen.csproj", "{2345FAF2-6462-465B-8D80-7BD2819D66CF}"
-EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -81,10 +79,6 @@ Global
 		{2C15062A-E9F6-47FC-A4CD-1190A49E3FE3}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{2C15062A-E9F6-47FC-A4CD-1190A49E3FE3}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{2C15062A-E9F6-47FC-A4CD-1190A49E3FE3}.Release|Any CPU.Build.0 = Release|Any CPU
-		{2345FAF2-6462-465B-8D80-7BD2819D66CF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{2345FAF2-6462-465B-8D80-7BD2819D66CF}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{2345FAF2-6462-465B-8D80-7BD2819D66CF}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{2345FAF2-6462-465B-8D80-7BD2819D66CF}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE

src/MaIN.Domain/Configuration/MaINSettings.cs

@@ -13,6 +13,7 @@ public class MaINSettings
     public string? DeepSeekKey { get; set; }
     public string? AnthropicKey { get; set; }
     public string? GroqCloudKey { get; set; }
+    public ImageGenSettings? ImageGenSettings { get; set; }
     public MongoDbSettings? MongoDbSettings { get; set; }
     public FileSystemSettings? FileSystemSettings { get; set; }
     public SqliteSettings? SqliteSettings { get; set; }
@@ -28,4 +29,13 @@ public enum BackendType
     DeepSeek = 3,
     GroqCloud = 4,
     Anthropic = 5,
+    ONNX = 6,
+}
+
+public class ImageGenSettings
+{
+    public string ExecutionProviderTarget { get; set; } = "Cpu"; // Options: "Cpu", "DirectML"
+    public int NumInferenceSteps { get; set; } = 8;
+    public double GuidanceScale { get; set; } = 7.5;
+    public string ModelPath { get; set; } = string.Empty;
 }

src/MaIN.Services/MaIN.Services.csproj

@@ -20,6 +20,7 @@
       <PackageReference Include="LLamaSharp.Backend.Cuda12" Version="0.25.0" />
       <PackageReference Include="LLamaSharp.kernel-memory" Version="0.25.0" />
       <PackageReference Include="LLamaSharp.semantic-kernel" Version="0.25.0" />
+      <PackageReference Include="MathNet.Numerics" Version="5.0.0" />
       <PackageReference Include="Microsoft.Extensions.Http" Version="9.0.4" />
       <PackageReference Include="Microsoft.KernelMemory" Version="0.98.250508.3" />
       <PackageReference Include="Microsoft.KernelMemory.SemanticKernelPlugin" Version="0.98.250508.3" />
@@ -29,9 +30,16 @@
       <PackageReference Include="ModelContextProtocol.AspNetCore" Version="0.2.0-preview.1" />
       <PackageReference Include="NAudio" Version="2.2.1" />
       <PackageReference Include="NumSharp" Version="0.30.0" />
+      <PackageReference Include="SixLabors.ImageSharp" Version="3.1.11" />
       <PackageReference Include="System.Data.SqlClient" Version="4.8.6" />
       <PackageReference Include="Tesseract" Version="5.2.0" />
       <PackageReference Include="Tesseract.Data.English" Version="4.0.0" />
     </ItemGroup>
 
+    <ItemGroup>
+      <Reference Include="StableDiffusion.ML.OnnxRuntime">
+        <HintPath>Services\ImageGenServices\StableDiffusion.ML.OnnxRuntime.dll</HintPath>
+      </Reference>
+    </ItemGroup>
+
 </Project>

src/MaIN.Services/Services/ChatService.cs

@@ -59,7 +59,7 @@ public async Task<ChatResult> Completions(
     }
 
     var result = chat.Visual 
-        ? await imageGenServiceFactory.CreateService(chat.Backend.Value)!.Send(chat) 
+        ? await imageGenServiceFactory.CreateService(BackendType.ONNX)!.Send(chat) 
         : await llmServiceFactory.CreateService(chat.Backend.Value).Send(chat, new ChatRequestOptions()
         {
             InteractiveUpdates = interactiveUpdates,

src/MaIN.Services/Services/ImageGenServices/ImageGenONNXService.cs

@@ -0,0 +1,92 @@
+using MaIN.Domain.Configuration;
+using MaIN.Domain.Entities;
+using MaIN.Services.Constants;
+using MaIN.Services.Services.Abstract;
+using MaIN.Services.Services.Models;
+using SixLabors.ImageSharp;
+using StableDiffusion.ML.OnnxRuntime;
+using MathNet.Numerics;
+using MathNet.Numerics.LinearAlgebra;
+
+namespace MaIN.Services.Services.ImageGenServices;
+
+public class ImageGenONNXService(MaINSettings settings) : IImageGenService
+{
+    private readonly MaINSettings _settings = settings ?? throw new ArgumentNullException(nameof(settings));
+    public Task<ChatResult?> Send(Chat chat)
+    {
+        var matrixCheck = Matrix<double>.Build.Dense(1, 1); // Ensures MathNet.Numerics is referenced
+        StableDiffusionConfig config = GetStableDiffusionConfig();
+
+        string prompt = chat.Messages
+            .Select((msg, index) => index == 0 ? msg.Content : $"&& {msg.Content}")
+            .Aggregate((current, next) => $"{current} {next}");
+
+        try
+        {
+            Image image = GenerateImage(prompt, config);
+            using var ms = new MemoryStream();
+            image.SaveAsPng(ms);
+            byte[] imageBytes = ms.ToArray();
+            return Task.FromResult<ChatResult?>(CreateChatResult(imageBytes));
+        }
+        catch (Exception ex)
+        {
+            throw new InvalidOperationException("Error generating image with ONNX", ex);
+        }
+    }
+
+    private StableDiffusionConfig GetStableDiffusionConfig()
+    {
+        ImageGenSettings imgSettings = new()
+        {
+            ExecutionProviderTarget = "Cpu",
+            NumInferenceSteps = 8,
+            GuidanceScale = 7.5,
+            ModelPath = "C:\\Users\\zlote\\Desktop\\MAIN\\ImageGeneration\\ImageGeneration\\OnnxModels\\CompVis"
+        };
+        //ImageGenSettings imgSettings = _settings.ImageGenSettings
+        //    ?? throw new InvalidOperationException("Image generation settings are not configured.");
+
+        return new StableDiffusionConfig
+        {
+            NumInferenceSteps = imgSettings.NumInferenceSteps,
+            GuidanceScale = imgSettings.GuidanceScale,
+            ExecutionProviderTarget = Enum.Parse<StableDiffusionConfig.ExecutionProvider>(imgSettings.ExecutionProviderTarget),
+            TextEncoderOnnxPath = Path.Combine(imgSettings.ModelPath, "text_encoder", "model.onnx"),
+            UnetOnnxPath = Path.Combine(imgSettings.ModelPath, "unet", "model.onnx"),
+            VaeDecoderOnnxPath = Path.Combine(imgSettings.ModelPath, "vae_decoder", "model.onnx"),
+            TokenizerOnnxPath = Path.Combine(imgSettings.ModelPath, "cliptokenizer.onnx"),
+            OrtExtensionsPath = Path.Combine(imgSettings.ModelPath, "ortextensions.dll"),
+        };
+    }
+
+    private Image GenerateImage(string prompt, StableDiffusionConfig config)
+    {
+        try
+        {
+            return UNet.Inference(prompt, config);
+        }
+        catch (Exception)
+        {
+            throw;
+        }
+    }
+
+    private ChatResult CreateChatResult(byte[] imageBytes)
+    {
+        return new ChatResult
+        {
+            Done = true,
+            Message = new Message
+            {
+                Content = ServiceConstants.Messages.GeneratedImageContent,
+                Role = ServiceConstants.Roles.Assistant,
+                Image = imageBytes,
+                Type = MessageType.Image
+            },
+            Model = "CompVis", //Path.GetFileName(_settings.ImageGenSettings!.ModelPath),
+            CreatedAt = DateTime.UtcNow
+        };
+    }
+}