Merge pull request #60 from wisedev-code/feat/vl-integration

wisedev-pstach · web-flow · commit 6ed0e1bb8001 · 2025-06-12T11:54:25.000+02:00
Feat/vl integration
diff --git a/Examples/Examples/Agents/AgentsWithRedirectImageExample.cs b/Examples/Examples/Agents/AgentsWithRedirectImageExample.cs
@@ -51,6 +51,6 @@ Generate image based on given prompt
                 }]
             });
         
-        ImagePreview.ShowImage(result.Message.Images);
+        ImagePreview.ShowImage(result.Message.Image);
     }
 }
diff --git a/Examples/Examples/Chat/ChatWithImageGenExample.cs b/Examples/Examples/Chat/ChatWithImageGenExample.cs
@@ -14,6 +14,6 @@ public async Task Start()
             .WithMessage("Generate cyberpunk godzilla cat warrior")
             .CompleteAsync();
         
-        ImagePreview.ShowImage(result.Message.Images);
+        ImagePreview.ShowImage(result.Message.Image);
     }
 }
diff --git a/Examples/Examples/Chat/ChatWithImageGenGeminiExample.cs b/Examples/Examples/Chat/ChatWithImageGenGeminiExample.cs
@@ -15,6 +15,6 @@ public async Task Start()
             .WithMessage("Generate hamster as a astronaut on the moon")
             .CompleteAsync();
 
-        ImagePreview.ShowImage(result.Message.Images);
+        ImagePreview.ShowImage(result.Message.Image);
     }
 }
diff --git a/Examples/Examples/Chat/ChatWithImageGenOpenAiExample.cs b/Examples/Examples/Chat/ChatWithImageGenOpenAiExample.cs
@@ -15,6 +15,6 @@ public async Task Start()
             .WithMessage("Generate rock style cow playing guitar")
             .CompleteAsync();
         
-        ImagePreview.ShowImage(result.Message.Images);
+        ImagePreview.ShowImage(result.Message.Image);
     }
 }
diff --git a/Examples/Examples/Chat/ChatWithVisionExample.cs b/Examples/Examples/Chat/ChatWithVisionExample.cs
@@ -1,25 +1,22 @@
-using System.Reflection;
 using MaIN.Core.Hub;
 
 namespace Examples;
 
 public class ChatWithVisionExample : IExample
 {
-    /// <summary>
-    /// Vision via Multimodal models as Llava is not supported yet
-    /// </summary>
     public async Task Start()
     {
-        Console.WriteLine("ChatExample with files is running!");
+        //https://huggingface.co/cjpais/llava-1.6-mistral-7b-gguf - Tried with this model
+        Console.WriteLine("ChatExample with vision model is running!");
 
-        List<string> images = [Path.Combine(AppContext.BaseDirectory, "Files", "gamex.jpg")];
+        var image = await File.ReadAllBytesAsync(
+            Path.Combine(AppContext.BaseDirectory, "Files", "gamex.jpg"));
         
-        var result = await AIHub.Chat()
-            .WithModel("llama3.2:3b")
-            .WithMessage("What is the title of game?")
-            .WithFiles(images)
-            .CompleteAsync();
-        
-        Console.WriteLine(result.Message.Content);
+        await AIHub.Chat()
+            .WithCustomModel("Llava1.6-Mistral",
+                path: "<path_to_model>.gguf",
+                mmProject: "<path_to_mmproj>.gguf")
+            .WithMessage("What can you see on the image?", image)
+            .CompleteAsync(interactive: true);
     }
 }
diff --git a/MaIN.Core.IntegrationTests/ChatTests.cs b/MaIN.Core.IntegrationTests/ChatTests.cs
@@ -98,7 +98,7 @@ public async Task Should_GenerateImage_BasedOnPrompt()
             throw new ArgumentException("Invalid file extension");
 
         Assert.True(result.Done);
-        Assert.NotNull(result.Message.Images);
+        Assert.NotNull(result.Message.Image);
     }
     
     [Fact]
diff --git a/Releases/0.2.5.md b/Releases/0.2.5.md
@@ -0,0 +1,3 @@
+# 0.2.5 release
+
+- Integrate with vision llava models
diff --git a/src/MaIN.Core/.nuspec b/src/MaIN.Core/.nuspec
@@ -2,7 +2,7 @@
 <package>
   <metadata>
     <id>MaIN.NET</id>
-    <version>0.2.4</version>
+    <version>0.2.5</version>
     <authors>Wisedev</authors>
     <owners>Wisedev</owners>
     <icon>favicon.png</icon>
diff --git a/src/MaIN.Core/Hub/Contexts/AgentContext.cs b/src/MaIN.Core/Hub/Contexts/AgentContext.cs
@@ -114,9 +114,9 @@ public AgentContext WithMemoryParams(MemoryParams memoryParams)
         return this;
     }
 
-    public AgentContext WithCustomModel(string model, string path)
+    public AgentContext WithCustomModel(string model, string path, string? mmProject = null)
     {
-        KnownModels.AddModel(model, path);
+        KnownModels.AddModel(model, path, mmProject);
         _agent.Model = model;
         return this;
     }
diff --git a/src/MaIN.Core/Hub/Contexts/ChatContext.cs b/src/MaIN.Core/Hub/Contexts/ChatContext.cs
@@ -54,9 +54,9 @@ public ChatContext WithMemoryParams(MemoryParams memoryParams)
         return this;
     }
 
-    public ChatContext WithCustomModel(string model, string path)
+    public ChatContext WithCustomModel(string model, string path, string? mmProject = null)
     {
-        KnownModels.AddModel(model, path);
+        KnownModels.AddModel(model, path, mmProject);
         _chat.Model = model;
         return this;
     }
@@ -79,6 +79,21 @@ public ChatContext WithMessage(string content)
         _chat.Messages.Add(message);
         return this;
     }
+    
+    
+    public ChatContext WithMessage(string content, byte[] image)
+    {
+        var message = new Message
+        {
+            Role = "User",
+            Content = content,
+            Time = DateTime.Now,
+            Image = image
+        };
+        
+        _chat.Messages.Add(message);
+        return this;
+    }
 
     public ChatContext WithSystemPrompt(string systemPrompt)
     {
diff --git a/src/MaIN.Domain/Entities/Message.cs b/src/MaIN.Domain/Entities/Message.cs
@@ -10,7 +10,7 @@ public class Message
     public List<LLMTokenValue> Tokens { get; set; } = [];
     public bool Tool { get; init; }
     public DateTime Time { get; set; }
-    public byte[]? Images { get; init; }
+    public byte[]? Image { get; init; }
     public List<FileInfo>? Files { get; set; } 
     public Dictionary<string, string> Properties { get; set; } = [];
 }
diff --git a/src/MaIN.Domain/Models/SupportedModels.cs b/src/MaIN.Domain/Models/SupportedModels.cs
@@ -4,6 +4,7 @@ public class Model
 {
     public required string Name { get; init; }
     public required string FileName { get; init; }
+    public string? MMProject { get; set; }
     public string? DownloadUrl { get; set; }
     public string? AdditionalPrompt { get; set; }
     public string? Description { get; set; }
@@ -231,12 +232,13 @@ public static Model GetModel(string path, string? name)
         throw new Exception($"Model {fileName} is not downloaded");
     }
 
-    public static void AddModel(string model, string path)
+    public static void AddModel(string model, string path, string? mmProject)
     {
         Models.Add(new Model()
         {
             Description = string.Empty,
             DownloadUrl = string.Empty,
+            MMProject = mmProject,
             Name = model,
             FileName = $"{Path.GetFileName(path)}",
             Path = path
diff --git a/src/MaIN.InferPage/Components/Pages/Home.razor b/src/MaIN.InferPage/Components/Pages/Home.razor
@@ -38,9 +38,9 @@
                             <FluentCard class="message-card-img bot-message"
                                         Style="height: 30rem !important; width: 30rem !important; ">
                                 <div>
-                                    <a href="data:image/png;base64,@Convert.ToBase64String(conversation.Message.Images!)"
+                                    <a href="data:image/png;base64,@Convert.ToBase64String(conversation.Message.Image!)"
                                        style="cursor: -webkit-zoom-in; cursor: zoom-in;" target="_blank">
-                                        <img src="data:image/png;base64,@Convert.ToBase64String(conversation.Message.Images!)"
+                                        <img src="data:image/png;base64,@Convert.ToBase64String(conversation.Message.Image!)"
                                              style="object-fit: fill; width:100%; height:100%;"
                                              alt="imageResponse"/>
                                     </a>
diff --git a/src/MaIN.Services/Mappers/ChatMapper.cs b/src/MaIN.Services/Mappers/ChatMapper.cs
@@ -28,7 +28,7 @@ private static MessageDto ToDto(this Message message)
         {
             Content = message.Content,
             Role = message.Tool ? "System" : message.Role,
-            Images = message.Images,
+            Images = message.Image,
             Time = message.Time,
             Properties = message.Properties,
             Files = message.Files?.Select(x => new FileInfoDto()
@@ -57,7 +57,7 @@ private static Message ToDomain(this MessageDto message)
         {
             Content = message.Content,
             Role = message.Role,
-            Images = message.Images,
+            Image = message.Images,
             Time = message.Time,
             Properties = message.Properties,
             Files = message.Files?.Select(x => new FileInfo()
@@ -74,7 +74,7 @@ private static MessageDocument ToDocument(this Message message)
             Content = message.Content,
             Role = message.Role,
             Time = message.Time,
-            Images = message.Images,
+            Images = message.Image,
             Tokens = message.Tokens.Select(x => x.ToDocument()).ToList(),
             Properties = message.Properties,
             Tool = message.Tool,
@@ -125,7 +125,7 @@ private static Message ToDomain(this MessageDocument message)
             Time = message.Time,
             Tokens = message.Tokens.Select(x => x.ToDomain()).ToList(),
             Role = message.Role,
-            Images = message.Images,
+            Image = message.Images,
             Properties = message.Properties,
         };
 
diff --git a/src/MaIN.Services/Services/ImageGenServices/GeminiImageGenService.cs b/src/MaIN.Services/Services/ImageGenServices/GeminiImageGenService.cs
@@ -64,7 +64,7 @@ private static ChatResult CreateChatResult(byte[] imageBytes)
             {
                 Content = ServiceConstants.Messages.GeneratedImageContent,
                 Role = ServiceConstants.Roles.Assistant,
-                Images = imageBytes
+                Image = imageBytes
             },
             Model = Models.IMAGEN_GENERATE,
             CreatedAt = DateTime.UtcNow
diff --git a/src/MaIN.Services/Services/ImageGenServices/ImageGenDalleService.cs b/src/MaIN.Services/Services/ImageGenServices/ImageGenDalleService.cs
@@ -70,7 +70,7 @@ private static ChatResult CreateChatResult(byte[] imageBytes)
             {
                 Content = ServiceConstants.Messages.GeneratedImageContent,
                 Role = ServiceConstants.Roles.Assistant,
-                Images = imageBytes
+                Image = imageBytes
             },
             Model = Models.DALLE,
             CreatedAt = DateTime.UtcNow
diff --git a/src/MaIN.Services/Services/ImageGenServices/ImageGenService.cs b/src/MaIN.Services/Services/ImageGenServices/ImageGenService.cs
@@ -49,7 +49,7 @@ private static ChatResult CreateChatResult(byte[] imageBytes)
             {
                 Content = ServiceConstants.Messages.GeneratedImageContent,
                 Role = ServiceConstants.Roles.Assistant,
-                Images = imageBytes
+                Image = imageBytes
             },
             Model = LocalImageModels.FLUX,
             CreatedAt = DateTime.UtcNow
diff --git a/src/MaIN.Services/Services/LLMService/LLMService.cs b/src/MaIN.Services/Services/LLMService/LLMService.cs
@@ -120,7 +120,7 @@ public Task CleanSessionCache(string? id)
         {
             llmModel.Dispose();
         }
-        
+
         memory.TextGenerationContext.Dispose();
         memory.EmbeddingGenerator.Dispose();
 
@@ -154,10 +154,15 @@ private async Task<List<LLMTokenValue>> ProcessChatRequest(
             ? await LLamaWeights.LoadFromFileAsync(parameters, cancellationToken)
             : await ModelLoader.GetOrLoadModelAsync(modelsPath, modelKey);
 
+        var llavaWeights = model.MMProject != null
+            ? await LLavaWeights.LoadFromFileAsync(model.MMProject, cancellationToken)
+            : null;
+
+
         using var executor = new BatchedExecutor(llmModel, parameters);
 
-        var (conversation, isComplete, hasFailed) = InitializeConversation(
-            chat, lastMsg, model, llmModel, executor, cancellationToken);
+        var (conversation, isComplete, hasFailed) = await InitializeConversation(
+            chat, lastMsg, model, llmModel, llavaWeights, executor, cancellationToken);
 
         if (!isComplete)
         {
@@ -196,49 +201,77 @@ private ModelParams CreateModelParameters(Chat chat, string modelKey)
         };
     }
 
-    private (Conversation Conversation, bool IsComplete, bool HasFailed) InitializeConversation(
-        Chat chat,
+    private async Task<(Conversation Conversation, bool IsComplete, bool HasFailed)> InitializeConversation(Chat chat,
         Message lastMsg,
         Model model,
         LLamaWeights llmModel,
+        LLavaWeights? llavaWeights,
         BatchedExecutor executor,
         CancellationToken cancellationToken)
     {
         var isNewConversation = chat.ConversationState == null;
-        Conversation conversation;
+        var conversation = isNewConversation
+            ? executor.Create()
+            : executor.Load(chat.ConversationState!);
+
+        if (lastMsg.Image != null)
+        {
+            await ProcessImageMessage(conversation, lastMsg, llmModel, llavaWeights, executor, cancellationToken);
+        }
+        else
+        {
+            ProcessTextMessage(conversation, chat, lastMsg, model, llmModel, executor, isNewConversation);
+        }
+
+        return (conversation, false, false);
+    }
+
+    private static async Task ProcessImageMessage(Conversation conversation,
+        Message lastMsg,
+        LLamaWeights llmModel,
+        LLavaWeights? llavaWeights,
+        BatchedExecutor executor,
+        CancellationToken cancellationToken)
+    {
+        var imageEmbeddings = llavaWeights?.CreateImageEmbeddings(lastMsg.Image!);
+        conversation.Prompt(imageEmbeddings!);
+
+        while (executor.BatchedTokenCount > 0)
+            await executor.Infer(cancellationToken);
+
+        var prompt = llmModel.Tokenize($"USER: {lastMsg.Content}\nASSISTANT:", true, false, Encoding.UTF8);
+        conversation.Prompt(prompt);
+    }
+
+    private static void ProcessTextMessage(Conversation conversation,
+        Chat chat,
+        Message lastMsg,
+        Model model,
+        LLamaWeights llmModel,
+        BatchedExecutor executor,
+        bool isNewConversation)
+    {
+        var template = new LLamaTemplate(llmModel);
+        var finalPrompt = ChatHelper.GetFinalPrompt(lastMsg, model, isNewConversation);
 
         if (isNewConversation)
         {
             var systemMsg = chat.Messages.FirstOrDefault(x => x.Role == nameof(AuthorRole.System));
-            var template = new LLamaTemplate(llmModel);
-            var finalPrompt = ChatHelper.GetFinalPrompt(lastMsg, model, true);
-
             if (systemMsg != null)
             {
                 template.Add(systemMsg.Role, systemMsg.Content);
             }
-
-            template.Add(ServiceConstants.Roles.User, finalPrompt);
-            template.AddAssistant = true;
-
-            var templatedMessage = Encoding.UTF8.GetString(template.Apply());
-            conversation = executor.Create();
-            conversation.Prompt(executor.Context.Tokenize(templatedMessage, addBos: true, special: true));
         }
-        else
-        {
-            conversation = executor.Load(chat.ConversationState!);
-            var template = new LLamaTemplate(llmModel);
-            var finalPrompt = ChatHelper.GetFinalPrompt(lastMsg, model, false);
 
-            template.Add(ServiceConstants.Roles.User, finalPrompt);
-            template.AddAssistant = true;
+        template.Add(ServiceConstants.Roles.User, finalPrompt);
+        template.AddAssistant = true;
 
-            var templatedMessage = Encoding.UTF8.GetString(template.Apply());
-            conversation.Prompt(executor.Context.Tokenize(templatedMessage));
-        }
+        var templatedMessage = Encoding.UTF8.GetString(template.Apply());
+        var tokens = isNewConversation
+            ? executor.Context.Tokenize(templatedMessage, addBos: true, special: true)
+            : executor.Context.Tokenize(templatedMessage);
 
-        return (conversation, false, false);
+        conversation.Prompt(tokens);
     }
 
     private async Task<(List<LLMTokenValue> Tokens, bool IsComplete, bool HasFailed)> ProcessTokens(
@@ -340,7 +373,7 @@ private string GetModelsPath()
 
         return path;
     }
-    
+
     private async Task<ChatResult> CreateChatResult(Chat chat, List<LLMTokenValue> tokens,
         ChatRequestOptions requestOptions)
     {
@@ -375,4 +408,4 @@ await notificationService.DispatchNotification(
             NotificationMessageBuilder.CreateChatCompletion(chatId, token, isComplete),
             ServiceConstants.Notifications.ReceiveMessageUpdate);
     }
-}
+}
diff --git a/src/MaIN.Services/Services/Steps/Commands/RedirectCommandHandler.cs b/src/MaIN.Services/Services/Steps/Commands/RedirectCommandHandler.cs

Original file line number	Diff line number	Diff line change
`@@ -51,6 +51,6 @@ Generate image based on given prompt`
`51`	`51`	`}]`
`52`	`52`	`});`
`53`	`53`
`54`		`- ImagePreview.ShowImage(result.Message.Images);`
	`54`	`+ ImagePreview.ShowImage(result.Message.Image);`
`55`	`55`	`}`
`56`	`56`	`}`
Original file line number	Diff line number	Diff line change
`@@ -14,6 +14,6 @@ public async Task Start()`
`14`	`14`	`.WithMessage("Generate cyberpunk godzilla cat warrior")`
`15`	`15`	`.CompleteAsync();`
`16`	`16`
`17`		`- ImagePreview.ShowImage(result.Message.Images);`
	`17`	`+ ImagePreview.ShowImage(result.Message.Image);`
`18`	`18`	`}`
`19`	`19`	`}`
Original file line number	Diff line number	Diff line change
`@@ -15,6 +15,6 @@ public async Task Start()`
`15`	`15`	`.WithMessage("Generate hamster as a astronaut on the moon")`
`16`	`16`	`.CompleteAsync();`
`17`	`17`
`18`		`- ImagePreview.ShowImage(result.Message.Images);`
	`18`	`+ ImagePreview.ShowImage(result.Message.Image);`
`19`	`19`	`}`
`20`	`20`	`}`
Original file line number	Diff line number	Diff line change
`@@ -98,7 +98,7 @@ public async Task Should_GenerateImage_BasedOnPrompt()`
`98`	`98`	`throw new ArgumentException("Invalid file extension");`
`99`	`99`
`100`	`100`	`Assert.True(result.Done);`
`101`		`- Assert.NotNull(result.Message.Images);`
	`101`	`+ Assert.NotNull(result.Message.Image);`
`102`	`102`	`}`
`103`	`103`
`104`	`104`	`[Fact]`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+# 0.2.5 release`
	`2`	`+`
	`3`	`+- Integrate with vision llava models`
Original file line number	Diff line number	Diff line change
`@@ -114,9 +114,9 @@ public AgentContext WithMemoryParams(MemoryParams memoryParams)`
`114`	`114`	`return this;`
`115`	`115`	`}`
`116`	`116`
`117`		`- public AgentContext WithCustomModel(string model, string path)`
	`117`	`+ public AgentContext WithCustomModel(string model, string path, string? mmProject = null)`
`118`	`118`	`{`
`119`		`- KnownModels.AddModel(model, path);`
	`119`	`+ KnownModels.AddModel(model, path, mmProject);`
`120`	`120`	`_agent.Model = model;`
`121`	`121`	`return this;`
`122`	`122`	`}`