Fix vision for local models

Author: Madzionator

src/MaIN.Core/MaIN.Core.csproj

@@ -8,7 +8,7 @@
 
     <ItemGroup>
         <PackageReference Include="AsyncKeyedLock" Version="8.0.2" />
-        <PackageReference Include="LLamaSharp.Backend.Cuda12" Version="0.25.0" />
+        <PackageReference Include="LLamaSharp.Backend.Cuda12" Version="0.26.0" />
         <PackageReference Include="Tesseract.Data.English" Version="4.0.0" />
     </ItemGroup>
 

src/MaIN.Domain/MaIN.Domain.csproj

@@ -8,7 +8,7 @@
     </PropertyGroup>
 
     <ItemGroup>
-      <PackageReference Include="LLamaSharp" Version="0.25.0" />
+      <PackageReference Include="LLamaSharp" Version="0.26.0" />
     </ItemGroup>
 
 </Project>

src/MaIN.Services/MaIN.Services.csproj

@@ -15,9 +15,9 @@
     <ItemGroup>
       <PackageReference Include="GTranslate" Version="2.3.1" />
       <PackageReference Include="HtmlAgilityPack" Version="1.12.4" />
-      <PackageReference Include="llamasharp.backend.cpu" Version="0.25.0" />
-      <PackageReference Include="LLamaSharp.Backend.Cuda12" Version="0.25.0" />
-      <PackageReference Include="LLamaSharp.kernel-memory" Version="0.25.0" />
+      <PackageReference Include="llamasharp.backend.cpu" Version="0.26.0" />
+      <PackageReference Include="LLamaSharp.Backend.Cuda12" Version="0.26.0" />
+      <PackageReference Include="LLamaSharp.kernel-memory" Version="0.26.0" />
       <PackageReference Include="Microsoft.Extensions.Http" Version="10.0.1" />
       <PackageReference Include="Microsoft.KernelMemory" Version="0.98.250508.3" />
       <PackageReference Include="Microsoft.SemanticKernel.Connectors.Google" Version="1.64.0-alpha" />

src/MaIN.Services/Services/LLMService/LLMService.cs

@@ -239,14 +239,19 @@ private async Task<List<LLMTokenValue>> ProcessChatRequest(
         var visionModel = model as IVisionModel;
         var mmProjName = visionModel?.MMProjectName
             ?? (chat.Properties.TryGetValue(ServiceConstants.Properties.MmProjNameProperty, out var p) ? p : null);
-        var llavaWeights = mmProjName is not null
-            ? await LLavaWeights.LoadFromFileAsync(ResolvePath(null, mmProjName), cancellationToken)
-            : null;
-        
-        using var executor = new BatchedExecutor(llmModel, parameters);
+
+        MtmdWeights? mtmdWeights = null;
+        if (mmProjName is not null && lastMsg.Image != null)
+        {
+            var mmProjPath = ResolvePath(null, mmProjName);
+            mtmdWeights = await MtmdWeights.LoadFromFileAsync(
+                mmProjPath, llmModel, MtmdContextParams.Default(), cancellationToken);
+        }
+
+        using var executor = new BatchedExecutor(llmModel, parameters, mtmdWeights);
 
         var (conversation, isComplete, hasFailed) = await LLMService.InitializeConversation(
-            chat, lastMsg, model, llmModel, llavaWeights, executor, cancellationToken);
+            chat, lastMsg, model, llmModel, mtmdWeights, executor, cancellationToken);
 
         if (!isComplete)
         {
@@ -271,6 +276,7 @@ private async Task<List<LLMTokenValue>> ProcessChatRequest(
             }
         }
 
+        mtmdWeights?.Dispose();
         return tokens;
     }
 
@@ -294,7 +300,7 @@ private ModelParams CreateModelParameters(Chat chat, string modelKey, string? cu
         Message lastMsg,
         LocalModel model,
         LLamaWeights llmModel,
-        LLavaWeights? llavaWeights,
+        MtmdWeights? mtmdWeights,
         BatchedExecutor executor,
         CancellationToken cancellationToken)
     {
@@ -303,9 +309,9 @@ private ModelParams CreateModelParameters(Chat chat, string modelKey, string? cu
             ? executor.Create()
             : executor.Load(chat.ConversationState!);
 
-        if (lastMsg.Image != null && llavaWeights != null)
+        if (lastMsg.Image != null && mtmdWeights != null)
         {
-            await ProcessImageMessage(conversation, lastMsg, llmModel, llavaWeights, executor, cancellationToken);
+            await ProcessImageMessage(conversation, lastMsg, mtmdWeights, executor, cancellationToken);
         }
         else
         {
@@ -317,21 +323,22 @@ private ModelParams CreateModelParameters(Chat chat, string modelKey, string? cu
 
     private static async Task ProcessImageMessage(Conversation conversation,
         Message lastMsg,
-        LLamaWeights llmModel,
-        LLavaWeights? llavaWeights,
+        MtmdWeights mtmdWeights,
         BatchedExecutor executor,
         CancellationToken cancellationToken)
     {
-        var imageEmbeddings = llavaWeights?.CreateImageEmbeddings(lastMsg.Image!);
-        conversation.Prompt(imageEmbeddings!);
+        using var imageEmbed = mtmdWeights.LoadMedia(lastMsg.Image!);
+
+        var mediaMarker = NativeApi.MtmdDefaultMarker() ?? "<image>";
+        conversation.Prompt(
+            $"USER: {mediaMarker}\n{lastMsg.Content}\nASSISTANT:",
+            new ReadOnlySpan<SafeMtmdEmbed>(new[] { imageEmbed }),
+            addBos: true);
 
         while (executor.BatchedTokenCount > 0)
         {
             await executor.Infer(cancellationToken);
         }
-
-        var prompt = llmModel.Tokenize($"USER: {lastMsg.Content}\nASSISTANT:", true, false, Encoding.UTF8);
-        conversation.Prompt(prompt);
     }
 
     private static void ProcessTextMessage(Conversation conversation,