Support MMProj files & extract image uploads

Add support for multimodal projector (.mmproj) names across the stack and ensure uploaded image files are treated as images. Implemented MMProjectName on several local vision models and added MmProjName property to InferPage settings, backend profiles, Utils, and ServiceConstants. Settings UI now exposes an MMProj File input for local unregistered vision models and saves/loads it per backend. LLMService now extracts image files from message.Files early (ChatHelper.ExtractImageFromFiles), resolves mmproj name from the model or chat properties to load LLava weights, and requires loaded weights before processing image messages. ChatHelper.ExtractImageFromFiles moves image files into message.Images and cleans up Files so they aren't misrouted to RAG/memory.

Author: Madzionator

src/MaIN.Domain/Models/Concrete/LocalModels.cs

@@ -18,23 +18,32 @@ public sealed record Gemma3_4b() : LocalModel(
     new Uri("https://huggingface.co/Inza124/Gemma3-4b/resolve/main/gemma3-4b.gguf?download=true"),
     "Gemma3 4B",
     8192,
-    "Balanced 4B model for writing, analysis, and mathematical reasoning");
+    "Balanced 4B model for writing, analysis, and mathematical reasoning"), IVisionModel
+{
+    public string MMProjectName => "mmproj-model-gemma3-4b.gguf";
+}
 
 public sealed record Gemma3_12b() : LocalModel(
     "gemma3-12b",
     "Gemma3-12b.gguf",
     new Uri("https://huggingface.co/Inza124/Gemma3-12b/resolve/main/gemma3-12b.gguf?download=true"),
     "Gemma3 12B",
     8192,
-    "Large 12B model for complex analysis, research, and creative writing");
+    "Large 12B model for complex analysis, research, and creative writing"), IVisionModel
+{
+    public string MMProjectName => "mmproj-model-gemma3-12b.gguf";
+}
 
 public sealed record Gemma3n_e4b() : LocalModel(
     "gemma3n-e4b",
     "Gemma3n-e4b.gguf",
     new Uri("https://huggingface.co/Inza124/Gemma-3n-e4b/resolve/main/gemma-3n-e4b.gguf?download=true"),
     "Gemma3n E4B",
     8192,
-    "Compact 4B model optimized for efficient reasoning and general-purpose tasks");
+    "Compact 4B model optimized for efficient reasoning and general-purpose tasks"), IVisionModel
+{
+    public string MMProjectName => "mmproj-model-gemma3n-e4b.gguf";
+}
 
 // ===== Llama Family =====
 

src/MaIN.InferPage/Components/Pages/Home.razor

@@ -12,6 +12,8 @@
 @using MaIN.Domain.Exceptions
 @using MaIN.Domain.Models
 @using MaIN.Domain.Models.Abstract
+@using MaIN.Services
+@using MaIN.Services.Constants
 @using Markdig
 @using Message = MaIN.Domain.Entities.Message
 @using MessageType = MaIN.Domain.Entities.MessageType
@@ -396,6 +398,7 @@
             settings.HasVision,
             settings.HasReasoning,
             settings.HasImageGen,
+            settings.MmProjName,
             MaINSettings,
             apiKey);
     }
@@ -592,6 +595,11 @@
             Chat.ModelId = Utils.Model!;
             Chat.ImageGen = Utils.ImageGen;
 
+            // Sync mmproj setting for local unregistered vision models
+            Chat.Properties.Remove(ServiceConstants.Properties.MmProjNameProperty);
+            if (Utils.BackendType == BackendType.Self && !string.IsNullOrEmpty(Utils.MmProjName))
+                Chat.Properties.AddProperty(ServiceConstants.Properties.MmProjNameProperty, Utils.MmProjName);
+
             bool wasAtBottom = await JS.InvokeAsync<bool>("scrollManager.isAtBottom", "messages-container");
 
             StateHasChanged();

src/MaIN.InferPage/Components/Pages/Settings.razor

@@ -115,6 +115,17 @@
                     }
                 </div>
             }
+
+            @if (_selectedBackend?.BackendType == BackendType.Self && !_isRegisteredModel && _manualVision)
+            {
+                <div class="settings-field">
+                    <label class="settings-label">MMProj File</label>
+                    <FluentTextField @bind-Value="_mmProjName"
+                                     Placeholder="e.g., mmproj-model-f16.gguf"
+                                     Style="width: 100%;" />
+                    <span class="api-key-hint">Multimodal projector file (must be in the models folder)</span>
+                </div>
+            }
         </div>
 
         <div class="settings-footer">
@@ -194,6 +205,7 @@
     private bool _manualVision;
     private bool _manualReasoning;
     private bool _manualImageGen;
+    private string? _mmProjName;
 
     private bool RequiresApiKey => _selectedBackend?.RequiresApiKey == true;
     private bool CanSave => !string.IsNullOrWhiteSpace(_modelName)
@@ -251,6 +263,7 @@
             _manualVision = settings.HasVision;
             _manualReasoning = settings.HasReasoning;
             _manualImageGen = settings.HasImageGen;
+            _mmProjName = settings.MmProjName;
 
             OnModelNameChanged();
         }
@@ -288,6 +301,7 @@
                 _manualVision    = profile.Vision;
                 _manualReasoning = profile.Reasoning;
                 _manualImageGen  = profile.ImageGen;
+                _mmProjName      = profile.MmProjName;
             }
         }
         else
@@ -382,12 +396,13 @@
             HasVision = hasVision,
             HasReasoning = hasReasoning,
             HasImageGen = hasImageGen,
-            ModelPath = _modelPath
+            ModelPath = _modelPath,
+            MmProjName = _mmProjName
         };
         await SettingsStorage.SaveSettingsAsync(settings);
 
         await SettingsStorage.SaveProfileForBackendAsync(
-            backendKey, _modelName, hasVision, hasReasoning, hasImageGen);
+            backendKey, _modelName, hasVision, hasReasoning, hasImageGen, _mmProjName);
 
         // Resolve API key: use new input, or fall back to saved key
         string? apiKey = null;
@@ -411,6 +426,7 @@
             hasVision,
             hasReasoning,
             hasImageGen,
+            _mmProjName,
             MaINSettings,
             apiKey);
 

src/MaIN.InferPage/Services/InferPageSettings.cs

@@ -9,4 +9,5 @@ public class InferPageSettings
     public bool HasReasoning { get; set; }
     public bool HasImageGen { get; set; }
     public string? ModelPath { get; set; }
+    public string? MmProjName { get; set; }
 }

src/MaIN.InferPage/Services/SettingsService.cs

@@ -26,11 +26,11 @@ public async Task<bool> HasSettingsAsync()
     private const string BackendProfilesKey = "inferpage-backend-profiles";
 
     public async Task SaveProfileForBackendAsync(string backend, string model,
-        bool vision, bool reasoning, bool imageGen)
+        bool vision, bool reasoning, bool imageGen, string? mmProjName = null)
     {
         var profiles = await js.InvokeAsync<Dictionary<string, BackendProfile>?>(
             "settingsManager.load", BackendProfilesKey) ?? new();
-        profiles[backend] = new BackendProfile(model, vision, reasoning, imageGen);
+        profiles[backend] = new BackendProfile(model, vision, reasoning, imageGen, mmProjName);
         await js.InvokeVoidAsync("settingsManager.save", BackendProfilesKey, profiles);
     }
 
@@ -55,4 +55,4 @@ private async Task<Dictionary<string, string>> LoadDictAsync(string storageKey)
         => await js.InvokeAsync<Dictionary<string, string>?>("settingsManager.load", storageKey) ?? new();
 }
 
-public record BackendProfile(string Model, bool Vision, bool Reasoning, bool ImageGen);
+public record BackendProfile(string Model, bool Vision, bool Reasoning, bool ImageGen, string? MmProjName = null);

src/MaIN.InferPage/Utils.cs

@@ -19,6 +19,7 @@ public static class Utils
     public static bool? ManualVision { get; set; }
     public static bool? ManualReasoning { get; set; }
     public static bool? ManualImageGen { get; set; }
+    public static string? MmProjName { get; set; }
 
     // registry → manual override → fallback set (null = no fallback)
     private static bool GetCapability<T>(bool? manual, HashSet<string>? fallback = null)
@@ -42,6 +43,7 @@ public static void ApplySettings(
         bool hasVision,
         bool hasReasoning,
         bool hasImageGen,
+        string? mmProjName,
         MaINSettings mainSettings,
         string? apiKey)
     {
@@ -69,6 +71,7 @@ public static void ApplySettings(
         ManualVision = hasVision;
         ManualReasoning = hasReasoning;
         ManualImageGen = hasImageGen;
+        MmProjName = string.IsNullOrWhiteSpace(mmProjName) ? null : mmProjName.Trim();
 
         mainSettings.BackendType = backendType;
 

src/MaIN.Services/Constants/ServiceConstants.cs

@@ -62,6 +62,7 @@ public static class Properties
         public const string PreProcessProperty = "Pre_Process";
         public const string DisableCacheProperty = "DisableCache";
         public const string AgentIdProperty = "AgentId";
+        public const string MmProjNameProperty = "MmProjName";
     }
 
     public static class Defaults

src/MaIN.Services/Services/LLMService/LLMService.cs

@@ -62,6 +62,8 @@ public LLMService(
 
         var lastMsg = chat.Messages.Last();
 
+        await ChatHelper.ExtractImageFromFiles(lastMsg);
+
         if (ChatHelper.HasFiles(lastMsg))
         {
             var memoryOptions = ChatHelper.ExtractMemoryOptions(lastMsg);
@@ -235,8 +237,10 @@ private async Task<List<LLMTokenValue>> ProcessChatRequest(
             : await ModelLoader.GetOrLoadModelAsync(modelsPath, modelKey);
 
         var visionModel = model as IVisionModel;
-        var llavaWeights = visionModel?.MMProjectName is not null
-            ? await LLavaWeights.LoadFromFileAsync(ResolvePath(null, visionModel.MMProjectName), cancellationToken)
+        var mmProjName = visionModel?.MMProjectName
+            ?? (chat.Properties.TryGetValue(ServiceConstants.Properties.MmProjNameProperty, out var p) ? p : null);
+        var llavaWeights = mmProjName is not null
+            ? await LLavaWeights.LoadFromFileAsync(ResolvePath(null, mmProjName), cancellationToken)
             : null;
 
         using var executor = new BatchedExecutor(llmModel, parameters);
@@ -299,7 +303,7 @@ private ModelParams CreateModelParameters(Chat chat, string modelKey, string? cu
             ? executor.Create()
             : executor.Load(chat.ConversationState!);
 
-        if (lastMsg.Image != null)
+        if (lastMsg.Image != null && llavaWeights != null)
         {
             await ProcessImageMessage(conversation, lastMsg, llmModel, llavaWeights, executor, cancellationToken);
         }

src/MaIN.Services/Services/LLMService/Utils/ChatHelper.cs

@@ -13,6 +13,53 @@ namespace MaIN.Services.Services.LLMService.Utils;
 /// </summary>
 public static class ChatHelper
 {
+    private static readonly HashSet<string> ImageExtensions =
+    [
+        ".jpg", ".jpeg", ".png", ".gif", ".webp",
+        ".bmp", ".tiff", ".tif", ".heic", ".heif", ".avif"
+    ];
+
+    /// <summary>
+    /// Extracts image files from message.Files into message.Images and removes them from Files.
+    /// This must be called before HasFiles() so images are not mistakenly routed to the RAG/memory path.
+    /// </summary>
+    public static async Task ExtractImageFromFiles(Message message)
+    {
+        if (message.Files == null || message.Files.Count == 0)
+            return;
+
+        var imageFiles = message.Files
+            .Where(f => ImageExtensions.Contains(f.Extension.ToLowerInvariant()))
+            .ToList();
+
+        if (imageFiles.Count == 0)
+            return;
+
+        var imageBytesList = new List<byte[]>();
+        foreach (var imageFile in imageFiles)
+        {
+            if (imageFile.StreamContent != null)
+            {
+                using var ms = new MemoryStream();
+                imageFile.StreamContent.Position = 0;
+                await imageFile.StreamContent.CopyToAsync(ms);
+                imageBytesList.Add(ms.ToArray());
+            }
+            else if (imageFile.Path != null)
+            {
+                imageBytesList.Add(await File.ReadAllBytesAsync(imageFile.Path));
+            }
+
+            message.Files.Remove(imageFile);
+        }
+
+        message.Images = imageBytesList;
+
+        if (message.Files.Count == 0)
+            message.Files = null;
+    }
+
+
     /// <summary>
     /// Generates final prompt including additional prompt if needed
     /// </summary>