Fix: Per-call LLama embedder; update Nomic model

Switch LLama embedding implementation to create and dispose contexts per call (aligns with LLamaSharp 0.26.0), removing the long-lived Context field and related state. Read EmbeddingSize from a temporary context at construction, call llama_set_embeddings on each per-call context, and normalize embeddings as before. Update LLamaSharpTextEmbedding defaults: use model-default context (ContextSize=0), enable Embeddings, reduce Batch/UBatch sizes, disable FlashAttention, and set pooling + metadata override for older Nomic GGUFs. Update KnownModels and LocalModels entries for the Nomic embedding model (filename, download URL, display name) and change its embedding dimension to 2048. Remove MemoryService's pre/post import context management and adjust MemoryFactory to load weights, inject pooling metadata, and return an embedding config that uses the model's native context.

Author: Madzionator

src/MaIN.Domain/Models/Concrete/LocalModels.cs

@@ -297,11 +297,11 @@ public sealed record Olmo2_7b() : LocalModel(
 
 public sealed record Nomic_Embedding() : LocalModel(
     "nomic-embedding",
-    "nomicv2.gguf",
-    new Uri("https://huggingface.co/Inza124/Nomic/resolve/main/nomicv2.gguf?download=true"),
-    "Nomic Embedding",
-    8192,
-    "Model used to generate embeddings");
+    "mxbai-embed-large-v1.Q4_K_M.gguf",
+    new Uri("https://huggingface.co/ChristianAzinn/mxbai-embed-large-v1-gguf/resolve/main/mxbai-embed-large-v1.Q4_K_M.gguf?download=true"),
+    "mxbai-embed-large v1",
+    512,
+    "Model used to generate embeddings with superior knowledge search recall");
 
 // ===== TTS Model =====
 

src/MaIN.Domain/Models/SupportedModels.cs

@@ -250,9 +250,9 @@ public static Model GetEmbeddingModel() =>
         new()
         {
             Name = KnownModelNames.Nomic_Embedding,
-            FileName = "nomicv2.gguf",
-            Description = "Model used to generate embeddings.",
-            DownloadUrl = "https://huggingface.co/Inza124/Nomic/resolve/main/nomicv2.gguf?download=true",
+            FileName = "mxbai-embed-large-v1.Q4_K_M.gguf",
+            Description = "Model used to generate embeddings with superior knowledge search recall.",
+            DownloadUrl = "https://huggingface.co/ChristianAzinn/mxbai-embed-large-v1-gguf/resolve/main/mxbai-embed-large-v1.Q4_K_M.gguf?download=true",
         };
 
     public static bool IsModelSupported(string name) =>

src/MaIN.Services/Services/LLMService/Memory/Embeddings/LLamaEmbedderMaINClone.cs

@@ -25,13 +25,7 @@ public sealed class LLamaEmbedderMaINClone
     /// <summary>
     /// Dimension of embedding vectors
     /// </summary>
-    public int EmbeddingSize => Context.EmbeddingSize;
-
-    /// <summary>
-    /// LLama Context
-    /// </summary>
-    public LLamaContext Context { get; set; }
-    public bool isContextDisposed { get; set; }
+    public int EmbeddingSize { get; }
 
     /// <summary>
     /// Create a new embedder, using the given LLamaWeights
@@ -46,17 +40,21 @@ public LLamaEmbedderMaINClone(LLamaWeights weights, IContextParams @params, ILog
         if (weights.NativeHandle is { HasEncoder: true, HasDecoder: true })
             throw new NotSupportedException("Computing embeddings in encoder-decoder models is not supported");
 
-        Context = weights.CreateContext(@params, logger);
+        // Create context only to read EmbeddingSize, then dispose immediately
+        // (matches LLamaSharp 0.26.0 LLamaEmbedder pattern)
+        using (var tempContext = weights.CreateContext(@params, logger))
+        {
+            EmbeddingSize = tempContext.EmbeddingSize;
+        }
+
         _weights = weights;
         _params = @params;
         _logger = logger;
-        NativeApi.llama_set_embeddings(Context.NativeHandle, true);
     }
 
     /// <inheritdoc />
     public void Dispose()
     {
-        Context.Dispose();
     }
 
     /// <summary>
@@ -74,22 +72,20 @@ public async Task<IReadOnlyList<float[]>> GetEmbeddings(string input, Cancellati
 
     private async Task<(IReadOnlyList<float[]> Embeddings, int Tokens)> GetEmbeddingsWithTokenCount(string input, CancellationToken cancellationToken = default)
     {
-        if (isContextDisposed)
-        {
-            Context = _weights.CreateContext(_params, _logger);
-            NativeApi.llama_set_embeddings(Context.NativeHandle, true);
-        }
-        
-        var tokens = Context.Tokenize(input, special: true);
-        if (tokens.Length > Context.ContextSize)
-            throw new ArgumentException($"Embedding prompt is longer than the context window ({tokens.Length} > {Context.ContextSize})", nameof(input));
+        // Create a fresh context for each embedding call (0.26.0 pattern)
+        using var context = _weights.CreateContext(_params, _logger);
+        NativeApi.llama_set_embeddings(context.NativeHandle, true);
+
+        var tokens = context.Tokenize(input, special: true);
+        if (tokens.Length > context.ContextSize)
+            throw new ArgumentException($"Embedding prompt is longer than the context window ({tokens.Length} > {context.ContextSize})", nameof(input));
 
         cancellationToken.ThrowIfCancellationRequested();
 
         // Evaluate prompt in batch-size chunks
         var n_past = 0;
         var batch = new LLamaBatch();
-        var batchSize = (int)Context.Params.BatchSize;
+        var batchSize = (int)context.Params.BatchSize;
         for (var i = 0; i < tokens.Length; i += batchSize)
         {
             var n_eval = tokens.Length - i;
@@ -101,19 +97,19 @@ public async Task<IReadOnlyList<float[]>> GetEmbeddings(string input, Cancellati
             n_past += n_eval;
 
             // Run model
-            switch (Context.NativeHandle.ModelHandle.HasEncoder, Context.NativeHandle.ModelHandle.HasDecoder)
+            switch (context.NativeHandle.ModelHandle.HasEncoder, context.NativeHandle.ModelHandle.HasDecoder)
             {
                 case (true, false):
                     {
-                        var result = await Context.EncodeAsync(batch, cancellationToken);
+                        var result = await context.EncodeAsync(batch, cancellationToken);
                         if (result != EncodeResult.Ok)
                             throw new RuntimeError($"Failed to encode: {result}");
                         break;
                     }
 
                 case (false, true):
                     {
-                        var result = await Context.DecodeAsync(batch, cancellationToken);
+                        var result = await context.DecodeAsync(batch, cancellationToken);
                         if (result != DecodeResult.Ok)
                             throw new RuntimeError($"Failed to decode: {result}");
                         break;
@@ -125,18 +121,17 @@ public async Task<IReadOnlyList<float[]>> GetEmbeddings(string input, Cancellati
         }
 
         // Extract results
-        var poolingType = Context.NativeHandle.PoolingType;
+        var poolingType = context.NativeHandle.PoolingType;
         var resultsCount = poolingType == LLamaPoolingType.None ? tokens.Length : 1;
         var results = new List<float[]>(resultsCount);
-        results.Add(Context.NativeHandle.GetEmbeddingsSeq(LLamaSeqId.Zero).ToArray());
+        results.Add(context.NativeHandle.GetEmbeddingsSeq(LLamaSeqId.Zero).ToArray());
 
         // Normalize the embeddings vector
-        // https://github.com/ggerganov/llama.cpp/blob/2891c8aa9af17f4ff636ff3868bc34ff72b56e25/examples/embedding/embedding.cpp#L92
         foreach (var embedding in results)
         {
             embedding.EuclideanNormalization();
         }
-        
+
         return (results, tokens.Length);
     }
-}
+}

src/MaIN.Services/Services/LLMService/Memory/Embeddings/LLamaSharpTextEmbeddingMaINClone.cs

@@ -31,17 +31,18 @@ public sealed class LLamaSharpTextEmbeddingMaINClone
         /// <param name="config">The configuration for LLamaSharp.</param>
         public LLamaSharpTextEmbeddingMaINClone(LLamaSharpConfig config)
         {
-            MaxTokens = (int?)config.ContextSize ?? 2048;
+            MaxTokens = (int)(config.ContextSize is > 0 ? config.ContextSize.Value : 2048);
 
             @params = new ModelParams(config.ModelPath)
             {
-                ContextSize = config?.ContextSize ?? 2048,
+                ContextSize = config?.ContextSize ?? 0, // 0 = use model default
                 GpuLayerCount = config?.GpuLayerCount ?? 20,
                 MainGpu = config?.MainGpu ?? 0,
                 SplitMode = config?.SplitMode ?? LLama.Native.GPUSplitMode.Layer,
-                BatchSize = 1024,
-                UBatchSize = 1024,
-                FlashAttention = true,
+                BatchSize = 512,
+                UBatchSize = 512,
+                Embeddings = true,
+                FlashAttention = false,
                 UseMemorymap = true,
                 PoolingType = LLamaPoolingType.Mean,
             };
@@ -59,17 +60,18 @@ public LLamaSharpTextEmbeddingMaINClone(LLamaSharpConfig config)
         /// <param name="weights">A LLamaWeights object.</param>
         public LLamaSharpTextEmbeddingMaINClone(LLamaSharpConfig config, LLamaWeights weights)
         {
-            MaxTokens = (int?)config.ContextSize ?? 2048;
+            MaxTokens = (int)(config.ContextSize is > 0 ? config.ContextSize.Value : 2048);
 
             @params = new ModelParams(config.ModelPath)
             {
-                ContextSize = config?.ContextSize ?? 2048,
+                ContextSize = config?.ContextSize ?? 0, // 0 = use model default
                 GpuLayerCount = config?.GpuLayerCount ?? 20,
                 MainGpu = config?.MainGpu ?? 0,
                 SplitMode = config?.SplitMode ?? LLama.Native.GPUSplitMode.Layer,
-                BatchSize = 1024,
-                UBatchSize = 1024,
-                FlashAttention = true,
+                BatchSize = 512,
+                UBatchSize = 512,
+                Embeddings = true,
+                FlashAttention = false,
                 UseMemorymap = true,
                 PoolingType = LLamaPoolingType.Mean,
             };

src/MaIN.Services/Services/LLMService/Memory/MemoryFactory.cs

@@ -1,6 +1,8 @@
 using System.Diagnostics.CodeAnalysis;
 using LLama;
 using LLama.Common;
+using LLama.Abstractions;
+using LLama.Native;
 using LLamaSharp.KernelMemory;
 using MaIN.Domain.Entities;
 using MaIN.Domain.Exceptions.Models;
@@ -108,11 +110,22 @@ private static LLamaSharpTextEmbeddingMaINClone ConfigureGeneratorOptions(string
 
         var parameters = new ModelParams(config.ModelPath)
         {
-            ContextSize = new uint?(config.ContextSize.GetValueOrDefault(2048U)),
+            ContextSize = 0, // let the model decide (mxbai-embed-large-v1 = 512)
             GpuLayerCount = config.GpuLayerCount.GetValueOrDefault(20),
+            Embeddings = true,
+            UseMemorymap = true,
+            PoolingType = LLamaPoolingType.CLS,
         };
 
         var weights = LLamaWeights.LoadFromFile(parameters);
+
+        // Override config context size for embedding — use model's native context
+        config = new LLamaSharpConfig(desiredPath)
+        {
+            DefaultInferenceParams = inferenceParams,
+            GpuLayerCount = memoryParams.GpuLayerCount,
+            ContextSize = 0,
+        };
         return new LLamaSharpTextEmbeddingMaINClone(config, weights);
     }
 
@@ -131,8 +144,8 @@ private static TextPartitioningOptions ConfigureParsingOptions()
     {
         return new TextPartitioningOptions
         {
-            MaxTokensPerParagraph = 512,
-            OverlappingTokens = 30,
+            MaxTokensPerParagraph = 400,
+            OverlappingTokens = 20,
         };
     }
 

src/MaIN.Services/Services/LLMService/Memory/MemoryService.cs

@@ -1,5 +1,3 @@
-using LLama.Native;
-using MaIN.Services.Services.LLMService.Memory.Embeddings;
 using MaIN.Services.Utils;
 using Microsoft.KernelMemory;
 using Microsoft.KernelMemory.AI;
@@ -22,7 +20,7 @@ public async Task ImportDataToMemory((IKernelMemory km, ITextEmbeddingGenerator?
         await ImportWebUrls(memory, options.WebUrls, cancellationToken);
         await ImportMemoryItems(memory, options.Memory, cancellationToken);
     }
-    
+
     public string CleanResponseText(string text)
     {
         return text
@@ -38,10 +36,8 @@ private async Task ImportTextData((IKernelMemory km, ITextEmbeddingGenerator? ge
 
         foreach (var item in textData)
         {
-            PreImport(memory.generator);
             var cleanedValue = JsonCleaner.CleanAndUnescape(item.Value);
             await memory.km.ImportTextAsync(cleanedValue!, item.Key, cancellationToken: cancellationToken);
-            PostImport(memory.generator);
         }
     }
 
@@ -51,15 +47,11 @@ private async Task ImportFilesData((IKernelMemory km, ITextEmbeddingGenerator? g
         if (fileData?.Any() != true)
             return;
 
-        
         foreach (var item in fileData)
         {
-            PreImport(memory.generator);
             await memory.km.ImportDocumentAsync(item.Value, item.Key, cancellationToken: cancellationToken);
-            PostImport(memory.generator);
         }
     }
-    
 
     private async Task ImportStreamData((IKernelMemory km, ITextEmbeddingGenerator? generator) memory, Dictionary<string, Stream>? streamData,
         CancellationToken cancellationToken)
@@ -69,9 +61,7 @@ private async Task ImportStreamData((IKernelMemory km, ITextEmbeddingGenerator?
 
         foreach (var item in streamData)
         {
-            PreImport(memory.generator);
             await memory.km.ImportDocumentAsync(item.Value, item.Key, cancellationToken: cancellationToken);
-            PostImport(memory.generator);
         }
     }
 
@@ -82,9 +72,7 @@ private async Task ImportWebUrls((IKernelMemory km, ITextEmbeddingGenerator? gen
 
         foreach (var item in webUrls)
         {
-            PreImport(memory.generator);
             await memory.km.ImportWebPageAsync(item, cancellationToken: cancellationToken);
-            PostImport(memory.generator);
         }
     }
 
@@ -97,15 +85,13 @@ private async Task ImportMemoryItems((IKernelMemory km, ITextEmbeddingGenerator?
 
         foreach (var item in memoryItems.Select((value, i) => (value, i)))
         {
-            PreImport(memory.generator);
             await memory.km.ImportTextAsync(
                 item.value,
                 $"ANSWER_MEMORY_{item.i + 1}-{memoryItems.Count}",
                 cancellationToken: cancellationToken);
-            PostImport(memory.generator);
         }
     }
-    
+
     private static async Task PreprocessAvailableDocuments(ChatMemoryOptions options, CancellationToken cancellationToken)
     {
         foreach (var file in options.FilesData!)
@@ -123,28 +109,4 @@ private static async Task PreprocessAvailableDocuments(ChatMemoryOptions options
             options.StreamData = [];
         }
     }
-    
-    private void PostImport(ITextEmbeddingGenerator? memoryGenerator)
-    {
-        if (memoryGenerator is LLamaSharpTextEmbeddingMaINClone llamaGenerator)
-        {
-            llamaGenerator._embedder.Context.Dispose();
-            llamaGenerator._embedder.isContextDisposed = true;
-        }
-    }
-
-    private void PreImport(ITextEmbeddingGenerator? memoryGenerator)
-    {
-        if (memoryGenerator is LLamaSharpTextEmbeddingMaINClone { _embedder.isContextDisposed: true } llamaGenerator)
-        {
-            llamaGenerator._embedder.Context = llamaGenerator
-                ._embedder
-                ._weights
-                .CreateContext(llamaGenerator.@params!);
-            llamaGenerator._embedder.isContextDisposed = false;
-            NativeApi.llama_set_embeddings(llamaGenerator._embedder.Context.NativeHandle, true);
-
-        }
-    }
-
-}
+}