feat: fin changes for CLI

Author: Piotr Stachaczynski

Examples/Examples/appsettings.json

@@ -7,8 +7,7 @@
   },
   "MaIN": {
     "ImageGenUrl": "http://localhost:5003",
-    "ModelsPath": "C:\\MAIN.Models",
-//    "SqliteSettings": {
+    //    "SqliteSettings": {
 //      "ConnectionString": "Data Source=Main_test.db"
 //    }
 },

mcli.ps1

@@ -8,6 +8,23 @@ param(
 
 # Set script root as a global variable for other scripts to use
 $global:MCLI_ROOT = $PSScriptRoot
+$modelsPath = $env:MaIN_ModelsPath
+function Get-ModelsMap {
+    $modelsMapFile = "$PSScriptRoot\models_map.txt"
+    if (-not (Test-Path $modelsMapFile)) {    
+        Write-Host "Models map file not found at $modelsMapFile. Please provide a valid file."    
+        return $null
+    }
+    
+    $modelsMap = @{}
+    Get-Content $modelsMapFile | ForEach-Object {    
+        $split = $_.Split("|")
+        $key = $split[0].Trim()
+        $value = $split[1].Trim()  
+        $modelsMap[$key] = $value
+    }
+    return $modelsMap
+}
 
 function Show-Usage {
     Write-Host @"
@@ -36,14 +53,15 @@ Options for 'api':
 
 Options for 'model':
     download <name>   Download a specific model
+    present          List installed models
     list             List available models
     update           Update all installed models
 
 Examples:
     mcli start-demo
     mcli start-demo --no-image-gen
     mcli api --hard
-    mcli model download llama2-7b
+    mcli model download gemma2-2b-maIN
     mcli help
 "@
 }
@@ -69,7 +87,7 @@ Options:
 Examples:
     mcli start-demo
     mcli start-demo --no-image-gen
-    mcli start-demo --models=llama3.2:3b,gemma2:2b
+    mcli start-demo --models=gemma2-2b-maIN
 "@
         }
         "api" {
@@ -108,11 +126,13 @@ Usage:
 Subcommands:
     download <name>   Download a specific model
     list             List available models
+    present          List installed models
     update           Update all installed models
 
 Examples:
     mcli model download llama2-7b
     mcli model list
+    mcli model present
     mcli model update
 "@
         }
@@ -142,8 +162,29 @@ switch ($command) {
                 & "$PSScriptRoot\download-models.ps1" $modelArgs[0]
             }
             "list" {
+                $modelsMap = Get-ModelsMap
+                if ($null -eq $modelsMap) { return }
+            
                 Write-Host "Available models:"
-                Get-Content "$PSScriptRoot\models_map.txt" | Where-Object { $_ -notmatch '^\s*#' -and $_ -notmatch '^\s*$' }
+                $modelsMap.Keys | Sort-Object | ForEach-Object {
+                    Write-Host "- $_"
+                }
+            }
+            "present" {
+                Write-Host "Downloaded models:"
+                Write-Host "Models path: $modelsPath"
+                $downloadedModels = Get-ChildItem -Path $modelsPath -Filter "*.gguf" | 
+                    Select-Object -ExpandProperty Name |
+                    ForEach-Object { $_ -replace '\.gguf$','' }
+                
+                if ($downloadedModels.Count -eq 0) {
+                    Write-Host "No models found in $modelsPath"
+                } 
+                else {
+                    $downloadedModels | Sort-Object | ForEach-Object {
+                        Write-Host "- $_"
+                    }
+                }
             }
             "update" {
                 Write-Host "Updating all installed models..."
@@ -170,4 +211,5 @@ switch ($command) {
             Write-Host "`nError: Unknown command '$command'" -ForegroundColor Red
         }
     }
-}
+}
+

src/MaIN.Core/Hub/Contexts/AgentContext.cs

@@ -1,6 +1,7 @@
 using MaIN.Domain.Entities;
 using MaIN.Domain.Entities.Agents;
 using MaIN.Domain.Entities.Agents.AgentSource;
+using MaIN.Domain.Models;
 using MaIN.Services.Mappers;
 using MaIN.Services.Models;
 using MaIN.Services.Models.Ollama;
@@ -63,7 +64,7 @@ public AgentContext WithSource(IAgentSource source, AgentSourceType type)
         };
         return this;
     }
-
+    
     public AgentContext WithName(string name)
     {
         _agent.Name = name;
@@ -76,6 +77,14 @@ public AgentContext WithModel(string model)
         return this;
     }
 
+    public AgentContext WithCustomModel(string model, string path)
+    {
+        KnownModels.AddModel(model, path);
+        _agent.Model = model;
+        return this;
+    }
+
+    
     public AgentContext WithInitialPrompt(string prompt)
     {
         _agent.Context.Instruction = prompt;
@@ -96,7 +105,6 @@ public AgentContext WithBehaviour(string name, string instruction)
         return this;
     }
 
-    // Creation and Processing
     public async Task<AgentContext> CreateAsync(bool flow = false, bool interactiveResponse = false)
     {
         await _agentService.CreateAgent(_agent, flow, interactiveResponse);
@@ -157,7 +165,6 @@ public async Task<ChatResult> ProcessAsync(Message message, bool translate = fal
         };
     }
 
-    // Chat Operations
     public async Task<Chat> GetChat()
     {
         return await _agentService.GetChatByAgent(_agent.Id);
@@ -168,7 +175,6 @@ public async Task<Chat> RestartChat()
         return await _agentService.Restart(_agent.Id);
     }
 
-    // Agent Management
     public async Task<List<Agent>> GetAllAgents()
     {
         return await _agentService.GetAgents();
@@ -184,7 +190,6 @@ public async Task<bool> Exists()
         return await _agentService.AgentExists(_agent.Id);
     }
 
-    // Static factory methods
     public static async Task<AgentContext> FromExisting(IAgentService agentService, string agentId)
     {
         var existingAgent = await agentService.GetAgentById(agentId);

src/MaIN.Core/Hub/Contexts/ChatContext.cs

@@ -1,4 +1,5 @@
 using MaIN.Domain.Entities;
+using MaIN.Domain.Models;
 using MaIN.Services.Models;
 using MaIN.Services.Models.Ollama;
 using MaIN.Services.Services.Abstract;
@@ -33,6 +34,13 @@ public ChatContext WithModel(string model)
         _chat.Model = model;
         return this;
     }
+    
+    public ChatContext WithCustomModel(string model, string path)
+    {
+        KnownModels.AddModel(model, path);
+        _chat.Model = model;
+        return this;
+    }
 
     public ChatContext WithMessage(string content)
     {

src/MaIN.Domain/Models/SupportedModels.cs

@@ -11,7 +11,7 @@ public struct Model
 
 public struct KnownModels
 {
-    internal static Dictionary<string, Model> Models => new()
+    internal static Dictionary<string, Model> Models => new(StringComparer.OrdinalIgnoreCase)
     {
         {
             KnownModelNames.Gemma2_2b, new Model()
@@ -130,7 +130,19 @@ public static Model GetModel(string path, string name)
         }
 
         throw new Exception($"Model {fileName} is not downloaded");
-    } 
+    }
+
+    public static void AddModel(string model, string path)
+    {
+        Models.Add(model, new Model()
+        {
+            Description = string.Empty,
+            DownloadUrl = string.Empty,
+            Name = model,
+            FileName = $"{Path.GetFileName(path)}",
+            Path = path
+        });
+    }
 }
 
 public struct KnownModelNames

src/MaIN.Services/Services/AgentService.cs

@@ -28,13 +28,15 @@ public AgentService(
         IChatRepository chatRepository,
         ILogger<AgentService> logger,
         INotificationService notificationService,
-        IStepProcessor stepProcessor)
+        IStepProcessor stepProcessor, 
+        ILLMService llmService)
     {
         _agentRepository = agentRepository;
         _chatRepository = chatRepository;
         _logger = logger;
         _notificationService = notificationService;
         _stepProcessor = stepProcessor;
+        _llmService = llmService;
     }
 
     public async Task<Chat> Process(Chat chat, string agentId, bool translatePrompt = false)

src/MaIN.Services/Services/LLMService/LLMService.cs

@@ -209,7 +209,7 @@ private void AddMessagesToHistory(ChatSession session, List<Message> messages)
         var model = KnownModels.GetModel(path, chat!.Model);
         var modelKey = model.FileName;
 
-        var kernelMemory = CreateMemory(modelKey, path);
+        var kernelMemory = CreateMemory(modelKey, path, out var generator);
 
         if (textData != null)
         {
@@ -251,13 +251,15 @@ private void AddMessagesToHistory(ChatSession session, List<Message> messages)
                 Role = AuthorRole.Assistant.ToString()
             }
         };
+        
+        generator.Dispose();
 
         return chatResult;
     }
 
 
     [Experimental("KMEXP01")]
-    private static IKernelMemory CreateMemory(string modelName, string path)
+    private static IKernelMemory CreateMemory(string modelName, string path, out KernelMemFix.LlamaSharpTextGenerator generator)
     {
         InferenceParams infParams = new() { AntiPrompts = ["INFO", "<|im_end|>", "Question:"] };
 
@@ -280,7 +282,7 @@ private static IKernelMemory CreateMemory(string modelName, string path)
 
         return new KernelMemoryBuilder()
             //.WithLLamaSharpDefaults2(lsConfig)
-            .WithLLamaSharpMaINTemp(lsConfig, Path.Combine(path, modelName))
+            .WithLLamaSharpMaINTemp(lsConfig, Path.Combine(path, modelName), out generator)
             .WithSearchClientConfig(searchClientConfig)
             .WithCustomImageOcr(new OcrWrapper())
             .With(parseOptions)
@@ -325,10 +327,10 @@ public Task CleanSessionCache(string id)
     }
 }
 
-file static class KernelMemFix
+internal static class KernelMemFix
 { 
     [Experimental("KMEXP00")]
-    public sealed class LlamaSharpTextGenerator2 : ITextGenerator, ITextTokenizer, IDisposable
+    public sealed class LlamaSharpTextGenerator : ITextGenerator, ITextTokenizer, IDisposable
   {
     private readonly StatelessExecutor _executor;
     private readonly LLamaWeights _weights;
@@ -339,22 +341,8 @@ public sealed class LlamaSharpTextGenerator2 : ITextGenerator, ITextTokenizer, I
 
     public int MaxTokenTotal { get; }
 
-    public LlamaSharpTextGenerator2(LLamaSharpConfig config)
-    {
-      ModelParams @params = new ModelParams(config.ModelPath)
-      {
-        ContextSize = new uint?(config.ContextSize.GetValueOrDefault(2048U)),
-        GpuLayerCount = config.GpuLayerCount.GetValueOrDefault(20)
-      };
-      this._weights = LLamaWeights.LoadFromFile((IModelParams) @params);
-      this._context = this._weights.CreateContext((IContextParams) @params);
-      this._executor = new StatelessExecutor(this._weights, (IContextParams) @params);
-      this._defaultInferenceParams = config.DefaultInferenceParams;
-      this._ownsWeights = this._ownsContext = true;
-      this.MaxTokenTotal = (int) @params.ContextSize.Value;
-    }
 
-    public LlamaSharpTextGenerator2(
+    public LlamaSharpTextGenerator(
       LLamaWeights weights,
       LLamaContext context,
       StatelessExecutor? executor = null,
@@ -431,45 +419,19 @@ public IReadOnlyList<string> GetTokens(string text)
   }
 
     [Experimental("KMEXP00")]
-    public static IKernelMemoryBuilder WithLLamaSharpTextGeneration2(
+    public static IKernelMemoryBuilder WithLLamaSharpTextGeneration(
         this IKernelMemoryBuilder builder,
-        LlamaSharpTextGenerator2 textGenerator)
+        LlamaSharpTextGenerator textGenerator)
     {
         builder.AddSingleton((ITextGenerator) textGenerator);
         return builder;
     }
 
-    [Experimental("KMEXP00")]
-    public static IKernelMemoryBuilder WithLLamaSharpDefaults2(
-        this IKernelMemoryBuilder builder,
-        LLamaSharpConfig config,
-        LLamaWeights? weights = null,
-        LLamaContext? context = null)
-    {
-        ModelParams @params = new ModelParams(config.ModelPath)
-        {
-            ContextSize = new uint?(config.ContextSize.GetValueOrDefault(2048U)),
-            GpuLayerCount = config.GpuLayerCount.GetValueOrDefault(20),
-            MainGpu = config.MainGpu,
-            SplitMode = new GPUSplitMode?(config.SplitMode)
-        };
-        if (weights == null || context == null)
-        {
-            weights = LLamaWeights.LoadFromFile((IModelParams) @params);
-            context = weights.CreateContext((IContextParams) @params);
-        }
-        StatelessExecutor executor = new StatelessExecutor(weights, (IContextParams) @params);
-        builder.WithLLamaSharpTextEmbeddingGeneration(new LLamaSharpTextEmbeddingGenerator(config, weights));
-        builder.WithLLamaSharpTextGeneration2(new LlamaSharpTextGenerator2(weights, context, executor,
-            config.DefaultInferenceParams));
-        return builder;
-    }
-    
     private static readonly ConcurrentDictionary<string, LLamaWeights> ModelCache = new();
 
     [Experimental("KMEXP01")]
     public static IKernelMemoryBuilder WithLLamaSharpMaINTemp(this IKernelMemoryBuilder builder,
-        LLamaSharpConfig config, string modelPath)
+        LLamaSharpConfig config, string modelPath, out LlamaSharpTextGenerator generator)
     {
         // Create ModelParams for the first model.
         var parameters1 = new ModelParams(modelPath)
@@ -495,26 +457,19 @@ public static IKernelMemoryBuilder WithLLamaSharpMaINTemp(this IKernelMemoryBuil
         var weights = GetOrLoadModel(parameters2);
 
         var context = model.CreateContext(parameters2);
-
         StatelessExecutor executor = new StatelessExecutor(model, parameters2);
+
+        generator = new LlamaSharpTextGenerator(model, context, executor,
+            config.DefaultInferenceParams);
+        
         builder.WithLLamaSharpTextEmbeddingGeneration(new LLamaSharpTextEmbeddingGenerator(config, weights));
-        builder.WithLLamaSharpTextGeneration2(new LlamaSharpTextGenerator2(model, context, executor,
-            config.DefaultInferenceParams));
+        builder.WithLLamaSharpTextGeneration(generator);
         return builder;
     }
 
     private static LLamaWeights GetOrLoadModel(ModelParams modelParams)
     {
-        // Use a unique key based on the serialized ModelParams object.
-        string cacheKey = GenerateCacheKey(modelParams);
-
-        // Retrieve from cache or load if not already cached.
-        return ModelCache.GetOrAdd(cacheKey, _ => LLamaWeights.LoadFromFile(modelParams));
+        return LLamaWeights.LoadFromFile(modelParams);
     }
 
-    private static string GenerateCacheKey(ModelParams modelParams)
-    {
-        // Create a unique key by combining important properties of ModelParams.
-        return $"{modelParams.ModelPath}:{modelParams.ContextSize}:{modelParams.GpuLayerCount}:{modelParams.MainGpu}:{modelParams.SplitMode}";
-    }
 }