refactor file handling to use dictionary for multiple files

Author: Krystian Moskal

Examples/Examples/Agents/AgentWithBecomeExample.cs

@@ -1,8 +1,6 @@
 using MaIN.Core.Hub;
 using MaIN.Core.Hub.Utils;
-using MaIN.Domain.Entities;
 using MaIN.Domain.Entities.Agents.AgentSource;
-using MaIN.Domain.Models;
 
 namespace Examples.Agents;
 
@@ -15,8 +13,10 @@ public async Task Start()
             .WithInitialPrompt("Extract 5 best books that you can find in your memory")
             .WithSource(new AgentFileSourceDetails()
             {
-                Path = "./Files/Books.json",
-                Name = "Books.json"
+                Files = new Dictionary<string, string>
+                {
+                    { "Books.json", "./Files/Books.json" }
+                },
             }, AgentSourceType.File)
             .WithBehaviour("SalesGod",
                 """

src/MaIN.Domain/Entities/Agents/AgentSource/AgentFileSourceDetails.cs

@@ -2,7 +2,6 @@ namespace MaIN.Domain.Entities.Agents.AgentSource;
 
 public class AgentFileSourceDetails : AgentSourceDetailsBase, IAgentSource
 {
-    public required string Path { get; init; }
-    public required string Name { get; init; }
+    public required Dictionary<string, string> Files { get; init; } = new();
     public bool PreProcess { get; init; } = false;
 }

src/MaIN.Services/Services/Abstract/IDataSourceProvider.cs

@@ -2,7 +2,7 @@ namespace MaIN.Services.Services.Abstract;
 
 public interface IDataSourceProvider
 {
-    Task<string> FetchFileData(object? sourceDetails);
+    Task<string> FetchFileData(Dictionary<string, string> source);
     string FetchTextData(object? sourceDetails);
     Task<string> FetchApiData(object? details, string? filter, IHttpClientFactory httpClientFactory, Dictionary<string, string> properties);
     Task<string> FetchSqlData(object? sourceDetails, string? filter, Dictionary<string, string> properties);

src/MaIN.Services/Services/DataSourceProvider.cs

@@ -13,12 +13,27 @@ public class DataSourceProvider : IDataSourceProvider
 {
     private static readonly JsonSerializerOptions JsonSerializerOptions = new() { PropertyNameCaseInsensitive = true };
 
-    public async Task<string> FetchFileData(object? sourceDetails)
+    public async Task<string> FetchFileData(Dictionary<string, string> source)
     {
-        var fileDetails = JsonSerializer.Deserialize<AgentFileSourceDetails>(
-            sourceDetails?.ToString() ?? "{}")!;
-
-        return await File.ReadAllTextAsync(fileDetails.Path);
+        var allContent = new StringBuilder();
+    
+        foreach (var (fileName, filePath) in source)
+        {
+            try
+            {
+                var content = await File.ReadAllTextAsync(filePath);
+                allContent.AppendLine($"=== {fileName} ===");
+                allContent.AppendLine(content);
+                allContent.AppendLine();
+            }
+            catch (Exception)
+            {
+                allContent.AppendLine($"=== Error reading {fileName} ===");
+                allContent.AppendLine();
+            }
+        }
+    
+        return allContent.ToString();
     }
 
     public string FetchTextData(object? sourceDetails)

src/MaIN.Services/Services/LLMService/ChatMemoryOptions.cs

@@ -3,7 +3,7 @@ namespace MaIN.Services.Services.LLMService;
 public class ChatMemoryOptions
 {
     public Dictionary<string, string>? TextData { get; set; }
-    public Dictionary<string, string>? FileData { get; set; }
+    public Dictionary<string, string>? FilesData { get; set; }
     public Dictionary<string, FileStream>? StreamData { get; set; }
     public List<string>? WebUrls { get; set; }
     public List<string>? Memory { get; set; }

src/MaIN.Services/Services/LLMService/Memory/MemoryService.cs

@@ -14,7 +14,7 @@ public async Task ImportDataToMemory(
             await PreprocessAvailableDocuments(options, cancellationToken);
         }
         await ImportTextData(memory, options.TextData, cancellationToken);
-        await ImportFileData(memory, options.FileData, cancellationToken);
+        await ImportFilesData(memory, options.FilesData, cancellationToken);
         await ImportStreamData(memory, options.StreamData, cancellationToken);
         await ImportWebUrls(memory, options.WebUrls, cancellationToken);
         await ImportMemoryItems(memory, options.Memory, cancellationToken);
@@ -39,7 +39,7 @@ private async Task ImportTextData(IKernelMemory memory, Dictionary<string, strin
         }
     }
 
-    private async Task ImportFileData(IKernelMemory memory, Dictionary<string, string>? fileData,
+    private async Task ImportFilesData(IKernelMemory memory, Dictionary<string, string>? fileData,
         CancellationToken cancellationToken)
     {
         if (fileData?.Any() != true)
@@ -92,10 +92,10 @@ await memory.ImportTextAsync(
 
     private static async Task PreprocessAvailableDocuments(ChatMemoryOptions options, CancellationToken cancellationToken)
     {
-        foreach (var file in options.FileData!)
+        foreach (var file in options.FilesData!)
         {
             options.TextData!.Add(file.Key ,DocumentProcessor.ProcessDocument(file.Value));
-            options.FileData = [];
+            options.FilesData = [];
         }
 
         foreach (var stream in options.StreamData!)

src/MaIN.Services/Services/LLMService/Utils/ChatHelper.cs

@@ -74,7 +74,7 @@ public static ChatMemoryOptions ExtractMemoryOptions(Message message)
         return new ChatMemoryOptions
         {
             TextData = textData,
-            FileData = fileData,
+            FilesData = fileData,
             StreamData = streamData,
             PreProcess = preProcess
         };

src/MaIN.Services/Services/Steps/Commands/FetchCommandHandler.cs

@@ -4,7 +4,6 @@
 using MaIN.Services.Services.Models.Commands;
 using System.Text.Json;
 using MaIN.Domain.Configuration;
-using MaIN.Services.Mappers;
 using MaIN.Services.Services.LLMService;
 using MaIN.Services.Services.LLMService.Factory;
 using MaIN.Services.Utils;
@@ -91,15 +90,15 @@ private async Task<Message> HandleFileSource(FetchCommand command, Dictionary<st
                     memoryChat!,
                     new ChatMemoryOptions
                     {
-                        FileData = new Dictionary<string, string> { { fileData!.Name, fileData.Path } },
+                        FilesData = fileData!.Files,
                         PreProcess = fileData.PreProcess
                     }
                 );
 
             return result!.Message;
         }
 
-        var data = await dataSourceService.FetchFileData(command.Context.Source.Details);
+        var data = await dataSourceService.FetchFileData(fileData!.Files);
         return CreateMessage(data, properties);
     }