Extend .WithSource to support APIs with API key authentication

src/MaIN.Domain/Entities/Agents/AgentSource/AgentAPISourceDetails.cs

@@ -8,4 +8,27 @@ public class AgentApiSourceDetails : AgentSourceDetailsBase, IAgentSource
     public string? Query { get; set; }
     public string? ResponseType { get; init; }
     public int? ChunkLimit { get; init; }
-}
+    public AuthTypeEnum? AuthorisationType { get; set; }
+    public string? AuthenticationToken { get; set; }
+    /// <summary>
+    /// Override payload and authorisation token
+    /// </summary>
+    public string? Curl { get; set; }
+    /// <summary>
+    /// Only use with BasicAuth
+    /// </summary>
+    public string? UserName { get; set; }
+    /// <summary>
+    /// Only use with BasicAuth
+    /// </summary>
+    public string? UserPassword { get; set; }
+}
+
+
+public enum AuthTypeEnum
+    {
+        Bearer = 0,
+        ApiKey = 1,
+        Basic = 2,
+
+    }

src/MaIN.Services/Services/DataSourceProvider.cs

@@ -1,11 +1,13 @@
-using System.Data.SqlClient;
-using System.Text;
-using System.Text.Json;
 using MaIN.Domain.Entities.Agents.AgentSource;
 using MaIN.Services.Services.Abstract;
 using MaIN.Services.Utils;
+using Microsoft.IdentityModel.Tokens;
 using MongoDB.Bson;
 using MongoDB.Driver;
+using System.Data.SqlClient;
+using System.Net.Http.Headers;
+using System.Text;
+using System.Text.Json;
 
 namespace MaIN.Services.Services;
 
@@ -61,18 +63,100 @@ public async Task<string> FetchApiData(object? details, string? filter,
         apiDetails.Query = apiDetails.Query?.Replace("@filter@", filter);
         apiDetails.Url = apiDetails.Url.Replace("@filter@", filter);
 
+
         var request = new HttpRequestMessage(
             HttpMethod.Parse(apiDetails?.Method),
             apiDetails?.Url + apiDetails?.Query);
 
-        if (!string.IsNullOrEmpty(apiDetails?.Payload))
+
+        if (!apiDetails.AuthenticationToken.IsNullOrEmpty())
+        {
+            if (!apiDetails.AuthorisationType.HasValue)
+                throw new InvalidOperationException("You need to specify an authorization type");
+
+            switch (apiDetails.AuthorisationType)
+            {
+                case AuthTypeEnum.Bearer:
+                    request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", apiDetails.AuthenticationToken);
+                    break;
+
+                case AuthTypeEnum.ApiKey:
+                    request.Headers.Authorization = new AuthenticationHeaderValue("ApiKey", apiDetails.AuthenticationToken);
+                    break;
+
+                case AuthTypeEnum.Basic:
+                    if (string.IsNullOrEmpty(apiDetails.UserName) || string.IsNullOrEmpty(apiDetails.UserPassword))
+                        throw new InvalidOperationException("Username and password are required for Basic Auth.");
+
+                    var credentials = $"{apiDetails.UserName}:{apiDetails.UserPassword}";
+                    var base64Credentials = Convert.ToBase64String(Encoding.UTF8.GetBytes(credentials));
+                    request.Headers.Authorization =
+                        new AuthenticationHeaderValue("Basic", base64Credentials);
+                    break;
+
+                default:
+                    throw new InvalidOperationException("Wrong Api Type");
+
+
+            }
+
+
+        }
+
+        if (!string.IsNullOrEmpty(apiDetails?.Curl))
         {
-            request.Content = new StringContent(
-                JsonSerializer.Serialize(apiDetails.Payload),
-                Encoding.UTF8,
-                "application/json");
+            CurlRequestParser.PopulateRequestFromCurl(request, apiDetails.Curl);
         }
+        else
+        {
+            if (!string.IsNullOrEmpty(apiDetails?.Payload))
+            {
 
+                var jsonString = apiDetails.Payload;
+
+                if (!(apiDetails.Payload is string))
+                {
+                    jsonString = JsonSerializer.Serialize(apiDetails.Payload);
+
+                }
+                else
+                {
+                    try
+                    {
+                        JsonDocument.Parse(jsonString);
+                    }
+                    catch (JsonException ex)
+                    {
+                        try
+                        {
+                            jsonString = JsonSerializer.Serialize(apiDetails.Payload);
+                            if (!jsonString.StartsWith('{'))
+                            {
+                                jsonString = $"{{{jsonString}}}";
+                            }
+                            jsonString = JsonCleaner.CleanAndUnescape(jsonString);
+
+                            JsonDocument.Parse(jsonString);
+                        }
+                        catch
+                        {
+                            throw new Exception($"Invalid JSON: {ex.Message}");
+                        }
+
+
+                    }
+                }
+
+                request.Content = new StringContent(
+                    jsonString,
+                    Encoding.UTF8,
+                    "application/json");
+
+
+            }
+
+
+        }
         var result = await httpClient.SendAsync(request);
         if (!result.IsSuccessStatusCode)
         {
@@ -81,7 +165,7 @@ public async Task<string> FetchApiData(object? details, string? filter,
         }
 
         var data = await result.Content.ReadAsStringAsync();
-
+        
         properties.TryAdd("api_response_type", apiDetails?.ResponseType ?? "JSON");
         if (apiDetails?.ChunkLimit != null)
         {

src/MaIN.Services/Services/LLMService/Memory/MemoryService.cs

@@ -1,4 +1,6 @@
+using MaIN.Services.Utils;
 using Microsoft.KernelMemory;
+using System.Text.Json;
 
 namespace MaIN.Services.Services.LLMService.Memory;
 
@@ -35,7 +37,8 @@ private async Task ImportTextData(IKernelMemory memory, Dictionary<string, strin
 
         foreach (var item in textData)
         {
-            await memory.ImportTextAsync(item.Value, item.Key, cancellationToken: cancellationToken);
+            var cleanedValue = JsonCleaner.CleanAndUnescape(item.Value);
+            await memory.ImportTextAsync(cleanedValue, item.Key, cancellationToken: cancellationToken);
         }
     }
 

src/MaIN.Services/Utils/CurlRequestParser.cs

@@ -0,0 +1,135 @@
+using System.Text;
+using System.Text.RegularExpressions;
+
+namespace MaIN.Services.Utils;
+public static class CurlRequestParser
+{
+    public static void PopulateRequestFromCurl(HttpRequestMessage request, string curlRaw)
+    {
+        if (request == null)
+            throw new ArgumentNullException(nameof(request));
+
+        if (string.IsNullOrWhiteSpace(curlRaw))
+            throw new ArgumentException("Curl string is empty");
+
+        string curl = Regex.Replace(curlRaw, @"\\\s*\n", " ");
+        curl = Regex.Replace(curl, @"\s{2,}", " ").Trim();
+
+        if (request.Method == null || request.Method == HttpMethod.Get)
+        {
+            var methodMatch = Regex.Match(curl, @"(?:-X|--request)\s+(\w+)", RegexOptions.IgnoreCase);
+            if (methodMatch.Success)
+            {
+                request.Method = new HttpMethod(methodMatch.Groups[1].Value.ToUpperInvariant());
+            }
+            else if (curl.Contains("--get"))
+            {
+                request.Method = HttpMethod.Get;
+            }
+            else
+            {
+                request.Method ??= HttpMethod.Get;
+            }
+        }
+
+        if (request.RequestUri == null)
+        {
+            var urlMatch = Regex.Match(curl, @"(?:--url\s+|curl\s+)(['""]?)(https?://[^\s'""]+)\1", RegexOptions.IgnoreCase);
+            if (!urlMatch.Success)
+            {
+                urlMatch = Regex.Match(curl, @"(['""])(https?://[^\s'""]+)\1", RegexOptions.IgnoreCase);
+            }
+            if (urlMatch.Success)
+            {
+                request.RequestUri = new Uri(urlMatch.Groups[2].Value);
+            }
+            else
+            {
+                throw new InvalidOperationException("No URL found in curl string and RequestUri is null.");
+            }
+        }
+
+        var headerPattern = @"(?:-H|--header)\s+['""]?([^:'""]+):\s*([^'""]+)['""]?";
+        foreach (Match headerMatch in Regex.Matches(curl, headerPattern, RegexOptions.IgnoreCase))
+        {
+            string key = headerMatch.Groups[1].Value.Trim();
+            string value = headerMatch.Groups[2].Value.Trim();
+
+            if (key.Equals("Authorization", StringComparison.OrdinalIgnoreCase) && request.Headers.Contains("Authorization"))
+            {
+                continue;
+            }
+
+            if (!request.Headers.TryAddWithoutValidation(key, value))
+            {
+                if (request.Content == null)
+                    request.Content = new StringContent(""); 
+
+                request.Content.Headers.TryAddWithoutValidation(key, value);
+            }
+        }
+
+        var processedCurl = PreprocessCurlString(curl);
+        var dataMatches = Regex.Matches(processedCurl, @"(?:-d|--data(?:-raw)?)\s+(['""])((?:\\.|(?!\1).)*)\1", RegexOptions.IgnoreCase | RegexOptions.Singleline);
+
+
+        if (request.Method == HttpMethod.Get)
+        {
+            var queryParams = new List<string>();
+            foreach (Match m in dataMatches)
+            {
+                var data = m.Groups[2].Value.Trim();
+                if (data.StartsWith("@"))
+                    throw new NotSupportedException("Unsupported -d with file for GET");
+
+                queryParams.Add(data);
+            }
+
+            if (queryParams.Count > 0)
+            {
+                var builder = new UriBuilder(request.RequestUri);
+                var existingQuery = builder.Query;
+                string newQuery = existingQuery.Length > 1 ? existingQuery.Substring(1) + "&" + string.Join("&", queryParams) : string.Join("&", queryParams);
+                builder.Query = newQuery;
+                request.RequestUri = builder.Uri;
+            }
+        }
+        else
+        {
+            if (dataMatches.Count > 0)
+            {
+                var rawData = dataMatches[0].Groups[2].Value;
+
+                // Replace escaped quotes and backslashes with placeholders for easier parsing
+                rawData = rawData.Replace("__SINGLE_QUOTE__", "'");
+                rawData = rawData.Replace("__DOUBLE_QUOTE__", "\"");
+                rawData = rawData.Replace("__BACKSLASH__", "\\");
+
+                // curl -d starting with '@' indicates a file; not supported here
+                if (rawData.StartsWith("@"))
+                    throw new NotSupportedException("Handling -d with file is not implemented.");
+
+                request.Content = new StringContent(rawData, Encoding.UTF8, "application/json");
+            }
+        }
+    }
+
+
+    public static string PreprocessCurlString(string curlRaw)
+    {
+        if (string.IsNullOrEmpty(curlRaw))
+            return curlRaw;
+
+        string result = curlRaw;
+        result = Regex.Replace(result, @"'\\''", "__SINGLE_QUOTE__");
+
+        result = result.Replace("\\\"", "__DOUBLE_QUOTE__");
+        result = result.Replace("\\\\", "__BACKSLASH__");
+
+        result = Regex.Replace(result, @"\\\s*\n", " ");
+
+        result = Regex.Replace(result, @"\s{2,}", " ").Trim();
+
+        return result;
+    }
+}

src/MaIN.Services/Utils/JsonCleaner.cs

@@ -0,0 +1,53 @@
+using MaIN.Domain.Models;
+using System;
+using System.Text.Encodings.Web;
+using System.Text.Json;
+
+public static class JsonCleaner
+{
+    public static string? CleanAndUnescape(string json, int maxDepth = 5)
+    {
+        if (string.IsNullOrWhiteSpace(json))
+            return null;
+
+        string current = json.Trim();
+        int depth = 0;
+
+        while (depth < maxDepth)
+        {
+            try
+            {
+                using JsonDocument doc = JsonDocument.Parse(current);
+                JsonElement root = doc.RootElement;
+
+                // Unwrap nested JSON strings up to maxDepth
+                if (root.ValueKind == JsonValueKind.String)
+                {
+                    current = root.GetString()!.Trim();
+                    depth++;
+                    continue;
+                }
+                else if (root.ValueKind == JsonValueKind.Object || root.ValueKind == JsonValueKind.Array)
+                {
+
+                    var options = new JsonSerializerOptions
+                    {
+                        WriteIndented = true,
+                        Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping
+                    };
+
+                    return JsonSerializer.Serialize(root, options);
+                }
+                else
+                {
+                    return null;
+                }
+            }
+            catch (JsonException)
+            {
+                return null;
+            }
+        }
+        return null;
+    }
+}