Fix vertex file send for pdf files (without ocr)

Madzionator · Madzionator · commit cec95a624391 · 2026-03-26T13:30:40.000+01:00
Expose CreateMemoryWithVertex on IMemoryFactory and implement it in MemoryFactory to build a KernelMemory configured for Vertex (Gemini) text generation and embeddings using a bearer token provider, location and projectId. Add PDF MIME detection in ChatHelper. Extend VertexService.AskMemory to bypass KernelMemory for multimodal requests: collect inline images/PDFs, convert non-native files to text via DocumentProcessor, aggregate text/context/memory, inject optional grammar JSON, and send combined content to Gemini; includes helper methods for file/stream processing and a list of Gemini-native extensions. Also add necessary usings and temporary file handling for stream processing.
diff --git a/src/MaIN.Services/Services/LLMService/Memory/IMemoryFactory.cs b/src/MaIN.Services/Services/LLMService/Memory/IMemoryFactory.cs
@@ -14,4 +14,5 @@ public interface IMemoryFactory
             MemoryParams memoryParams);
     IKernelMemory CreateMemoryWithOpenAi(string openAiKey, MemoryParams memoryParams);
     IKernelMemory CreateMemoryWithGemini(string geminiKey, MemoryParams memoryParams);
+    IKernelMemory CreateMemoryWithVertex(Func<ValueTask<string>> bearerTokenProvider, string location, string projectId, MemoryParams memoryParams);
 }
diff --git a/src/MaIN.Services/Services/LLMService/Memory/MemoryFactory.cs b/src/MaIN.Services/Services/LLMService/Memory/MemoryFactory.cs
@@ -74,6 +74,29 @@ public IKernelMemory CreateMemoryWithGemini(string geminiKey, MemoryParams memor
             .WithSemanticKernelTextEmbeddingGenerationService(
                 new GoogleAITextEmbeddingGenerationService("gemini-embedding-001", geminiKey), new SemanticKernelConfig())
 #pragma warning restore SKEXP0070
+            .WithCustomImageOcr(new OcrWrapper())
+            .WithSimpleVectorDb()
+            .Build();
+
+        return kernelMemory;
+    }
+
+    public IKernelMemory CreateMemoryWithVertex(Func<ValueTask<string>> bearerTokenProvider, string location, string projectId, MemoryParams memoryParams)
+    {
+        var searchOptions = ConfigureSearchOptions(memoryParams);
+
+        var kernelMemory = new KernelMemoryBuilder()
+            .WithSearchClientConfig(searchOptions)
+#pragma warning disable SKEXP0070
+            .WithSemanticKernelTextGenerationService(
+                new GeminiTextGeneratorAdapter(
+                    new VertexAIGeminiChatCompletionService("gemini-2.5-flash", bearerTokenProvider, location, projectId)),
+                new SemanticKernelConfig())
+            .WithSemanticKernelTextEmbeddingGenerationService(
+                new VertexAITextEmbeddingGenerationService("text-embedding-005", bearerTokenProvider, location, projectId),
+                new SemanticKernelConfig())
+#pragma warning restore SKEXP0070
+            .WithCustomImageOcr(new OcrWrapper())
             .WithSimpleVectorDb()
             .Build();
 
diff --git a/src/MaIN.Services/Services/LLMService/Utils/ChatHelper.cs b/src/MaIN.Services/Services/LLMService/Utils/ChatHelper.cs
@@ -259,6 +259,11 @@ private static string DetectImageMimeType(byte[] imageBytes)
         if (imageBytes.Length < 4)
             return "image/jpeg";
 
+        // PDF: %PDF (0x25 0x50 0x44 0x46)
+        if (imageBytes[0] == 0x25 && imageBytes[1] == 0x50 &&
+            imageBytes[2] == 0x44 && imageBytes[3] == 0x46)
+            return "application/pdf";
+
         if (imageBytes[0] == 0xFF && imageBytes[1] == 0xD8)
             return "image/jpeg";
 
diff --git a/src/MaIN.Services/Services/LLMService/VertexService.cs b/src/MaIN.Services/Services/LLMService/VertexService.cs
@@ -1,12 +1,15 @@
+using System.Text;
 using MaIN.Domain.Configuration;
 using MaIN.Domain.Configuration.BackendInferenceParams;
 using MaIN.Domain.Entities;
+using MaIN.Domain.Models;
 using MaIN.Domain.Models.Concrete;
 using MaIN.Services.Constants;
 using MaIN.Services.Services.Abstract;
 using MaIN.Services.Services.LLMService.Auth;
 using MaIN.Services.Services.LLMService.Memory;
 using MaIN.Services.Services.Models;
+using MaIN.Services.Utils;
 using Microsoft.Extensions.Logging;
 
 namespace MaIN.Services.Services.LLMService;
@@ -97,16 +100,153 @@ protected override void ApplyBackendParams(Dictionary<string, object> requestBod
         return await base.Send(chat, options, cancellationToken);
     }
 
-    public new async Task<ChatResult?> AskMemory(
+    /// <summary>
+    /// Bypasses KernelMemory and sends files directly to Gemini via multimodal API.
+    /// PDFs and images are sent inline (Gemini handles OCR natively),
+    /// other formats are pre-processed to text via DocumentProcessor.
+    /// </summary>
+    public override async Task<ChatResult?> AskMemory(
         Chat chat,
         ChatMemoryOptions memoryOptions,
         ChatRequestOptions requestOptions,
         CancellationToken cancellationToken = default)
     {
         ExtractLocation(chat);
-        return await base.AskMemory(chat, memoryOptions, requestOptions, cancellationToken);
+
+        if (!chat.Messages.Any())
+            return null;
+
+        var lastMessage = chat.Messages.Last();
+        var originalContent = lastMessage.Content;
+        var originalFiles = lastMessage.Files;
+        var originalImages = lastMessage.Images;
+
+        try
+        {
+            var inlineBytes = new List<byte[]>();
+            var textContext = new StringBuilder();
+
+            CollectTextData(memoryOptions, textContext);
+            await CollectFilesData(memoryOptions, inlineBytes, textContext, cancellationToken);
+            await CollectStreamData(memoryOptions, inlineBytes, textContext, cancellationToken);
+            CollectMemoryItems(memoryOptions, textContext);
+
+            var queryBuilder = new StringBuilder();
+            if (textContext.Length > 0)
+            {
+                queryBuilder.AppendLine("Use the following document content to answer the question:\n");
+                queryBuilder.Append(textContext);
+                queryBuilder.AppendLine();
+            }
+            queryBuilder.Append(originalContent);
+
+            if (chat.MemoryParams.Grammar != null)
+            {
+                var jsonGrammar = new GrammarToJsonConverter().ConvertToJson(chat.MemoryParams.Grammar);
+                queryBuilder.Append(
+                    $" | For your next response only, please respond using exactly the following JSON format: \n{jsonGrammar}\n. Do not include any explanations, code blocks, or additional content. After this single JSON response, resume your normal conversational style.");
+            }
+
+            lastMessage.Content = queryBuilder.ToString();
+            lastMessage.Files = null;
+
+            // Merge existing images with inline file bytes (PDFs sent as native multimodal content)
+            var allInline = new List<byte[]>(originalImages ?? []);
+            allInline.AddRange(inlineBytes);
+            lastMessage.Images = allInline.Count > 0 ? allInline : null;
+
+            return await Send(chat, requestOptions, cancellationToken);
+        }
+        finally
+        {
+            lastMessage.Content = originalContent;
+            lastMessage.Files = originalFiles;
+            lastMessage.Images = originalImages;
+        }
+    }
+
+    #region Multimodal File Processing
+
+    private static readonly HashSet<string> GeminiNativeExtensions =
+        [".pdf", ".jpg", ".jpeg", ".png", ".gif", ".webp", ".bmp", ".tiff", ".tif", ".heic", ".heif", ".avif"];
+
+    private static bool IsGeminiNativeFile(string fileName)
+        => GeminiNativeExtensions.Contains(Path.GetExtension(fileName).ToLowerInvariant());
+
+    private static void CollectTextData(ChatMemoryOptions options, StringBuilder textContext)
+    {
+        foreach (var (name, content) in options.TextData)
+        {
+            textContext.AppendLine($"[Document: {name}]");
+            textContext.AppendLine(content);
+            textContext.AppendLine();
+        }
     }
 
+    private static async Task CollectFilesData(
+        ChatMemoryOptions options, List<byte[]> inlineBytes, StringBuilder textContext,
+        CancellationToken cancellationToken)
+    {
+        foreach (var (name, path) in options.FilesData)
+        {
+            if (IsGeminiNativeFile(name))
+            {
+                inlineBytes.Add(await File.ReadAllBytesAsync(path, cancellationToken));
+            }
+            else
+            {
+                textContext.AppendLine($"[Document: {name}]");
+                textContext.AppendLine(DocumentProcessor.ProcessDocument(path));
+                textContext.AppendLine();
+            }
+        }
+    }
+
+    private static async Task CollectStreamData(
+        ChatMemoryOptions options, List<byte[]> inlineBytes, StringBuilder textContext,
+        CancellationToken cancellationToken)
+    {
+        foreach (var (name, stream) in options.StreamData)
+        {
+            using var ms = new MemoryStream();
+            if (stream.CanSeek) stream.Position = 0;
+            await stream.CopyToAsync(ms, cancellationToken);
+            var bytes = ms.ToArray();
+
+            if (IsGeminiNativeFile(name))
+            {
+                inlineBytes.Add(bytes);
+            }
+            else
+            {
+                var tempPath = Path.Combine(Path.GetTempPath(), $"vertex_tmp_{Guid.NewGuid()}{Path.GetExtension(name)}");
+                try
+                {
+                    await File.WriteAllBytesAsync(tempPath, bytes, cancellationToken);
+                    textContext.AppendLine($"[Document: {name}]");
+                    textContext.AppendLine(DocumentProcessor.ProcessDocument(tempPath));
+                    textContext.AppendLine();
+                }
+                finally
+                {
+                    if (File.Exists(tempPath)) File.Delete(tempPath);
+                }
+            }
+        }
+    }
+
+    private static void CollectMemoryItems(ChatMemoryOptions options, StringBuilder textContext)
+    {
+        if (options.Memory is not { Count: > 0 }) return;
+        foreach (var item in options.Memory)
+        {
+            textContext.AppendLine(item);
+            textContext.AppendLine();
+        }
+    }
+
+    #endregion
+
     private void ExtractLocation(Chat chat)
     {
         if (chat.BackendParams is VertexInferenceParams vp)

Original file line number	Diff line number	Diff line change
`@@ -14,4 +14,5 @@ public interface IMemoryFactory`
`14`	`14`	`MemoryParams memoryParams);`
`15`	`15`	`IKernelMemory CreateMemoryWithOpenAi(string openAiKey, MemoryParams memoryParams);`
`16`	`16`	`IKernelMemory CreateMemoryWithGemini(string geminiKey, MemoryParams memoryParams);`
	`17`	`+ IKernelMemory CreateMemoryWithVertex(Func<ValueTask<string>> bearerTokenProvider, string location, string projectId, MemoryParams memoryParams);`
`17`	`18`	`}`