Skip to content

Commit ed05edf

Browse files
MadzionatorMadzionator
committed
Handle unregistered ai models; Support images input in cloud LLM
Add multi-image support and extract image bytes from uploaded files for LLM services; improve model ID/instance handling and model selection flow. - Message: replace single byte[] Image with List<byte[]> Images and keep a backward-compatible Image getter/setter. - Chat: preserve raw ModelId string, safely try to resolve model instance (no throws), and sync ModelInstance with internal id field. - Home.razor: unify model resolution into a local variable and choose GenericLocalModel/GenericCloudModel when registry lookup fails. - AnthropicService & OpenAiCompatibleService: add ExtractImageFromFiles to load image file bytes into Message.Images, remove consumed file entries, update HasImages/BuildMessageContent to iterate images, and extend image type detection (HEIC/HEIF, AVIF and more extensions). These changes enable passing uploaded images to compatible LLM backends while maintaining backward compatibility and preventing exceptions when models are missing.
1 parent c4a0f57 commit ed05edf

5 files changed

Lines changed: 148 additions & 26 deletions

File tree

‎src/MaIN.Domain/Entities/Chat.cs‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -9,25 +9,31 @@ public class Chat
99
{
1010
public string Id { get; init; } = string.Empty;
1111
public required string Name { get; init; }
12+
private string? _modelId;
1213
public required string ModelId
1314
{
14-
get => _modelInstance?.Id ?? string.Empty;
15+
get => _modelInstance?.Id ?? _modelId ?? string.Empty;
1516
set
1617
{
18+
_modelId = value;
1719
if (string.IsNullOrEmpty(value))
1820
{
1921
_modelInstance = null;
2022
return;
2123
}
2224

23-
_modelInstance = ModelRegistry.GetById(value);
25+
ModelRegistry.TryGetById(value, out _modelInstance);
2426
}
2527
}
2628
private AIModel? _modelInstance;
2729
public AIModel? ModelInstance
2830
{
2931
get => _modelInstance;
30-
set => (_modelInstance, ModelId) = (value, value?.Id ?? string.Empty);
32+
set
33+
{
34+
_modelInstance = value;
35+
_modelId = value?.Id ?? string.Empty;
36+
}
3137
}
3238
public List<Message> Messages { get; set; } = [];
3339
public ChatType Type { get; set; } = ChatType.Conversation;

‎src/MaIN.Domain/Entities/Message.cs‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,19 @@ public Message()
1919
public List<LLMTokenValue> Tokens { get; set; } = [];
2020
public bool Tool { get; init; }
2121
public DateTime Time { get; set; }
22-
public byte[]? Image { get; init; }
22+
public List<byte[]>? Images { get; set; }
23+
24+
// Backward-compat wrapper – single image access
25+
public byte[]? Image
26+
{
27+
get => Images?.Count > 0 ? Images[0] : null;
28+
set
29+
{
30+
if (value == null) Images = null;
31+
else Images = [value];
32+
}
33+
}
34+
2335
public byte[]? Speech { get; set; }
2436
public List<FileInfo>? Files { get; set; }
2537
public Dictionary<string, string> Properties { get; set; } = [];

‎src/MaIN.InferPage/Components/Pages/Home.razor‎

Lines changed: 16 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -226,13 +226,26 @@
226226

227227
protected override Task OnInitializedAsync()
228228
{
229+
AIModel? model = null;
230+
229231
try
230232
{
233+
if (Utils.BackendType == BackendType.Self && Utils.Path != null)
234+
{
235+
model = ModelRegistry.TryGetById(Utils.Model!, out var foundModel1)
236+
? foundModel1!
237+
: new GenericLocalModel($"{Utils.Model}.gguf");
238+
}
239+
else
240+
{
241+
model = ModelRegistry.TryGetById(Utils.Model!, out var foundModel1)
242+
? foundModel1!
243+
: new GenericCloudModel(Id: Utils.Model!, Backend: Utils.BackendType);
244+
}
245+
231246
ctx = Utils.Visual
232247
? AIHub.Chat().EnableVisual()
233-
: Utils.BackendType == BackendType.Self && Utils.Path != null
234-
? AIHub.Chat().WithModel(new GenericLocalModel(FileName: Utils.Model!, CustomPath: Utils.Path))
235-
: AIHub.Chat().WithModel(ModelRegistry.GetById(Utils.Model!));
248+
: AIHub.Chat().WithModel(model);
236249
}
237250
catch (MaINCustomException ex)
238251
{
@@ -243,7 +256,6 @@
243256
_errorMessage = ex.Message;
244257
}
245258

246-
var model = ModelRegistry.TryGetById(Utils.Model!, out var foundModel) ? foundModel : null;
247259
_reasoning = !Utils.Visual && model?.HasReasoning == true;
248260
Utils.Reason = _reasoning;
249261

‎src/MaIN.Services/Services/LLMService/AnthropicService.cs‎

Lines changed: 41 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ public sealed class AnthropicService(
2626
{
2727
private readonly MaINSettings _settings = settings ?? throw new ArgumentNullException(nameof(settings));
2828

29+
private static readonly HashSet<string> AnthropicImageExtensions = [".jpg", ".jpeg", ".png", ".gif", ".webp"];
2930
private static readonly ConcurrentDictionary<string, List<ChatMessage>> SessionCache = new();
3031

3132
private const string CompletionsUrl = ServiceConstants.ApiUrls.AnthropicChatMessages;
@@ -64,12 +65,14 @@ private void ValidateApiKey()
6465
return null;
6566

6667
var apiKey = GetApiKey();
68+
69+
var lastMessage = chat.Messages.Last();
70+
await ExtractImageFromFiles(lastMessage);
71+
6772
var conversation = GetOrCreateConversation(chat, options.CreateSession);
6873
var resultBuilder = new StringBuilder();
6974
var tokens = new List<LLMTokenValue>();
7075

71-
var lastMessage = chat.Messages.Last();
72-
7376
if (HasFiles(lastMessage))
7477
{
7578
var result = ChatHelper.ExtractMemoryOptions(lastMessage);
@@ -632,6 +635,42 @@ private static bool HasFiles(Message message)
632635
return message.Files != null && message.Files.Count > 0;
633636
}
634637

638+
private static async Task ExtractImageFromFiles(Message message)
639+
{
640+
if (message.Files == null || message.Files.Count == 0)
641+
return;
642+
643+
var imageFiles = message.Files
644+
.Where(f => AnthropicImageExtensions.Contains(f.Extension.ToLowerInvariant()))
645+
.ToList();
646+
647+
if (imageFiles.Count == 0)
648+
return;
649+
650+
var imageBytesList = new List<byte[]>();
651+
foreach (var imageFile in imageFiles)
652+
{
653+
if (imageFile.StreamContent != null)
654+
{
655+
using var ms = new MemoryStream();
656+
imageFile.StreamContent.Position = 0;
657+
await imageFile.StreamContent.CopyToAsync(ms);
658+
imageBytesList.Add(ms.ToArray());
659+
}
660+
else if (imageFile.Path != null)
661+
{
662+
imageBytesList.Add(await File.ReadAllBytesAsync(imageFile.Path));
663+
}
664+
665+
message.Files.Remove(imageFile);
666+
}
667+
668+
message.Images = imageBytesList;
669+
670+
if (message.Files.Count == 0)
671+
message.Files = null;
672+
}
673+
635674
private async Task ProcessStreamingChatAsync(
636675
Chat chat,
637676
List<ChatMessage> conversation,

‎src/MaIN.Services/Services/LLMService/OpenAiCompatibleService.cs‎

Lines changed: 69 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -28,16 +28,13 @@ public abstract class OpenAiCompatibleService(
2828
ILogger<OpenAiCompatibleService>? logger = null)
2929
: ILLMService
3030
{
31-
private readonly INotificationService _notificationService =
32-
notificationService ?? throw new ArgumentNullException(nameof(notificationService));
33-
34-
private readonly IHttpClientFactory _httpClientFactory =
35-
httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory));
31+
private readonly INotificationService _notificationService = notificationService ?? throw new ArgumentNullException(nameof(notificationService));
32+
private readonly IHttpClientFactory _httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory));
3633

3734
private static readonly ConcurrentDictionary<string, List<ChatMessage>> SessionCache = new();
35+
private static readonly HashSet<string> ImageExtensions = [".jpg", ".jpeg", ".png", ".gif", ".webp", ".bmp", ".tiff", ".tif", ".heic", ".heif", ".avif"];
3836

39-
private static readonly JsonSerializerOptions DefaultJsonSerializerOptions =
40-
new() { PropertyNameCaseInsensitive = true };
37+
private static readonly JsonSerializerOptions DefaultJsonSerializerOptions = new() { PropertyNameCaseInsensitive = true };
4138

4239
private const string ToolCallsProperty = "ToolCalls";
4340
private const string ToolCallIdProperty = "ToolCallId";
@@ -63,10 +60,11 @@ public abstract class OpenAiCompatibleService(
6360
List<LLMTokenValue> tokens = new();
6461
string apiKey = GetApiKey();
6562

63+
var lastMessage = chat.Messages.Last();
64+
await ExtractImageFromFiles(lastMessage);
65+
6666
List<ChatMessage> conversation = GetOrCreateConversation(chat, options.CreateSession);
6767
StringBuilder resultBuilder = new();
68-
69-
var lastMessage = chat.Messages.Last();
7068
if (HasFiles(lastMessage))
7169
{
7270
var result = ChatHelper.ExtractMemoryOptions(lastMessage);
@@ -576,6 +574,42 @@ private void UpdateSessionCache(string chatId, string assistantResponse, bool cr
576574
}
577575
}
578576

577+
private static async Task ExtractImageFromFiles(Message message)
578+
{
579+
if (message.Files == null || message.Files.Count == 0)
580+
return;
581+
582+
var imageFiles = message.Files
583+
.Where(f => ImageExtensions.Contains(f.Extension.ToLowerInvariant()))
584+
.ToList();
585+
586+
if (imageFiles.Count == 0)
587+
return;
588+
589+
var imageBytesList = new List<byte[]>();
590+
foreach (var imageFile in imageFiles)
591+
{
592+
if (imageFile.StreamContent != null)
593+
{
594+
using var ms = new MemoryStream();
595+
imageFile.StreamContent.Position = 0;
596+
await imageFile.StreamContent.CopyToAsync(ms);
597+
imageBytesList.Add(ms.ToArray());
598+
}
599+
else if (imageFile.Path != null)
600+
{
601+
imageBytesList.Add(await File.ReadAllBytesAsync(imageFile.Path));
602+
}
603+
604+
message.Files.Remove(imageFile);
605+
}
606+
607+
message.Images = imageBytesList;
608+
609+
if (message.Files.Count == 0)
610+
message.Files = null;
611+
}
612+
579613
private static bool HasFiles(Message message)
580614
{
581615
return message.Files != null && message.Files.Count > 0;
@@ -912,7 +946,7 @@ private static async Task InvokeTokenCallbackAsync(Func<LLMTokenValue, Task>? ca
912946

913947
private static bool HasImages(Message message)
914948
{
915-
return message.Image != null && message.Image.Length > 0;
949+
return message.Images?.Count > 0;
916950
}
917951

918952
private static object BuildMessageContent(Message message, ImageType imageType)
@@ -933,10 +967,10 @@ private static object BuildMessageContent(Message message, ImageType imageType)
933967
});
934968
}
935969

936-
if (message.Image != null && message.Image.Length > 0)
970+
foreach (var imageBytes in message.Images!)
937971
{
938-
var base64Data = Convert.ToBase64String(message.Image);
939-
var mimeType = DetectImageMimeType(message.Image);
972+
var base64Data = Convert.ToBase64String(imageBytes);
973+
var mimeType = DetectImageMimeType(imageBytes);
940974

941975
switch (imageType)
942976
{
@@ -976,24 +1010,43 @@ private static string DetectImageMimeType(byte[] imageBytes)
9761010

9771011
if (imageBytes[0] == 0xFF && imageBytes[1] == 0xD8)
9781012
return "image/jpeg";
979-
1013+
9801014
if (imageBytes.Length >= 8 &&
9811015
imageBytes[0] == 0x89 && imageBytes[1] == 0x50 &&
9821016
imageBytes[2] == 0x4E && imageBytes[3] == 0x47)
9831017
return "image/png";
984-
1018+
9851019
if (imageBytes.Length >= 6 &&
9861020
imageBytes[0] == 0x47 && imageBytes[1] == 0x49 &&
9871021
imageBytes[2] == 0x46 && imageBytes[3] == 0x38)
9881022
return "image/gif";
989-
1023+
9901024
if (imageBytes.Length >= 12 &&
9911025
imageBytes[0] == 0x52 && imageBytes[1] == 0x49 &&
9921026
imageBytes[2] == 0x46 && imageBytes[3] == 0x46 &&
9931027
imageBytes[8] == 0x57 && imageBytes[9] == 0x45 &&
9941028
imageBytes[10] == 0x42 && imageBytes[11] == 0x50)
9951029
return "image/webp";
9961030

1031+
// HEIC/HEIF format (iPhone photos)
1032+
if (imageBytes.Length >= 12 &&
1033+
imageBytes[4] == 0x66 && imageBytes[5] == 0x74 &&
1034+
imageBytes[6] == 0x79 && imageBytes[7] == 0x70)
1035+
{
1036+
// Check for heic/heif brands
1037+
if ((imageBytes[8] == 0x68 && imageBytes[9] == 0x65 && imageBytes[10] == 0x69 && imageBytes[11] == 0x63) ||
1038+
(imageBytes[8] == 0x68 && imageBytes[9] == 0x65 && imageBytes[10] == 0x69 && imageBytes[11] == 0x66))
1039+
return "image/heic";
1040+
}
1041+
1042+
// AVIF format
1043+
if (imageBytes.Length >= 12 &&
1044+
imageBytes[4] == 0x66 && imageBytes[5] == 0x74 &&
1045+
imageBytes[6] == 0x79 && imageBytes[7] == 0x70 &&
1046+
imageBytes[8] == 0x61 && imageBytes[9] == 0x76 &&
1047+
imageBytes[10] == 0x69 && imageBytes[11] == 0x66)
1048+
return "image/avif";
1049+
9971050
return "image/jpeg";
9981051
}
9991052
}

0 commit comments

Comments
 (0)