Remove media queue

SignalRT · SignalRT · commit 855b8c5dc3d4 · 2026-01-13T22:32:44.000+01:00
diff --git a/LLama.Examples/Examples/BatchedExecutorMtmd.cs b/LLama.Examples/Examples/BatchedExecutorMtmd.cs
@@ -19,7 +19,7 @@ public class BatchedExecutorMtmd
     /// <summary>
     /// Number of completion tokens to generate after sending the image prompt.
     /// </summary>
-    public const int TokenCount = 10000;
+    public const int TokenCount = 100;
 
     public static async Task Run()
     {
@@ -60,10 +60,11 @@ public static async Task Run()
         {
             // Each conversation tracks its own KV cache sequence IDs.
             var conversation = executor.Create();
-            // enqueue the image so MtmdHelper sees it
-            conversation.QueueMedia(imagePath); 
-            // schedule multimodal prompt
-            conversation.Prompt(promptText, addBos: true, special: true); 
+            // Load the media embed explicitly so ownership is clear.
+            using var embed = mtmd.LoadMedia( imagePath)
+                ?? throw new RuntimeError($"Failed to load media '{imagePath}'.");
+            // Schedule the multimodal prompt with explicit embeds.
+            conversation.Prompt(promptText, new[] { embed }, addBos: true); 
 
             Console.ForegroundColor = ConsoleColor.Yellow;
             Console.WriteLine("Prompt queued with multimodal chunks. Generating response...\n");
diff --git a/LLama.Examples/Examples/MtmdInteractiveModeExecute.cs b/LLama.Examples/Examples/MtmdInteractiveModeExecute.cs
@@ -19,7 +19,7 @@ public static async Task Run()
         {
             string multiModalProj = UserSettings.GetMMProjPath();
             string modelPath = UserSettings.GetModelPath();
-            const int maxTokens = 8192;
+            const int maxTokens = 4096;
 
             string? prompt = await File.ReadAllTextAsync("Assets/chat-with-bob.json");
 
diff --git a/LLama/Batched/Conversation.cs b/LLama/Batched/Conversation.cs
@@ -22,7 +22,6 @@ public sealed class Conversation
     /// Indicates if this conversation has been "forked" and may share logits with another conversation.
     /// </summary>
     private bool _forked;
-    private readonly List<SafeMtmdEmbed> _mtmdEmbeds = new();
     private MtmdChunkSequence? _pendingMtmdSequence;
     private readonly List<LLamaToken> _embed_inps = new();
     private readonly List<LLamaToken> _session_tokens = new();
@@ -133,8 +132,6 @@ public void Dispose()
         _pendingMtmdSequence?.Dispose();
         _pendingMtmdSequence = null;
 
-        DisposeQueuedMedia();
-
         // Remove this conversation from the KV cache
         Executor.Context.NativeHandle.MemorySequenceRemove(ConversationId, -1, -1);
 
@@ -251,37 +248,29 @@ private void AssertCanBePrompted()
             throw new AlreadyPromptedConversationException();
     }
 
-    public void QueueMedia(string path)
+    public void Prompt(string promptText, bool addBos = true, bool special = true)
     {
-        AssertCanBePrompted();
-
-        if (Executor.ClipModel is null)
-            throw new InvalidOperationException("This conversation is not configured for multimodal prompts.");
-
-        var embed = Executor.ClipModel.LoadMedia(path);
-        _mtmdEmbeds.Add(embed);
+        var tokens = Executor.Context.Tokenize(promptText, addBos, special);
+        Prompt(tokens);
     }
 
-    public void QueueMedia(SafeMtmdEmbed embed)
+    /// <summary>
+    /// Prompt this conversation with explicit multimodal embeddings.
+    /// The caller retains ownership of <paramref name="embeds"/>.
+    /// </summary>
+    /// <param name="promptText">Prompt text for the model.</param>
+    /// <param name="embeds">Media embeddings to include in the multimodal prompt.</param>
+    /// <param name="addBos">Whether to add the BOS token.</param>
+    public void Prompt(string promptText, ReadOnlySpan<SafeMtmdEmbed> embeds, bool addBos = true)
     {
         AssertCanBePrompted();
 
         if (Executor.ClipModel is null)
             throw new InvalidOperationException("This conversation is not configured for multimodal prompts.");
+        if (embeds.IsEmpty)
+            throw new ArgumentException("Embeds cannot be empty for multimodal prompts.", nameof(embeds));
 
-        _mtmdEmbeds.Add(embed);
-    }
-
-    public void Prompt(string promptText, bool addBos = true, bool special = true)
-    {
-        if (Executor.ClipModel != null && _mtmdEmbeds.Count > 0)
-        {
-            PromptMultimodal(promptText, addBos);
-            return;
-        }
-
-        var tokens = Executor.Context.Tokenize(promptText, addBos, special);
-        Prompt(tokens);
+        PromptMultimodal(promptText, addBos, embeds);
     }
 
     /// <summary>
@@ -367,36 +356,21 @@ public void Prompt(ReadOnlySpan<LLamaToken> tokens, bool allLogits = false)
         _forked = false;
     }
 
-    private void PromptMultimodal(string text, bool addBos)
+    private void PromptMultimodal(string text, bool addBos, ReadOnlySpan<SafeMtmdEmbed> embeds)
     {
         AssertCanBePrompted();
 
         if (Executor.ClipModel is null)
             throw new InvalidOperationException("This conversation is not configured for multimodal prompts.");
-        if (_mtmdEmbeds.Count == 0)
-            throw new InvalidOperationException("Queue media before prompting with multimodal input.");
 
-        var marker = Executor.GetMtmdMarker();
-        var prompt = text;
-
-        if (prompt.Contains("<image>"))
-            prompt = prompt.Replace("<image>", marker);
-
-        if (!prompt.Contains(marker))
-        {
-            var suffix = string.Concat(Enumerable.Repeat(marker, _mtmdEmbeds.Count));
-            prompt = string.Concat(prompt, suffix);
-        }
+        var prompt = BuildMtmdPrompt(text, embeds.Length);
 
         SafeMtmdInputChunks? chunks = null;
         try
         {
-            var status = Executor.ClipModel.Tokenize(prompt, addBos, parseSpecial: true, out chunks);
+            var status = Executor.ClipModel.Tokenize(prompt, addBos, parseSpecial: true, embeds, out chunks);
             if (status != 0 || chunks is null)
-            {
-                Executor.ClipModel.ClearMedia();
                 throw new RuntimeError($"Failed to tokenize multimodal prompt. Status: {status}.");
-            }
 
             var sequence = MtmdChunkSequence.Create(chunks, Executor.ClipModel);
             _pendingMtmdSequence = sequence;
@@ -413,11 +387,27 @@ private void PromptMultimodal(string text, bool addBos)
         }
         finally
         {
-            DisposeQueuedMedia();
             chunks?.Dispose();
         }
     }
 
+    private string BuildMtmdPrompt(string text, int embedCount)
+    {
+        var marker = Executor.GetMtmdMarker();
+        var prompt = text;
+
+        if (prompt.Contains("<image>"))
+            prompt = prompt.Replace("<image>", marker);
+
+        if (!prompt.Contains(marker))
+        {
+            var suffix = string.Concat(Enumerable.Repeat(marker, embedCount));
+            prompt = string.Concat(prompt, suffix);
+        }
+
+        return prompt;
+    }
+
     /// <summary>
     /// Add a single token to this conversation
     /// </summary>
@@ -525,7 +515,6 @@ internal void OnMtmdEvaluationFailed(int status)
         _pendingMtmdSequence?.Dispose();
         _pendingMtmdSequence = null;
         _requiredEpoch = Executor.Epoch;
-        DisposeQueuedMedia();
     }
 
     private LLamaToken GetFillerToken(string marker)
@@ -541,18 +530,6 @@ private LLamaToken GetFillerToken(string marker)
         return default;
     }
 
-    private void DisposeQueuedMedia()
-    {
-        if (_mtmdEmbeds.Count == 0)
-            return;
-
-        foreach (var embed in _mtmdEmbeds)
-            embed.Dispose();
-
-        _mtmdEmbeds.Clear();
-        Executor.ClipModel?.ClearMedia();
-    }
-
     /// <summary>
     /// Provides direct access to the KV cache of a <see cref="Conversation"/>.
     /// See <see cref="Modify"/> for how to use this.
diff --git a/LLama/MtmdWeights.cs b/LLama/MtmdWeights.cs
@@ -54,6 +54,13 @@ public static Task<MtmdWeights> LoadFromFileAsync(string mmProject, LLamaWeights
     public int Tokenize(string text, bool addSpecial, bool parseSpecial, out SafeMtmdInputChunks? chunks)
         => NativeHandle.Tokenize(text, addSpecial, parseSpecial, out chunks);
 
+    /// <summary>
+    /// Tokenize text (with optional special tokens) against explicit media embeddings.
+    /// The caller retains ownership of <paramref name="embeds"/>.
+    /// </summary>
+    public int Tokenize(string text, bool addSpecial, bool parseSpecial, ReadOnlySpan<SafeMtmdEmbed> embeds, out SafeMtmdInputChunks? chunks)
+        => NativeHandle.Tokenize(text, addSpecial, parseSpecial, embeds, out chunks);
+
     /// <summary>
     /// Evaluate a chunk batch using the helper that performs mtmd encode + llama decode.
     /// </summary>
diff --git a/LLama/Native/SafeMtmdModelHandle.cs b/LLama/Native/SafeMtmdModelHandle.cs
@@ -149,6 +149,47 @@ public int Tokenize(string text, bool addSpecial, bool parseSpecial, out SafeMtm
             return result;
         }
 
+        /// <summary>
+        /// Tokenize a prompt alongside the provided media embeddings.
+        /// The caller retains ownership of <paramref name="embeds"/>.
+        /// </summary>
+        /// <param name="text">Prompt text to tokenize.</param>
+        /// <param name="addSpecial">Whether to append special tokens automatically.</param>
+        /// <param name="parseSpecial">Whether special tokens should be treated as user-provided text.</param>
+        /// <param name="embeds">Media embeddings to include in the multimodal prompt.</param>
+        /// <param name="chunks">Receives the native chunk collection when tokenization succeeds.</param>
+        /// <returns>Zero on success; otherwise the native mtmd tokenize error code.</returns>
+        /// <exception cref="ObjectDisposedException">The model handle has been disposed.</exception>
+        /// <exception cref="RuntimeError">The native tokenizer failed to allocate output chunks.</exception>
+        public int Tokenize(string text, bool addSpecial, bool parseSpecial, ReadOnlySpan<SafeMtmdEmbed> embeds, out SafeMtmdInputChunks? chunks)
+        {
+            EnsureNotDisposed();
+
+            chunks = null;
+            var output = NativeApi.mtmd_input_chunks_init();
+            if (output == IntPtr.Zero)
+                throw new RuntimeError("Failed to allocate mtmd_input_chunks.");
+
+            var bitmapHandles = new IntPtr[embeds.Length];
+            for (var i = 0; i < embeds.Length; i++)
+            {
+                var embed = embeds[i] ?? throw new ArgumentNullException(nameof(embeds), "Embeds cannot contain null.");
+                bitmapHandles[i] = embed.NativePtr;
+            }
+
+            var result = NativeApi.mtmd_tokenize(DangerousGetHandle(), output, text, addSpecial, parseSpecial, bitmapHandles, (UIntPtr)bitmapHandles.Length);
+            if (result == 0)
+            {
+                chunks = new SafeMtmdInputChunks(output);
+            }
+            else
+            {
+                NativeApi.mtmd_input_chunks_free(output);
+            }
+
+            return result;
+        }
+
         /// <summary>
         /// Evaluate a batch of chunks using the helper (mirrors mtmd-helper eval logic).
         /// </summary>

Original file line number	Diff line number	Diff line change
`@@ -19,7 +19,7 @@ public static async Task Run()`
`19`	`19`	`{`
`20`	`20`	`string multiModalProj = UserSettings.GetMMProjPath();`
`21`	`21`	`string modelPath = UserSettings.GetModelPath();`
`22`		`- const int maxTokens = 8192;`
	`22`	`+ const int maxTokens = 4096;`
`23`	`23`
`24`	`24`	`string? prompt = await File.ReadAllTextAsync("Assets/chat-with-bob.json");`
`25`	`25`