Updated to: ggml-org/llama.cpp@3f7c29d

Build: https://github.com/SciSharp/LLamaSharp/actions/runs/24541043843

Author: martindevans

LLama.Unittest/MtmdWeightsTests.cs

@@ -81,7 +81,7 @@ public void BasicPropertyChecks()
             Assert.True(_mtmdWeights.SupportsVision);
             Assert.False(_mtmdWeights.UsesMRope);
             Assert.True(_mtmdWeights.UsesNonCausalAttention);
-            Assert.Equal(-1, _mtmdWeights.AudioBitrate);
+            Assert.Equal(-1, _mtmdWeights.SampleRate);
         }
 
         [Fact,Trait("Category", "NoCI")]
@@ -143,8 +143,8 @@ public void TokenizeProvidesChunkMetadata()
             Assert.True(_mtmdWeights.SupportsVision);
             Assert.False(_mtmdWeights.SupportsAudio);
 
-            var audioBitrate = _mtmdWeights.AudioBitrate;
-            Assert.True(audioBitrate <= 0);
+            var audioSampleRate = _mtmdWeights.SampleRate;
+            Assert.True(audioSampleRate <= 0);
         }
     }
 }

LLama/LLamaSharp.csproj

@@ -57,7 +57,7 @@
   </ItemGroup>
 
   <PropertyGroup>
-    <BinaryReleaseId>73c9eb8ceda397b</BinaryReleaseId>
+    <BinaryReleaseId>3f7c29d318e317b6</BinaryReleaseId>
   </PropertyGroup>
 
   <PropertyGroup>

LLama/MtmdWeights.cs

@@ -137,9 +137,9 @@ public int DecodeImageChunk(IntPtr chunkPtr, SafeLLamaContextHandle llamaContext
     public bool UsesMRope => NativeHandle.DecodeUseMRope();
 
     /// <summary>
-    /// Gets the audio bitrate advertised by the model.
+    /// Gets the audio sample rate advertised by the model.
     /// </summary>
-    public int AudioBitrate => NativeHandle.GetAudioBitrate();
+    public int SampleRate => NativeHandle.GetAudioSampleRate();
 
     /// <inheritdoc />
     public void Dispose() => NativeHandle.Dispose();

LLama/Native/GPUSplitMode.cs

@@ -20,4 +20,10 @@ public enum GPUSplitMode
     /// split layers and KV across GPUs, use tensor parallelism if supported
     /// </summary>
     Row = 2,
+    
+    // Undocumented in llama.h
+    /// <summary>
+    /// 
+    /// </summary>
+    Tensor = 3,
 }

LLama/Native/LLamaFtype.cs

@@ -212,6 +212,11 @@ public enum LLamaFtype
         /// </summary>
         LLAMA_FTYPE_MOSTLY_NVFP4 = 39,
 
+        /// <summary>
+        /// Except 1d tensors
+        /// </summary>
+        LLAMA_FTYPE_MOSTLY_Q1_0 = 40,
+
         /// <summary>
         /// File type was not specified
         /// </summary>

LLama/Native/LLamaModelImatrixData.cs

@@ -0,0 +1,12 @@
+namespace LLama.Native;
+
+/* /// <summary>
+/// 
+/// </summary>
+/// <remarks>llama_model_imatrix_data</remarks>
+public unsafe struct LLamaModelImatrixData
+{
+    char* name;
+    float* data;
+    nuint size;
+} */

LLama/Native/LLamaModelQuantizeParams.cs

@@ -92,22 +92,22 @@ public bool dry_run
         /// <summary>
         /// pointer to importance matrix data
         /// </summary>
-        public IntPtr imatrix;
+        public IntPtr imatrix; // LLamaModelImatrixData *
 
         /// <summary>
         /// pointer to vector containing overrides
         /// </summary>
-        public IntPtr kv_overrides;
+        public IntPtr kv_overrides; // llama_model_kv_override *
 
         /// <summary>
         /// pointer to vector containing tensor types
         /// </summary>
-        public IntPtr tensor_types;
+        public IntPtr tensor_types; // llama_model_tensor_override *
 
         /// <summary>
         /// Pointer to vector containing layer indices to prune
         /// </summary>
-        public IntPtr prune_layers;
+        public IntPtr prune_layers; // int32 *
 
         /// <summary>
         /// Create a LLamaModelQuantizeParams with default values

LLama/Native/LLamaModelTensorOverride.cs

@@ -0,0 +1,9 @@
+namespace LLama.Native;
+
+// Unsupported - we can't handle ggml_type since LlamaSharp doesn't wrap/expose ggml
+/*
+ * struct llama_model_tensor_override {
+       const char * pattern;
+       enum ggml_type type; // GGMLType might work?
+   };
+*/

LLama/Native/LoraAdapter.cs

@@ -22,10 +22,42 @@ public class LoraAdapter
     /// </summary>
     internal IntPtr Pointer { get; }
 
+    /// <summary>
+    /// Indicates if this adapter has been unloaded
+    /// </summary>
+    internal bool Loaded { get; private set; }
+
     internal LoraAdapter(SafeLlamaModelHandle model, string path, IntPtr nativePtr)
     {
         Model = model;
         Path = path;
         Pointer = nativePtr;
+        Loaded = true;
+    }
+
+    /// <summary>
+    /// Unload this adapter
+    /// </summary>
+    public void Unload()
+    {
+        // Early exit if already unloaded
+        if (!Loaded)
+            return;
+        
+        // If the model has been unloaded this handle will have been auto unloaded
+        if (Model.IsClosed)
+        {
+            Loaded = false;
+            return;
+        }
+
+        // Unload
+        Loaded = false;
+        llama_adapter_lora_free(Pointer);
+
+        // Manually free a LoRA adapter. loaded adapters which have not been
+        // freed will be automatically freed when the associated model is deleted
+        [DllImport(NativeApi.libraryName, CallingConvention = CallingConvention.Cdecl)]
+        static extern void llama_adapter_lora_free(IntPtr adapter);
     }
 }

LLama/Native/NativeApi.Mtmd.cs

@@ -41,24 +41,50 @@ internal struct mtmd_context_params
     [DllImport(mtmdLibraryName, EntryPoint = "mtmd_context_params_default", CallingConvention = CallingConvention.Cdecl)]
     internal static extern mtmd_context_params mtmd_context_params_default();
 
+    /// <summary>
+    /// whether we need to set non-causal mask before llama_decode
+    /// if chunk is nullptr, we assume the default case where chunk is an image chunk
+    /// </summary>
+    /// <param name="ctx"></param>
+    /// <returns></returns>
     [DllImport(mtmdLibraryName, EntryPoint = "mtmd_decode_use_non_causal", CallingConvention = CallingConvention.Cdecl)]
     [return: MarshalAs(UnmanagedType.I1)]
     internal static extern bool mtmd_decode_use_non_causal(SafeMtmdModelHandle ctx);
 
+    /// <summary>
+    /// whether the current model use M-RoPE for llama_decode
+    /// </summary>
+    /// <param name="ctx"></param>
+    /// <returns></returns>
     [DllImport(mtmdLibraryName, EntryPoint = "mtmd_decode_use_mrope", CallingConvention = CallingConvention.Cdecl)]
     [return: MarshalAs(UnmanagedType.I1)]
     internal static extern bool mtmd_decode_use_mrope(SafeMtmdModelHandle ctx);
 
+    /// <summary>
+    /// whether the current model supports vision input
+    /// </summary>
+    /// <param name="ctx"></param>
+    /// <returns></returns>
     [DllImport(mtmdLibraryName, EntryPoint = "mtmd_support_vision", CallingConvention = CallingConvention.Cdecl)]
     [return: MarshalAs(UnmanagedType.I1)]
     internal static extern bool mtmd_support_vision(SafeMtmdModelHandle ctx);
 
+    /// <summary>
+    /// whether the current model supports audio input
+    /// </summary>
+    /// <param name="ctx"></param>
+    /// <returns></returns>
     [DllImport(mtmdLibraryName, EntryPoint = "mtmd_support_audio", CallingConvention = CallingConvention.Cdecl)]
     [return: MarshalAs(UnmanagedType.I1)]
     internal static extern bool mtmd_support_audio(SafeMtmdModelHandle ctx);
 
-    [DllImport(mtmdLibraryName, EntryPoint = "mtmd_get_audio_bitrate", CallingConvention = CallingConvention.Cdecl)]
-    internal static extern int mtmd_get_audio_bitrate(SafeMtmdModelHandle ctx);
+    /// <summary>
+    /// get audio sample rate in Hz, for example 16000 for Whisper
+    /// </summary>
+    /// <param name="ctx"></param>
+    /// <returns></returns>
+    [DllImport(mtmdLibraryName, EntryPoint = "mtmd_get_audio_sample_rate", CallingConvention = CallingConvention.Cdecl)]
+    internal static extern int mtmd_get_audio_sample_rate(SafeMtmdModelHandle ctx);
 
     // bitmap ------------------------------------------------------------
 
@@ -153,9 +179,11 @@ internal static unsafe void mtmd_bitmap_set_id(SafeMtmdEmbed bitmap, string? id)
     [DllImport(mtmdLibraryName, EntryPoint = "mtmd_image_tokens_get_n_tokens", CallingConvention = CallingConvention.Cdecl)]
     internal static extern UIntPtr mtmd_image_tokens_get_n_tokens(IntPtr image_tokens);
 
+    [Obsolete("use mtmd_image_tokens_get_decoder_pos() instead")]
     [DllImport(mtmdLibraryName, EntryPoint = "mtmd_image_tokens_get_nx", CallingConvention = CallingConvention.Cdecl)]
     internal static extern UIntPtr mtmd_image_tokens_get_nx(IntPtr image_tokens);
 
+    [Obsolete("use mtmd_image_tokens_get_decoder_pos() instead")]
     [DllImport(mtmdLibraryName, EntryPoint = "mtmd_image_tokens_get_ny", CallingConvention = CallingConvention.Cdecl)]
     internal static extern UIntPtr mtmd_image_tokens_get_ny(IntPtr image_tokens);
 
@@ -165,6 +193,28 @@ internal static unsafe void mtmd_bitmap_set_id(SafeMtmdEmbed bitmap, string? id)
     [DllImport(mtmdLibraryName, EntryPoint = "mtmd_image_tokens_get_n_pos", CallingConvention = CallingConvention.Cdecl)]
     internal static extern int mtmd_image_tokens_get_n_pos(IntPtr image_tokens);
 
+    [StructLayout(LayoutKind.Explicit)]
+    internal struct mtmd_decoder_pos
+    {
+        [FieldOffset(0)]
+        uint t;
+
+        [FieldOffset(4)]
+        uint x;
+
+        [FieldOffset(8)]
+        uint y;
+    };
+
+    /// <summary>
+    /// get position for decoder attention, to be used by M-RoPE models
+    /// </summary>
+    /// <param name="image_tokens"></param>
+    /// <param name="i">i is the index of the embedding token, ranging from 0 to mtmd_image_tokens_get_n_tokens() - 1</param>
+    /// <returns>return relative position (for example, embedding 0 will have position (0, 0, 0); remember to adjust it to the current absolute position)</returns>
+    [DllImport(mtmdLibraryName, EntryPoint = "mtmd_image_tokens_get_decoder_pos", CallingConvention = CallingConvention.Cdecl)]
+    internal static extern mtmd_decoder_pos mtmd_image_tokens_get_decoder_pos(IntPtr image_tokens, nuint i);
+
     // tokenize ----------------------------------------------------------
 
     /// <summary>
@@ -259,6 +309,11 @@ internal static unsafe IntPtr mtmd_helper_bitmap_init_from_file(SafeMtmdModelHan
     [DllImport(mtmdLibraryName, EntryPoint = "mtmd_helper_get_n_pos", CallingConvention = CallingConvention.Cdecl)]
     internal static extern int mtmd_helper_get_n_pos(SafeMtmdInputChunks chunks);
 
+    [DllImport(mtmdLibraryName, EntryPoint = "mtmd_helper_image_get_decoder_pos", CallingConvention = CallingConvention.Cdecl)]
+    // helper to get the list of relative positions corresponding to the embedding tokens, to be used by M-RoPE
+    // out_pos must have length == mtmd_helper_get_n_tokens(image)
+    internal static extern void mtmd_helper_image_get_decoder_pos(IntPtr /* mtmd_image_tokens* */ image, IntPtr /* mtmd_decoder_pos* */ out_pos);
+
     [DllImport(mtmdLibraryName, EntryPoint = "mtmd_helper_eval_chunks", CallingConvention = CallingConvention.Cdecl)]
     internal static extern int mtmd_helper_eval_chunks(
         SafeMtmdModelHandle ctx,