feat: Exclude Top Choices (XTC) (#553)

* feat: Exclude Top Choices (XTC) support
* feat: DRY (Don't Repeat Yourself) repeat penalty support
* feat: Tiny Aya support
* fix: adjust the default VRAM padding config to reserve enough memory for compute buffers
* fix: adapt to breaking `llama.cpp` changes
* fix: support function call syntax with optional whitespace prefix
* fix: find the provided cmake path
* fix: change the default value of `useDirectIo` to `false`
* fix: Vulkan device dedupe

Author: giladgd

.github/workflows/build.yml

@@ -168,18 +168,30 @@ jobs:
         env:
           VULKAN_VERSION: 1.4.313.2
         run: |
+          # $env:VULKAN_VERSION = & curl.exe -fsSL https://vulkan.lunarg.com/sdk/latest/linux.txt
+          # Write-Host "Downloading Vulkan SDK version $env:VULKAN_VERSION"
+          
           curl.exe -o $env:RUNNER_TEMP/VulkanSDK-Installer.exe -L "https://sdk.lunarg.com/sdk/download/${env:VULKAN_VERSION}/windows/vulkansdk-windows-X64-${env:VULKAN_VERSION}.exe"
           & "$env:RUNNER_TEMP\VulkanSDK-Installer.exe" --accept-licenses --default-answer --confirm-command install
           Add-Content $env:GITHUB_ENV "VULKAN_SDK=C:\VulkanSDK\${env:VULKAN_VERSION}"
           Add-Content $env:GITHUB_PATH "C:\VulkanSDK\${env:VULKAN_VERSION}\bin"
 
       - name: Install Vulkan SDK on Ubuntu
         if: matrix.config.name == 'Ubuntu (1)'
+        env:
+          VULKAN_VERSION: 1.4.341.1
         run: |
-          wget -qO- https://packages.lunarg.com/lunarg-signing-key-pub.asc | sudo tee /etc/apt/trusted.gpg.d/lunarg.asc
-          sudo wget -qO /etc/apt/sources.list.d/lunarg-vulkan-jammy.list https://packages.lunarg.com/vulkan/lunarg-vulkan-jammy.list
-          sudo apt update
-          sudo apt install vulkan-sdk
+          # export VULKAN_VERSION="$(curl -fsSL https://vulkan.lunarg.com/sdk/latest/linux.txt)"
+          # echo "Downloading Vulkan SDK version ${VULKAN_VERSION}"
+          
+          sudo apt-get update
+          sudo apt-get install xz-utils
+          curl --no-progress-meter "https://sdk.lunarg.com/sdk/download/${VULKAN_VERSION}/linux/vulkan_sdk.tar.xz" -o "/opt/vulkan-sdk.tar.xz"
+          mkdir -p "/opt/vulkan-sdk"
+          tar -Jxf "/opt/vulkan-sdk.tar.xz" --strip-components=1 -C "/opt/vulkan-sdk"
+          rm -f "/opt/vulkan-sdk.tar.xz"
+          echo "VULKAN_SDK=/opt/vulkan-sdk/x86_64" >> $GITHUB_ENV
+          echo "/opt/vulkan-sdk/x86_64/bin" >> $GITHUB_PATH
 
       - name: Install dependencies on macOS
         if: matrix.config.name == 'macOS x64' || matrix.config.name == 'macOS arm64'

.github/workflows/prLint.yml

@@ -6,11 +6,11 @@ on:
 
 jobs:
   lint:
-    name: Lint
-    runs-on: ubuntu-latest
+    name: Validate PR title
+    runs-on: ubuntu-slim
     permissions:
       pull-requests: read
     steps:
-      - uses: amannn/action-semantic-pull-request@v5
+      - uses: amannn/action-semantic-pull-request@v6
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.vitepress/config.ts

@@ -387,6 +387,9 @@ export default defineConfig({
         },
         codeTransformers: [
             transformerTwoslash({
+                floatingVue: {
+                    classFloatingPanel: "twoslash-floating vp-code"
+                },
                 explicitTrigger: false,
                 filter(lang, code, options) {
                     return options.lang?.toLowerCase() === "typescript";

.vitepress/theme/style.css

@@ -256,13 +256,19 @@ img.blog-coverImage {
     transform: scaleX(1);
 }
 
-.v-popper--theme-dropdown .v-popper__inner {
+.v-popper--theme-dropdown .v-popper__wrapper {
     box-shadow: 0 6px 30px 0px rgb(0 0 0 / 32%);
     border-radius: 12px;
+    background-color: var(--twoslash-popup-bg);
+}
+.v-popper--theme-dropdown .v-popper__inner {
+    box-shadow: none;
+    border-radius: 12px;
 }
 
-.twoslash-floating {
+.twoslash-floating[class][class] {
     --twoslash-border-color: color-mix(in srgb, var(--vp-c-border), transparent 64%);
+    content-visibility: initial;
 }
 .twoslash-floating .v-popper__inner {
     border: none;

docs/guide/Vulkan.md

@@ -41,23 +41,25 @@ If you see `Vulkan used VRAM` in the output, it means that Vulkan support is wor
   #### Windows: [Vulkan SDK installer](https://sdk.lunarg.com/sdk/download/latest/windows/vulkan-sdk.exe) {#vulkan-sdk-windows}
   >
   #### Ubuntu {#vulkan-sdk-ubuntu}
-  ::: code-group
-  
-  ```shell [Ubuntu 24.04]
-  wget -qO- https://packages.lunarg.com/lunarg-signing-key-pub.asc | sudo tee /etc/apt/trusted.gpg.d/lunarg.asc
-  sudo wget -qO /etc/apt/sources.list.d/lunarg-vulkan-noble.list https://packages.lunarg.com/vulkan/lunarg-vulkan-noble.list
+  ```shell
   sudo apt update
-  sudo apt install vulkan-sdk
-  ```
+  sudo apt install curl wget xz-utils
 
-  ```shell [Ubuntu 22.04]
-  wget -qO- https://packages.lunarg.com/lunarg-signing-key-pub.asc | sudo tee /etc/apt/trusted.gpg.d/lunarg.asc
-  sudo wget -qO /etc/apt/sources.list.d/lunarg-vulkan-jammy.list https://packages.lunarg.com/vulkan/lunarg-vulkan-jammy.list
-  sudo apt update
-  sudo apt install vulkan-sdk
-  ```
+  export VULKAN_VERSION="$(curl -fsSL https://vulkan.lunarg.com/sdk/latest/linux.txt)"                                                                                      
 
-  :::
+  echo "Downloading Vulkan SDK version ${VULKAN_VERSION}"
+  curl --progress-bar "https://sdk.lunarg.com/sdk/download/${VULKAN_VERSION}/linux/vulkan_sdk.tar.xz" -o "/opt/vulkan-sdk.tar.xz"
+  
+  echo "Installing Vulkan SDK to /opt/vulkan-sdk"
+  rm -rf "/opt/vulkan-sdk" && mkdir -p "/opt/vulkan-sdk"
+  tar -Jxf "/opt/vulkan-sdk.tar.xz" --strip-components=1 -C "/opt/vulkan-sdk"
+  rm -f "/opt/vulkan-sdk.tar.xz"
+  
+  echo "Adding Vulkan SDK environment variables to shell profiles"
+  ([ ! -f "$HOME/.bashrc" ] || grep -qxF "source /opt/vulkan-sdk/setup-env.sh" "$HOME/.bashrc") || (echo "source /opt/vulkan-sdk/setup-env.sh" >> "$HOME/.bashrc")
+  ([ ! -f "$HOME/.zshrc" ] || grep -qxF "source /opt/vulkan-sdk/setup-env.sh" "$HOME/.zshrc") || (echo "source /opt/vulkan-sdk/setup-env.sh" >> "$HOME/.zshrc")
+  source /opt/vulkan-sdk/setup-env.sh
+  ```
 
 * :::details Windows only: enable long paths support
   Open cmd as Administrator and run this command:

eslint.config.js

@@ -8,12 +8,13 @@ import n from "eslint-plugin-n";
 import tseslint from "typescript-eslint";
 import stylistic from "@stylistic/eslint-plugin";
 import {includeIgnoreFile} from "@eslint/compat";
+import {defineConfig} from "eslint/config";
 
 const __dirname = path.dirname(fileURLToPath(import.meta.url));
 const gitignorePath = path.join(__dirname, ".gitignore");
 
 
-export default tseslint.config({
+export default defineConfig({
     ignores: [
         "dist/",
         "**/dist/",
@@ -25,12 +26,12 @@ export default tseslint.config({
         "packages/create-node-llama-cpp/dist/",
         "packages/@node-llama-cpp/*/dist/",
         ".releaserc.ts",
-        ...includeIgnoreFile(gitignorePath).ignores
+        ...(includeIgnoreFile(gitignorePath).ignores ?? [])
     ]
 }, {
     files: ["**/**.{,c,m}{js,ts}"],
     extends: [
-        stylistic.configs["recommended-flat"],
+        stylistic.configs["recommended"],
         jsdoc.configs["flat/recommended"],
         importPlugin.flatConfigs.recommended
     ],
@@ -61,7 +62,7 @@ export default tseslint.config({
         }
     },
     rules: {
-        "@stylistic/indent": ["off"],
+        "@stylistic/indent": ["warn", 4],
         "indent": ["warn", 4, {
             SwitchCase: 1,
             FunctionDeclaration: {
@@ -80,7 +81,8 @@ export default tseslint.config({
             args: "none",
             ignoreRestSiblings: true,
             varsIgnorePattern: "^set",
-            caughtErrors: "none"
+            caughtErrors: "none",
+            ignoreUsingDeclarations: true
         }],
         "@stylistic/no-prototype-builtins": ["off"],
         "@stylistic/object-curly-spacing": ["warn", "never"],
@@ -174,7 +176,8 @@ export default tseslint.config({
             args: "none",
             ignoreRestSiblings: true,
             varsIgnorePattern: "^set",
-            caughtErrors: "none"
+            caughtErrors: "none",
+            ignoreUsingDeclarations: true
         }],
         "@typescript-eslint/no-empty-object-type": ["off"],
         "@typescript-eslint/member-ordering": ["warn", {

llama/addon/AddonContext.cpp

@@ -942,11 +942,25 @@ Napi::Value AddonContext::EnsureDraftContextIsCompatibleForSpeculative(const Nap
     return info.Env().Undefined();
 }
 
-Napi::Value AddonContext::SetLora(const Napi::CallbackInfo& info) {
-    AddonModelLora* lora = Napi::ObjectWrap<AddonModelLora>::Unwrap(info[0].As<Napi::Object>());
-    float scale = info[1].As<Napi::Number>().FloatValue();
+Napi::Value AddonContext::SetLoras(const Napi::CallbackInfo& info) {
+    Napi::Array loraArray = info[0].As<Napi::Array>();
+    Napi::Array scaleArray = info[1].As<Napi::Array>();
 
-    llama_set_adapter_lora(ctx, lora->lora_adapter, scale);
+    std::vector<llama_adapter_lora *> loras;
+    std::vector<float> scales;
+
+    loras.reserve(loraArray.Length());
+    scales.reserve(scaleArray.Length());
+
+    for (size_t i = 0; i < loraArray.Length() && i < scaleArray.Length(); i++) {
+        AddonModelLora* lora = Napi::ObjectWrap<AddonModelLora>::Unwrap(loraArray.Get(i).As<Napi::Object>());
+        float scale = scaleArray.Get(i).As<Napi::Number>().FloatValue();
+
+        loras.push_back(lora->lora_adapter);
+        scales.push_back(scale);
+    }
+
+    llama_set_adapters_lora(ctx, loras.data(), loras.size(), scales.data());
 
     return info.Env().Undefined();
 }
@@ -977,7 +991,7 @@ void AddonContext::init(Napi::Object exports) {
                 InstanceMethod("ensureDraftContextIsCompatibleForSpeculative", &AddonContext::EnsureDraftContextIsCompatibleForSpeculative),
                 InstanceMethod("saveSequenceStateToFile", &AddonContext::SaveSequenceStateToFile),
                 InstanceMethod("loadSequenceStateFromFile", &AddonContext::LoadSequenceStateFromFile),
-                InstanceMethod("setLora", &AddonContext::SetLora),
+                InstanceMethod("setLoras", &AddonContext::SetLoras),
                 InstanceMethod("dispose", &AddonContext::Dispose),
             }
         )

llama/addon/AddonContext.h

@@ -52,7 +52,7 @@ class AddonContext : public Napi::ObjectWrap<AddonContext> {
         Napi::Value PrintTimings(const Napi::CallbackInfo& info);
         Napi::Value EnsureDraftContextIsCompatibleForSpeculative(const Napi::CallbackInfo& info);
 
-        Napi::Value SetLora(const Napi::CallbackInfo& info);
+        Napi::Value SetLoras(const Napi::CallbackInfo& info);
 
         static void init(Napi::Object exports);
 };