init

Author: jim3692

README.md

@@ -0,0 +1,5 @@
+# KoboldCpp Vulkan Flake
+
+Run `nix run github:jim3692/koboldcpp-flake`
+
+It downloads [Meta-Llama-3.1-8B-Instruct-Q4_K_M.gguf](https://huggingface.co/lmstudio-community/Meta-Llama-3.1-8B-Instruct-GGUF) using `huggingface-cli`, if it doesn't already exist in `~/.cache/huggingface`, and then starts KoboldCpp in Vulkan mode.

flake.lock

@@ -0,0 +1,35 @@
+{
+  inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable";
+
+  outputs = { nixpkgs, ... }:
+    let
+      system = "x86_64-linux";
+      pkgs = import nixpkgs { inherit system; };
+
+      repoName = "lmstudio-community/Meta-Llama-3.1-8B-Instruct-GGUF";
+      fileName = "Meta-Llama-3.1-8B-Instruct-Q4_K_M.gguf";
+    in {
+      apps.${system}.default = let
+        run = pkgs.writeShellApplication {
+          name = "run";
+          text = with pkgs; ''
+            findModel() {
+              find ~/.cache/huggingface -name "${fileName}" | head -n1
+            }
+
+            if [ "$(findModel)" == "" ]; then
+              ${python312Packages.huggingface-hub}/bin/huggingface-cli download "${repoName}" "${fileName}"
+            fi
+
+            modelPath="$(findModel)"
+            echo "Found ${fileName} at $modelPath"
+            AMD_VULKAN_ICD=AMDVLK ${koboldcpp}/bin/koboldcpp --usevulkan --model "$modelPath" --gpulayers 100
+          '';
+        };
+      in {
+        type = "app";
+        program = "${run}/bin/run";
+      };
+    };
+}
+