feat: initial disk cache dir option for future optimizations (disabled for now)

Author: giladgd

src/bindings/Llama.ts

@@ -10,6 +10,7 @@ import {LlamaJsonSchemaGrammar} from "../evaluator/LlamaJsonSchemaGrammar.js";
 import {LlamaGrammar, LlamaGrammarOptions} from "../evaluator/LlamaGrammar.js";
 import {ThreadsSplitter} from "../utils/ThreadsSplitter.js";
 import {getLlamaClasses, LlamaClasses} from "../utils/getLlamaClasses.js";
+import {getTempDir, FsPathHandle} from "../utils/getTempDir.js";
 import {BindingModule} from "./AddonTypes.js";
 import {
     BuildGpu, BuildMetadataFile, LlamaGpuType, LlamaLocks, LlamaLogLevel,
@@ -46,6 +47,8 @@ export class Llama {
     /** @internal */ public readonly _swapOrchestrator: MemoryOrchestrator;
     /** @internal */ public readonly _debug: boolean;
     /** @internal */ public readonly _threadsSplitter: ThreadsSplitter;
+    /** @internal */ public readonly _tempDir?: FsPathHandle;
+    /** @internal */ private _tempDirNextId: number = 0;
     /** @internal */ public _hadErrorLogs: boolean = false;
     /** @internal */ private readonly _gpu: LlamaGpuType;
     /** @internal */ private readonly _numa: LlamaNuma;
@@ -74,7 +77,7 @@ export class Llama {
     public readonly onDispose = new EventRelay<void>();
 
     private constructor({
-        bindings, bindingPath, extBackendsPath, logLevel, logger, buildType, cmakeOptions, llamaCppRelease, debug, numa, buildGpu,
+        bindings, bindingPath, extBackendsPath, logLevel, logger, buildType, cmakeOptions, llamaCppRelease, debug, tempDir, numa, buildGpu,
         maxThreads, vramOrchestrator, vramPadding, ramOrchestrator, ramPadding, swapOrchestrator, skipLlamaInit
     }: {
         bindings: BindingModule,
@@ -89,6 +92,7 @@ export class Llama {
             release: string
         },
         debug: boolean,
+        tempDir?: FsPathHandle,
         numa?: LlamaNuma,
         buildGpu: BuildGpu,
         maxThreads?: number,
@@ -104,6 +108,7 @@ export class Llama {
 
         this._bindings = bindings;
         this._debug = debug;
+        this._tempDir = tempDir;
         this._numa = numa ?? false;
         this._logLevel = this._debug
             ? LlamaLogLevel.debug
@@ -175,6 +180,7 @@ export class Llama {
         this.onDispose.dispatchEvent();
         await this._backendDisposeGuard.acquireDisposeLock();
         await this._bindings.dispose();
+        await this._tempDir?.dispose();
 
         process.off("beforeExit", this._onBeforeExit);
         unregisterDisposeBeforeExit(this._selfWeakRef);
@@ -413,6 +419,15 @@ export class Llama {
         this._onAddonLog(LlamaLogLevelToAddonLogLevel.get(level) ?? defaultLogLevel, message + "\n");
     }
 
+    /** @internal */
+    public _createTempFilePath() {
+        if (this._tempDir == null)
+            return undefined;
+
+        const fileId = this._tempDirNextId++;
+        return new FsPathHandle(path.join(this._tempDir.path, fileId + ".nlc"));
+    }
+
     /** @internal */
     private _onAddonLog(level: number, message: string) {
         const llamaLogLevel = addonLogLevelToLlamaLogLevel.get(level) ?? LlamaLogLevel.fatal;
@@ -507,7 +522,7 @@ export class Llama {
     /** @internal */
     public static async _create({
         bindings, bindingPath, extBackendsPath, buildType, buildMetadata, logLevel, logger, vramPadding, ramPadding, maxThreads,
-        skipLlamaInit = false, debug, numa
+        skipLlamaInit = false, debug, numa, tempDir
     }: {
         bindings: BindingModule,
         bindingPath: string,
@@ -521,7 +536,8 @@ export class Llama {
         ramPadding: number | ((totalRam: number) => number),
         skipLlamaInit?: boolean,
         debug: boolean,
-        numa?: LlamaNuma
+        numa?: LlamaNuma,
+        tempDir?: string | string[] | false
     }) {
         const vramOrchestrator = new MemoryOrchestrator(() => {
             const {total, used, unifiedSize} = bindings.getGpuVramInfo();
@@ -566,6 +582,14 @@ export class Llama {
         else
             resolvedRamPadding = ramOrchestrator.reserveMemory(ramPadding);
 
+        const resolvedTempDir = tempDir === false
+            ? undefined
+            : await getTempDir(
+                typeof tempDir === "string"
+                    ? [tempDir]
+                    : tempDir
+            );
+
         const llama = new Llama({
             bindings,
             bindingPath,
@@ -579,6 +603,7 @@ export class Llama {
             logLevel,
             logger,
             debug,
+            tempDir: resolvedTempDir,
             numa,
             buildGpu: buildMetadata.buildOptions.gpu,
             vramOrchestrator,

src/bindings/getLlama.ts

@@ -208,7 +208,27 @@ export type LlamaOptions = {
      *
      * Defaults to `false` (no NUMA policy).
      */
-    numa?: LlamaNuma
+    numa?: LlamaNuma,
+
+    /**
+     * A list of directories to use for internal cache to speed up processes and reduce memory usage.
+     * Cache created by this process will be automatically cleaned up when it's no longer needed,
+     * on process exit, and on the disposal of the Llama instance.
+     *
+     * The first directory that is writable from this list will be used.
+     *
+     * If no directories are provided or non are writable, these fallback will be used instead:
+     * - The directory of where `node-llama-cpp` is installed inside `node_modules`.
+     * - A `nlc.` prefixed directory under the OS's temp dir (e.g. `/tmp` on Linux).
+     * - A `.node-llama-cpp/.temp` directory under the home directory of the current user.
+     *
+     * To disable the usage of any temporary directory and caching, set this option to `false`.
+     *
+     *
+     * Hidden since currently unused - defaults to `false` for now
+     * @internal
+     */
+    tempDir?: string | string[] | false
 };
 
 export type LastBuildOptions = {
@@ -318,7 +338,27 @@ export type LastBuildOptions = {
      *
      * Defaults to `false` (no NUMA policy).
      */
-    numa?: LlamaNuma
+    numa?: LlamaNuma,
+
+    /**
+     * A list of directories to use for internal cache to speed up processes and reduce memory usage.
+     * Cache created by this process will be automatically cleaned up when it's no longer needed,
+     * on process exit, and on the disposal of the Llama instance.
+     *
+     * The first directory that is writable from this list will be used.
+     *
+     * If no directories are provided or non are writable, these fallback will be used instead:
+     * - The directory of where `node-llama-cpp` is installed inside `node_modules`.
+     * - A `nlc.` prefixed directory under the OS's temp dir (e.g. `/tmp` on Linux).
+     * - A `.node-llama-cpp/.temp` directory under the home directory of the current user.
+     *
+     * To disable the usage of any temporary directory and caching, set this option to `false`.
+     *
+     *
+     * Hidden since currently unused - defaults to `false` for now
+     * @internal
+     */
+    tempDir?: string | string[] | false
 };
 
 export const getLlamaFunctionName = "getLlama";
@@ -335,6 +375,7 @@ export const defaultLlamaRamPadding = (totalRam: number) => {
 const defaultBuildOption: Exclude<LlamaOptions["build"], undefined> = runningInElectron
     ? "never"
     : "auto";
+const defaultTempDir: LlamaOptions["tempDir"] = false;
 
 /**
  * Get a `llama.cpp` binding.
@@ -377,6 +418,7 @@ export async function getLlama(options?: LlamaOptions | "lastBuild", lastBuildOp
             ramPadding: lastBuildOptions?.ramPadding ?? defaultLlamaRamPadding,
             debug: lastBuildOptions?.debug ?? defaultLlamaCppDebugMode,
             numa: lastBuildOptions?.numa,
+            tempDir: lastBuildOptions?.tempDir ?? defaultTempDir,
             dryRun
         };
 
@@ -405,6 +447,7 @@ export async function getLlama(options?: LlamaOptions | "lastBuild", lastBuildOp
                     ramPadding: lastBuildOptions?.ramPadding ?? defaultLlamaRamPadding,
                     debug: lastBuildOptions?.debug ?? defaultLlamaCppDebugMode,
                     numa: lastBuildOptions?.numa,
+                    tempDir: lastBuildOptions?.tempDir ?? defaultTempDir,
                     skipLlamaInit: dryRun
                 });
 
@@ -440,6 +483,7 @@ export async function getLlamaForOptions({
     ramPadding = defaultLlamaRamPadding,
     debug = defaultLlamaCppDebugMode,
     numa = false,
+    tempDir = defaultTempDir,
     dryRun = false
 }: LlamaOptions, {
     updateLastBuildInfoOnCompile = false,
@@ -514,6 +558,7 @@ export async function getLlamaForOptions({
                     ramPadding,
                     debug,
                     numa,
+                    tempDir,
                     dryRun
                 });
             } catch (err) {
@@ -532,6 +577,7 @@ export async function getLlamaForOptions({
                     ramPadding,
                     debug,
                     numa,
+                    tempDir,
                     dryRun
                 });
             }
@@ -579,6 +625,7 @@ export async function getLlamaForOptions({
                     ),
                 debug,
                 numa,
+                tempDir,
                 pipeBinaryTestErrorLogs
             });
 
@@ -614,7 +661,8 @@ export async function getLlamaForOptions({
                         ramPadding,
                         skipLlamaInit,
                         debug,
-                        numa
+                        numa,
+                        tempDir
                     });
                 } catch (err) {
                     console.error(
@@ -691,7 +739,8 @@ export async function getLlamaForOptions({
                 ramPadding,
                 skipLlamaInit,
                 debug,
-                numa
+                numa,
+                tempDir
             });
         } catch (err) {
             console.error(
@@ -737,6 +786,7 @@ async function loadExistingLlamaBinary({
     fallbackMessage,
     debug,
     numa,
+    tempDir,
     pipeBinaryTestErrorLogs
 }: {
     buildOptions: BuildOptions,
@@ -754,6 +804,7 @@ async function loadExistingLlamaBinary({
     fallbackMessage: string | null,
     debug: boolean,
     numa?: LlamaNuma,
+    tempDir: LlamaOptions["tempDir"],
     pipeBinaryTestErrorLogs: boolean
 }) {
     const buildFolderName = await getBuildFolderNameForBuildOptions(buildOptions);
@@ -791,7 +842,8 @@ async function loadExistingLlamaBinary({
                     ramPadding,
                     skipLlamaInit,
                     debug,
-                    numa
+                    numa,
+                    tempDir
                 });
             } else if (progressLogs) {
                 console.warn(
@@ -857,7 +909,8 @@ async function loadExistingLlamaBinary({
                         ramPadding,
                         skipLlamaInit,
                         debug,
-                        numa
+                        numa,
+                        tempDir
                     });
                 } else if (progressLogs) {
                     const binaryDescription = describeBinary({
@@ -913,7 +966,8 @@ async function buildAndLoadLlamaBinary({
     ramPadding,
     skipLlamaInit,
     debug,
-    numa
+    numa,
+    tempDir
 }: {
     buildOptions: BuildOptions,
     skipDownload: boolean,
@@ -925,7 +979,8 @@ async function buildAndLoadLlamaBinary({
     ramPadding: Required<LlamaOptions>["ramPadding"],
     skipLlamaInit: boolean,
     debug: boolean,
-    numa?: LlamaNuma
+    numa?: LlamaNuma,
+    tempDir: LlamaOptions["tempDir"]
 }) {
     const buildFolderName = await getBuildFolderNameForBuildOptions(buildOptions);
 
@@ -960,7 +1015,8 @@ async function buildAndLoadLlamaBinary({
         ramPadding,
         skipLlamaInit,
         debug,
-        numa
+        numa,
+        tempDir
     });
 }
 

src/config.ts

@@ -20,11 +20,13 @@ export const llamaPrebuiltBinsDirectory = path.join(__dirname, "..", "bins");
 export const llamaLocalBuildBinsDirectory = path.join(llamaDirectory, "localBuilds");
 export const llamaBinsGrammarsDirectory = path.join(__dirname, "..", "llama", "grammars");
 export const projectTemplatesDirectory = path.join(__dirname, "..", "templates");
+export const localTempDirectory = path.join(__dirname, "..", ".temp");
 export const packedProjectTemplatesDirectory = path.join(projectTemplatesDirectory, "packed");
 export const llamaCppDirectory = path.join(llamaDirectory, "llama.cpp");
 export const llamaCppGrammarsDirectory = path.join(llamaDirectory, "llama.cpp", "grammars");
 export const tempDownloadDirectory = path.join(os.tmpdir(), "node-llama-cpp", nanoid());
 export const cliHomedirDirectory = path.join(os.homedir(), ".node-llama-cpp");
+export const cliHomedirTempDirectory = path.join(os.homedir(), ".node-llama-cpp", ".temp");
 export const chatCommandHistoryFilePath = path.join(cliHomedirDirectory, ".chat_repl_history");
 export const cliModelsDirectory = path.join(cliHomedirDirectory, "models");
 export const lastBuildInfoJsonPath = path.join(llamaDirectory, "lastBuild.json");

src/utils/getFirstWritableDir.ts

@@ -0,0 +1,72 @@
+import path from "path";
+import fs from "fs-extra";
+import {runningInElectron} from "./runtime.js";
+import {LruCache} from "./LruCache.js";
+
+export async function getFirstWritableDir(dirPaths: string[]): Promise<string | null> {
+    for (const dirPath of dirPaths) {
+        if (isPathInsideAsar(dirPath))
+            continue;
+
+        if (await isPathWritable(dirPath))
+            return dirPath;
+    }
+
+    return null;
+}
+
+const writableCheckCache = new LruCache<string, boolean>(20);
+export async function isPathWritableWithCache(dirPath: string) {
+    const isWritable = writableCheckCache.get(dirPath) ?? await isPathWritable(dirPath);
+    writableCheckCache.set(dirPath, isWritable);
+    return isWritable;
+}
+
+export async function isPathWritable(dirPath: string) {
+    let checkPath = path.resolve(dirPath);
+
+    while (true) {
+        try {
+            const stat = await fs.lstat(checkPath);
+            if (!stat.isDirectory())
+                return false;
+
+            break;
+        } catch (error: any) {
+            if (error?.code !== "ENOENT")
+                return false;
+
+            const nextCheckPath = path.dirname(checkPath);
+            if (nextCheckPath === checkPath)
+                return false;
+
+            checkPath = nextCheckPath;
+        }
+    }
+
+    try {
+        await fs.access(path.dirname(checkPath), fs.constants.W_OK | fs.constants.X_OK);
+    } catch {
+        return false;
+    }
+
+    return true;
+}
+
+/**
+ * Check whether a path is inside an asar when running in Electron,
+ * which means that the path is not writable and inaccessible outside the Electron app.
+ */
+export function isPathInsideAsar(dirPath: string, excludeUnpacked: boolean = false) {
+    if (!runningInElectron)
+        return false;
+
+    const normalizedPath = dirPath.toLowerCase();
+    if (normalizedPath.endsWith(".asar") ||
+        (!excludeUnpacked && normalizedPath.endsWith(".asar.unpacked"))
+    )
+        return true;
+
+    return normalizedPath.includes(".asar" + path.sep) ||
+        (!excludeUnpacked && normalizedPath.includes(".asar.unpacked" + path.sep));
+}