chore: merge master into beta

Author: giladgd

llama/addon.cpp

@@ -559,6 +559,12 @@ class AddonContext : public Napi::ObjectWrap<AddonContext> {
             return result;
         }
 
+        Napi::Value PrintTimings(const Napi::CallbackInfo& info) {
+            llama_print_timings(ctx);
+            llama_reset_timings(ctx);
+            return info.Env().Undefined();
+        }
+
         static void init(Napi::Object exports) {
             exports.Set(
                 "AddonContext",
@@ -576,6 +582,7 @@ class AddonContext : public Napi::ObjectWrap<AddonContext> {
                         InstanceMethod("sampleToken", &AddonContext::SampleToken),
                         InstanceMethod("acceptGrammarEvaluationStateToken", &AddonContext::AcceptGrammarEvaluationStateToken),
                         InstanceMethod("getEmbedding", &AddonContext::GetEmbedding),
+                        InstanceMethod("printTimings", &AddonContext::PrintTimings),
                         InstanceMethod("dispose", &AddonContext::Dispose)
                     }
                 )

src/cli/commands/ChatCommand.ts

@@ -43,7 +43,8 @@ type ChatCommand = {
     repeatPresencePenalty?: number,
     maxTokens: number,
     noHistory: boolean,
-    environmentFunctions: boolean
+    environmentFunctions: boolean,
+    printTimings: boolean
 };
 
 export const ChatCommand: CommandModule<object, ChatCommand> = {
@@ -197,20 +198,27 @@ export const ChatCommand: CommandModule<object, ChatCommand> = {
                 default: false,
                 description: "Provide access to environment functions like `getDate` and `getTime`",
                 group: "Optional:"
+            })
+            .option("printTimings", {
+                alias: "pt",
+                type: "boolean",
+                default: false,
+                description: "Print llama.cpp timings after each response",
+                group: "Optional:"
             });
     },
     async handler({
         model, systemInfo, systemPrompt, prompt, wrapper, contextSize,
         grammar, jsonSchemaGrammarFile, threads, temperature, topK, topP,
         gpuLayers, repeatPenalty, lastTokensRepeatPenalty, penalizeRepeatingNewLine,
         repeatFrequencyPenalty, repeatPresencePenalty, maxTokens, noHistory,
-        environmentFunctions
+        environmentFunctions, printTimings
     }) {
         try {
             await RunChat({
                 model, systemInfo, systemPrompt, prompt, wrapper, contextSize, grammar, jsonSchemaGrammarFile, threads, temperature, topK,
                 topP, gpuLayers, lastTokensRepeatPenalty, repeatPenalty, penalizeRepeatingNewLine, repeatFrequencyPenalty,
-                repeatPresencePenalty, maxTokens, noHistory, environmentFunctions
+                repeatPresencePenalty, maxTokens, noHistory, environmentFunctions, printTimings
             });
         } catch (err) {
             console.error(err);
@@ -223,7 +231,8 @@ export const ChatCommand: CommandModule<object, ChatCommand> = {
 async function RunChat({
     model: modelArg, systemInfo, systemPrompt, prompt, wrapper, contextSize, grammar: grammarArg,
     jsonSchemaGrammarFile: jsonSchemaGrammarFilePath, threads, temperature, topK, topP, gpuLayers, lastTokensRepeatPenalty, repeatPenalty,
-    penalizeRepeatingNewLine, repeatFrequencyPenalty, repeatPresencePenalty, maxTokens, noHistory, environmentFunctions
+    penalizeRepeatingNewLine, repeatFrequencyPenalty, repeatPresencePenalty, maxTokens, noHistory, environmentFunctions,
+    printTimings
 }: ChatCommand) {
     const {LlamaChatSession} = await import("../../llamaEvaluator/LlamaChatSession/LlamaChatSession.js");
     const {LlamaModel} = await import("../../llamaEvaluator/LlamaModel.js");
@@ -370,6 +379,9 @@ async function RunChat({
         });
         process.stdout.write(endColor);
         console.log();
+
+        if (printTimings)
+            context.printTimings();
     }
 }
 

src/llamaEvaluator/LlamaContext/LlamaContext.ts

@@ -320,6 +320,10 @@ export class LlamaContext {
         });
     }
 
+    public printTimings() {
+        this._ctx.printTimings();
+    }
+
     /** @internal */
     public async _decodeTokens<T>({
         sequenceId, firstTokenSequenceIndex, tokens, generateLogitAtTheEnd = false, evaluationPriority = 5

src/utils/getBin.ts

@@ -174,8 +174,8 @@ export type AddonContext = {
     shiftSequenceTokenCells(sequenceId: number, startPos: number, endPos: number, shiftDelta: number): void,
 
     acceptGrammarEvaluationStateToken(grammarEvaluationState: AddonGrammarEvaluationState, token: Token): void,
-
-    getEmbedding(): Float64Array
+    getEmbedding(): Float64Array,
+    printTimings(): void
 };
 
 export type BatchLogitIndex = number & {