@@ -29,8 +29,8 @@ const __dirname = path.dirname(fileURLToPath(import.meta.url));
2929
3030const model = new LlamaModel ({
3131 modelPath: path .join (__dirname , " models" , " vicuna-13b-v1.5-16k.ggmlv3.q5_1.bin" )
32- })
33- const session = new LlamaChatSession ({model });
32+ });
33+ const session = new LlamaChatSession ({context: model . createContext () });
3434
3535
3636const q1 = " Hi there, how are you?" ;
@@ -73,7 +73,7 @@ const model = new LlamaModel({
7373 modelPath: path .join (__dirname , " models" , " vicuna-13b-v1.5-16k.ggmlv3.q5_1.bin" ),
7474 promptWrapper: new MyCustomChatPromptWrapper () // by default, LlamaChatPromptWrapper is used
7575})
76- const session = new LlamaChatSession ({model });
76+ const session = new LlamaChatSession ({context: model . createContext () });
7777
7878
7979const q1 = " Hi there, how are you?" ;
@@ -98,29 +98,31 @@ import {LlamaModel, LlamaChatSession} from "node-llama-cpp";
9898
9999const __dirname = path .dirname (fileURLToPath (import .meta .url ));
100100
101- const model = new LlamaChatSession ({
101+ const model = new LlamaModel ({
102102 modelPath: path .join (__dirname , " models" , " vicuna-13b-v1.5-16k.ggmlv3.q5_1.bin" )
103103});
104104
105+ const context = model .createContext ();
106+
105107const q1 = " Hi there, how are you?" ;
106108console .log (" AI: " + q1 );
107109
108- const tokens = model .encode (q1 );
110+ const tokens = context .encode (q1 );
109111const res: number [] = [];
110- for await (const chunk of model .evaluate (tokens )) {
112+ for await (const chunk of context .evaluate (tokens )) {
111113 res .push (chunk );
112114
113115 // it's important to not concatinate the results as strings,
114116 // as doing so will break some characters (like some emojis) that are made of multiple tokens.
115117 // by using an array of tokens, we can decode them correctly together.
116- const resString: string = model .decode (Uint32Array .from (res ));
118+ const resString: string = context .decode (Uint32Array .from (res ));
117119
118120 const lastPart = resString .split (" ASSISTANT:" ).reverse ()[0 ];
119121 if (lastPart .includes (" USER:" ))
120122 break ;
121123}
122124
123- const a1 = model .decode (Uint32Array .from (res )).split (" USER:" )[0 ];
125+ const a1 = context .decode (Uint32Array .from (res )).split (" USER:" )[0 ];
124126console .log (" AI: " + a1 );
125127```
126128
0 commit comments