fix: unkown characters in generation streaming (#293)

Author: giladgd

src/evaluator/LlamaChat/LlamaChat.ts

@@ -2215,7 +2215,7 @@ class GenerateResponseState<const Functions extends ChatModelFunctions | undefin
     }
 
     public waitOnPartialCharactersOrWhiteSpaceTokens() {
-        if (this.currentText === UNKNOWN_UNICODE_CHAR || (
+        if (this.currentText.endsWith(UNKNOWN_UNICODE_CHAR) || (
             (this.grammar?.trimWhitespaceSuffix || this.trimWhitespaceSuffix) && this.currentText?.trim() === ""
         )) {
             if (this.currentQueuedTokenRelease != null)

src/evaluator/LlamaCompletion.ts

@@ -662,7 +662,7 @@ export class LlamaCompletion {
                 const text = model.detokenize([token]);
                 const queuedTokenRelease = streamRegulator.addChunk({tokens, text});
 
-                if (text === UNKNOWN_UNICODE_CHAR || (
+                if (text.endsWith(UNKNOWN_UNICODE_CHAR) || (
                     (grammar?.trimWhitespaceSuffix || trimWhitespaceSuffix) && text.trim() === ""
                 )) {
                     locksToReleaseOnValidGeneration.push(queuedTokenRelease.createTextIndexLock(0));