fix to corrupted utf-8 encoding for special characters in transcription

Author: Mateusz Kopciński

packages/react-native-executorch/common/rnexecutorch/host_objects/JsiConversions.h

@@ -280,6 +280,15 @@ inline jsi::Value getJsiValue(const std::vector<int32_t> &vec,
   return {runtime, array};
 }
 
+inline jsi::Value getJsiValue(const std::vector<char> &vec,
+                              jsi::Runtime &runtime) {
+  jsi::Array array(runtime, vec.size());
+  for (size_t i = 0; i < vec.size(); i++) {
+    array.setValueAtIndex(runtime, i, jsi::Value(static_cast<char>(vec[i])));
+  }
+  return {runtime, array};
+}
+
 inline jsi::Value getJsiValue(int val, jsi::Runtime &runtime) {
   return {runtime, val};
 }

packages/react-native-executorch/common/rnexecutorch/models/speech_to_text/SpeechToText.cpp

@@ -1,6 +1,7 @@
 #include <thread>
 
 #include "SpeechToText.h"
+#include <vector>
 
 namespace rnexecutorch::models::speech_to_text {
 
@@ -41,7 +42,7 @@ SpeechToText::decode(std::span<int32_t> tokens,
   return std::make_shared<OwningArrayBuffer>(decoderOutput);
 }
 
-std::string SpeechToText::transcribe(std::span<float> waveform,
+std::vector<char> SpeechToText::transcribe(std::span<float> waveform,
                                      std::string languageOption) const {
   std::vector<Segment> segments =
       this->asr->transcribe(waveform, DecodingOptions(languageOption));
@@ -60,7 +61,9 @@ std::string SpeechToText::transcribe(std::span<float> waveform,
       transcription += word.content;
     }
   }
-  return transcription;
+
+  std::vector<char> charVector(transcription.begin(), transcription.end());
+  return charVector;
 }
 
 size_t SpeechToText::getMemoryLowerBound() const noexcept {
@@ -74,13 +77,13 @@ void SpeechToText::stream(std::shared_ptr<jsi::Function> callback,
     throw std::runtime_error("Streaming is already in progress");
   }
 
-  auto nativeCallback = [this, callback](const std::string &committed,
-                                         const std::string &nonCommitted,
+  auto nativeCallback = [this, callback](const std::vector<char> &committedVec,
+                                         const std::vector<char> &nonCommittedVec,
                                          bool isDone) {
     this->callInvoker->invokeAsync(
-        [callback, committed, nonCommitted, isDone](jsi::Runtime &rt) {
-          callback->call(rt, jsi::String::createFromUtf8(rt, committed),
-                         jsi::String::createFromUtf8(rt, nonCommitted),
+        [callback, committedVec, nonCommittedVec, isDone](jsi::Runtime &rt) {
+          callback->call(rt, rnexecutorch::jsi_conversion::getJsiValue(committedVec, rt),
+                         rnexecutorch::jsi_conversion::getJsiValue(nonCommittedVec, rt),
                          jsi::Value(isDone));
         });
   };
@@ -94,12 +97,18 @@ void SpeechToText::stream(std::shared_ptr<jsi::Function> callback,
     }
     ProcessResult res =
         this->processor->processIter(DecodingOptions(languageOption));
-    nativeCallback(res.committed, res.nonCommitted, false);
+
+    std::vector<char> committedVec(res.committed.begin(), res.committed.end());
+    std::vector<char> nonCommittedVec(res.nonCommitted.begin(), res.nonCommitted.end());
+
+    nativeCallback(committedVec, nonCommittedVec, false);
     this->readyToProcess = false;
   }
 
   std::string committed = this->processor->finish();
-  nativeCallback(committed, "", true);
+  std::vector<char> committedVec(committed.begin(), committed.end());
+
+  nativeCallback(committedVec, {}, true);
 
   this->resetStreamState();
 }

packages/react-native-executorch/common/rnexecutorch/models/speech_to_text/SpeechToText.h

@@ -1,6 +1,7 @@
 #pragma once
 
 #include "rnexecutorch/models/speech_to_text/stream/OnlineASRProcessor.h"
+#include <vector>
 
 namespace rnexecutorch {
 
@@ -17,7 +18,7 @@ class SpeechToText {
   std::shared_ptr<OwningArrayBuffer> encode(std::span<float> waveform) const;
   std::shared_ptr<OwningArrayBuffer>
   decode(std::span<int32_t> tokens, std::span<float> encoderOutput) const;
-  std::string transcribe(std::span<float> waveform,
+  std::vector<char> transcribe(std::span<float> waveform,
                          std::string languageOption) const;
 
   size_t getMemoryLowerBound() const noexcept;

packages/react-native-executorch/src/modules/natural_language_processing/SpeechToTextModule.ts

@@ -7,6 +7,11 @@ export class SpeechToTextModule {
 
   private modelConfig!: SpeechToTextModelConfig;
 
+  private textDecoder = new TextDecoder("utf-8", {
+      fatal: false,
+      ignoreBOM: true,
+  });
+
   public async load(
     model: SpeechToTextModelConfig,
     onDownloadProgressCallback: (progress: number) => void = () => {}
@@ -87,8 +92,8 @@ export class SpeechToTextModule {
       );
       waveform = new Float32Array(waveform);
     }
-
-    return this.nativeModule.transcribe(waveform, options.language || '');
+    const transcriptionBytes = await this.nativeModule.transcribe(waveform, options.language || '');
+    return this.textDecoder.decode(new Uint8Array(transcriptionBytes));
   }
 
   public async *stream(
@@ -109,8 +114,11 @@ export class SpeechToTextModule {
     (async () => {
       try {
         await this.nativeModule.stream(
-          (committed: string, nonCommitted: string, isDone: boolean) => {
-            queue.push({ committed, nonCommitted });
+          (committed: number[], nonCommitted: number[], isDone: boolean) => {
+            queue.push({
+              committed: this.textDecoder.decode(new Uint8Array(committed)),
+              nonCommitted: this.textDecoder.decode(new Uint8Array(nonCommitted))
+            });
             if (isDone) {
               finished = true;
             }