feat: refactor and fix aec bug on HarmonyOS (#271)

* feat: refactor and fix aec bug on HarmonyOS

* chore: Publish feat/refactor

* fix(api): Fix sentence bug

---------

Co-authored-by: shenxiaojie.316 <shenxiaojie.316@bytedance.com>

Author: jackshen310

common/changes/@coze/api/feat-refactor_2025-06-18-10-37.json

@@ -0,0 +1,11 @@
+{
+  "changes": [
+    {
+      "packageName": "@coze/api",
+      "comment": "refactor and fix aec bug on HarmonyOS",
+      "type": "minor"
+    }
+  ],
+  "packageName": "@coze/api",
+  "email": "shenxiaojie.316@bytedance.com"
+}

common/changes/@coze/api/feat-refactor_2025-06-18-10-53.json

@@ -0,0 +1,11 @@
+{
+  "changes": [
+    {
+      "packageName": "@coze/api",
+      "comment": "Publish feat/refactor",
+      "type": "patch"
+    }
+  ],
+  "packageName": "@coze/api",
+  "email": "shenxiaojie.316@bytedance.com"
+}

common/changes/@coze/api/feat-refactor_2025-06-19-09-18.json

@@ -0,0 +1,11 @@
+{
+  "changes": [
+    {
+      "packageName": "@coze/api",
+      "comment": "Fix sentence bug",
+      "type": "patch"
+    }
+  ],
+  "packageName": "@coze/api",
+  "email": "shenxiaojie.316@bytedance.com"
+}

common/changes/@coze/api/feat-refactor_2025-06-19-09-26.json

@@ -0,0 +1,11 @@
+{
+  "changes": [
+    {
+      "packageName": "@coze/api",
+      "comment": "Fix sentence bug",
+      "type": "patch"
+    }
+  ],
+  "packageName": "@coze/api",
+  "email": "shenxiaojie.316@bytedance.com"
+}

examples/coze-js-web/src/pages/chat-x/use-ws-api.ts

@@ -216,7 +216,7 @@ const useWsAPI = (
 
     transcriptionClientRef.current = transcriptionClient;
 
-    transcriptionClient.on('data', data => {
+    transcriptionClient.on(WebsocketsEventType.ALL, data => {
       console.log('[transcriptions] ws data', data);
 
       if (
@@ -237,8 +237,7 @@ const useWsAPI = (
   }, []);
 
   const stopTranscriptions = useCallback(async () => {
-    const finalAudio = await transcriptionClientRef.current?.stop();
-    console.log('[transcriptions] finalAudio:', finalAudio);
+    await transcriptionClientRef.current?.stop();
   }, []);
 
   const interruptAudio = useCallback(() => {

examples/realtime-websocket/src/pages/chat/sentence-message.tsx

@@ -124,6 +124,7 @@ const SentenceMessage = forwardRef(
           case ClientEventType.AUDIO_SENTENCE_PLAYBACK_START: {
             // 处理句子开始事件
             const { content } = (event as AudioSentencePlaybackStartEvent).data;
+            console.log('handleAudioSentencePlaybackStart', event);
             if (isFirstSentenceRef.current) {
               // 首个句子，创建新消息
               setMessageList(prev => [

packages/coze-js/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@coze/api",
-  "version": "1.3.0",
+  "version": "1.3.0-beta.1",
   "description": "Official Coze Node.js SDK for seamless AI integration into your applications | 扣子官方 Node.js SDK，助您轻松集成 AI 能力到应用中",
   "keywords": [
     "coze",
@@ -116,4 +116,4 @@
       "agora-rtc-sdk-ng": "$agora-rtc-sdk-ng"
     }
   }
-}
+}

packages/coze-js/src/ws-tools/chat/base.ts

@@ -1,18 +1,16 @@
 import { v4 as uuid } from 'uuid';
 
 import { type WavStreamPlayer } from '../wavtools';
+import SentenceSynchronizer from './sentence-synchronizer';
 import {
   type WsChatClientOptions,
   WsChatEventNames,
   type WsChatCallbackHandler,
   type WsChatEventData,
-  type SentenceItem,
-  ClientEventType,
 } from '../types';
 import {
   APIError,
   type AudioCodec,
-  type ConversationAudioSentenceStartEvent,
   COZE_CN_BASE_WS_URL,
   CozeAPI,
   type CreateChatWsReq,
@@ -30,19 +28,11 @@ abstract class BaseWsChatClient {
   protected trackId = 'default';
   protected api: CozeAPI;
   protected audioDeltaList: string[] = [];
-  /** 句子列表队列 */
-  protected sentenceList: SentenceItem[] = [];
-  /** 首个音频delta的时间戳（用于计算实际经过的时间）*/
-  protected firstAudioDeltaTime: number | null = null;
-  // 当前播放的句子索引
-  protected currentSentenceIndex = -1;
-  // 句子切换定时器
-  protected sentenceSwitchTimer: NodeJS.Timeout | null = null;
-  // 音频完成定时器
-  protected audioCompletedTimer: NodeJS.Timeout | null = null;
   public config: WsChatClientOptions;
   protected outputAudioCodec: AudioCodec = 'pcm';
   protected outputAudioSampleRate = 24000;
+  // 音字同步器实例
+  protected sentenceSynchronizer: SentenceSynchronizer;
 
   constructor(config: WsChatClientOptions) {
     this.api = new CozeAPI({
@@ -52,6 +42,11 @@ abstract class BaseWsChatClient {
     });
 
     this.config = config;
+
+    // 初始化音字同步器，传入事件发射器
+    this.sentenceSynchronizer = new SentenceSynchronizer({
+      eventEmitter: (eventName, eventData) => this.emit(eventName, eventData),
+    });
   }
 
   protected async init() {
@@ -116,20 +111,19 @@ abstract class BaseWsChatClient {
               break;
 
             case WebsocketsEventType.CONVERSATION_AUDIO_SENTENCE_START:
-              this.handleSentenceStart(data);
+              this.sentenceSynchronizer.handleSentenceStart(data);
               break;
 
             case WebsocketsEventType.INPUT_AUDIO_BUFFER_SPEECH_STARTED:
+              // 打断当前播放
               this.clear();
               break;
 
             case WebsocketsEventType.CONVERSATION_AUDIO_COMPLETED:
-              this.handleAudioCompleted();
+              this.sentenceSynchronizer.handleAudioCompleted();
               break;
 
             case WebsocketsEventType.CONVERSATION_CHAT_CANCELED:
-              // this.isInterrupted = false;
-              this.emitSentenceEnd();
               this.clear();
               break;
             default:
@@ -170,7 +164,6 @@ abstract class BaseWsChatClient {
   }
 
   sendTextMessage(text: string) {
-    this.clear();
     this.sendMessage({
       id: uuid(),
       event_type: WebsocketsEventType.CONVERSATION_MESSAGE_CREATE,
@@ -220,14 +213,13 @@ abstract class BaseWsChatClient {
   }
 
   async clear() {
-    this.audioDeltaList.length = 0;
-
-    // 重置音字同步状态
-    this.resetSentenceSyncState();
-
+    this.audioDeltaList = [];
     // 打断当前播放
-    await this.wavStreamPlayer?.interrupt();
     this.trackId = `my-track-id-${uuid()}`;
+    await this.wavStreamPlayer?.interrupt();
+
+    // 重置音字同步状态
+    this.sentenceSynchronizer.resetSentenceSyncState();
   }
 
   protected emit(eventName: string, event: WsChatEventData) {
@@ -249,19 +241,13 @@ abstract class BaseWsChatClient {
       view[i] = decodedContent.charCodeAt(i);
     }
 
-    // 记录首个音频delta的时间
-    if (this.firstAudioDeltaTime === null) {
-      this.firstAudioDeltaTime = performance.now();
-    }
+    // 设置首个音频 Delta 时间
+    this.sentenceSynchronizer.setFirstAudioDeltaTime();
 
-    if (this.sentenceList.length > 0) {
-      // 计算音频时长
-      // 例如：PCM 16bit 采样率为24000的计算公式: (字节数 / 2) / 24000 * 1000 毫秒
-      const audioDurationMs =
-        (decodedContent.length / 2 / this.outputAudioSampleRate) * 1000;
-      this.sentenceList[this.sentenceList.length - 1].audioDuration +=
-        audioDurationMs; // 更新当前句子的音频时长
-    }
+    // 更新最后一个句子的音频时长
+    this.sentenceSynchronizer.updateLatestSentenceAudioDuration(
+      decodedContent.length,
+    );
 
     try {
       await this.wavStreamPlayer?.add16BitPCM(arrayBuffer, this.trackId);
@@ -275,132 +261,14 @@ abstract class BaseWsChatClient {
     }
   };
 
-  private handleAudioCompleted() {
-    // 标记最后一个句子
-    this.audioCompletedTimer = setInterval(() => {
-      // 确保音频delta列表为空
-      if (this.audioDeltaList.length === 0) {
-        if (this.sentenceList.length > 0) {
-          this.sentenceList[this.sentenceList.length - 1].isLastSentence = true;
-        }
-        this.audioCompletedTimer && clearInterval(this.audioCompletedTimer);
-      }
-    }, 50);
-  }
-
-  /**
-   * 处理句子开始事件
-   * @param event 句子开始事件
-   */
-  private handleSentenceStart(
-    event: ConversationAudioSentenceStartEvent,
-  ): void {
-    // 将句子加入队列，存储文本和初始音频累计时长
-    const sentenceItem = {
-      id: event.id,
-      content: event.data.text,
-      audioDuration: 0, // 初始时该句子的音频累计时长为0
-      isLastSentence: false,
-    };
-    this.sentenceList.push(sentenceItem);
-
-    // 如果是首个句子，立即触发客户端句子开始事件
-    if (this.sentenceList.length === 1 && this.currentSentenceIndex === -1) {
-      this.currentSentenceIndex = 0;
-      this.emitSentenceStart(sentenceItem);
-      this.scheduleSentenceSwitch();
-    }
-  }
-
-  private scheduleSentenceSwitch(): void {
-    if (this.sentenceSwitchTimer) {
-      clearTimeout(this.sentenceSwitchTimer);
-    }
-
-    const { isLastSentence, audioDuration } =
-      this.sentenceList[this.currentSentenceIndex];
-
-    // 是否还有下一个句子
-    const hasNextSentence =
-      this.currentSentenceIndex + 1 < this.sentenceList.length;
-
-    let delay = 0;
-    if (this.currentSentenceIndex === 0) {
-      // 处理第一个句子 delay = 句子已累计时长 - 已播放时长
-      delay =
-        audioDuration -
-        (performance.now() - (this.firstAudioDeltaTime || performance.now()));
-      if (delay <= 0) {
-        // postpone until we have a meaningful duration
-        this.sentenceSwitchTimer = setTimeout(
-          () => this.scheduleSentenceSwitch(),
-          50,
-        );
-        return;
-      }
-    } else {
-      // 处理后续句子 delay = 句子累计时长
-      delay = audioDuration;
-    }
-
-    this.sentenceSwitchTimer = setTimeout(() => {
-      if (hasNextSentence) {
-        this.currentSentenceIndex++;
-        const nextSentence = this.sentenceList[this.currentSentenceIndex];
-        this.emitSentenceStart(nextSentence);
-      }
-      if (isLastSentence) {
-        this.emitSentenceEnd();
-      } else {
-        this.scheduleSentenceSwitch();
-      }
-    }, delay);
-  }
-
-  /**
-   * 发送客户端句子开始事件
-   * @param sentenceItem 句子开始事件
-   */
-  private emitSentenceStart(sentenceItem: SentenceItem): void {
-    this.emit(WsChatEventNames.AUDIO_SENTENCE_PLAYBACK_START, {
-      event_type: ClientEventType.AUDIO_SENTENCE_PLAYBACK_START,
-      data: {
-        content: sentenceItem.content,
-        id: sentenceItem.id,
-      },
-    });
-  }
-
-  /**
-   * 发送客户端句子结束事件
-   */
-  private emitSentenceEnd(): void {
-    this.emit(WsChatEventNames.AUDIO_SENTENCE_PLAYBACK_ENDED, {
-      event_type: ClientEventType.AUDIO_SENTENCE_PLAYBACK_ENDED,
-    });
-  }
-
-  private resetSentenceSyncState() {
-    this.currentSentenceIndex = -1;
-    this.sentenceList = [];
-    this.firstAudioDeltaTime = null;
-    if (this.sentenceSwitchTimer) {
-      clearTimeout(this.sentenceSwitchTimer);
-    }
-    if (this.audioCompletedTimer) {
-      clearInterval(this.audioCompletedTimer);
-    }
-    this.sentenceSwitchTimer = null;
-    this.audioCompletedTimer = null;
-  }
-
   // eslint-disable-next-line @typescript-eslint/no-explicit-any
   protected log(...args: any[]) {
     if (this.config.debug) {
       console.log('[WsChatClient]', ...args);
     }
     return true;
   }
+
   // eslint-disable-next-line @typescript-eslint/no-explicit-any
   protected warn(...args: any[]) {
     if (this.config.debug) {