fix: fix: OOM when add many clips to AVCanvas

Author: hughfenghen

.changeset/full-banks-hope.md

@@ -0,0 +1,6 @@
+---
+'@webav/av-canvas': patch
+'@webav/av-cliper': patch
+---
+
+fix: OOM when add many clips to AVCanvas

packages/av-canvas/demo/video-editor.tsx

@@ -183,6 +183,18 @@ function App() {
     //   addSprite2Track('4-text', spr, '文字');
     // })();
 
+    // 模拟添加 60个素材，验证内存占用
+    // (async () => {
+    //   const f = file('/test-1.mp4');
+    //   await write(f, (await fetch('./video/incorrect-frame-type.mp4')).body!);
+    //   for (let i = 0; i < 60; i++) {
+    //     const spr = new VisibleSprite(new MP4Clip(f));
+    //     await spr.ready;
+    //     await cvs.addSprite(spr);
+    //     addSprite2Track('1-video', spr, '视频');
+    //   }
+    // })();
+
     return () => {
       cvs.destroy();
     };

packages/av-canvas/package.json

@@ -46,6 +46,7 @@
     "@xzdarcy/react-timeline-editor": "^0.1.9",
     "autoprefixer": "^10.4.19",
     "esbuild-plugin-react-virtualized": "^1.0.5",
+    "opfs-tools": "^0.7.2",
     "postcss": "^8.4.38",
     "react": "^18.3.1",
     "react-dom": "^18.3.1",

packages/av-canvas/src/av-canvas.ts

@@ -7,7 +7,7 @@ import {
   Rect,
   VisibleSprite,
 } from '@webav/av-cliper';
-import { EventTool, workerTimer } from '@webav/internal-utils';
+import { EventTool, throttle, workerTimer } from '@webav/internal-utils';
 import { renderCtrls } from './sprites/render-ctrl';
 import { ESpriteManagerEvt, SpriteManager } from './sprites/sprite-manager';
 import { activeSprite, draggabelSprite } from './sprites/sprite-op';
@@ -163,6 +163,7 @@ export class AVCanvas {
   #updateRenderTime(time: number) {
     this.#renderTime = time;
     this.#spriteManager.updateRenderTime(time);
+    this.#autoPreFrame.updateTime(time);
   }
 
   #pause() {
@@ -177,6 +178,7 @@ export class AVCanvas {
       asn.disconnect();
     }
     this.#playingAudioCache.clear();
+    this.#autoPreFrame.reset();
   }
 
   #audioCtx = new AudioContext();
@@ -259,11 +261,7 @@ export class AVCanvas {
     }
 
     this.#updateRenderTime(opts.start);
-    this.#spriteManager.getSprites({ time: false }).forEach((vs) => {
-      const { offset, duration } = vs.time;
-      const selfOffset = this.#renderTime - offset;
-      vs.preFrame(selfOffset > 0 && selfOffset < duration ? selfOffset : 0);
-    });
+    this.#autoPreFrame.reset();
 
     this.#playState.start = opts.start;
     this.#playState.end = end;
@@ -276,6 +274,27 @@ export class AVCanvas {
     Log.info('AVCanvs play by:', this.#playState);
   }
 
+  #autoPreFrame = (() => {
+    const readyVS = new Set<VisibleSprite>();
+    return {
+      reset() {
+        readyVS.clear();
+      },
+      updateTime: throttle((curTime: number) => {
+        const sprs = this.#spriteManager.getSprites({ time: false });
+        // 匹配接下来 300ms 内即将要播放的 Sprite
+        const matchPreSprs = sprs.filter((vs) => {
+          const { offset } = vs.time;
+          return offset > curTime && offset - 300e3 <= curTime;
+        });
+        for (const vs of matchPreSprs) {
+          if (!readyVS.has(vs)) vs.preFrame(0);
+          readyVS.add(vs);
+        }
+      }, 300),
+    };
+  })();
+
   /**
    * 暂停播放，画布内容不再更新
    */
@@ -288,6 +307,9 @@ export class AVCanvas {
    */
   previewFrame(time: number) {
     this.#updateRenderTime(time);
+    this.#spriteManager.getSprites().forEach((vs) => {
+      vs.preFrame(time - vs.time.offset);
+    });
     this.#pause();
   }
 
@@ -330,7 +352,6 @@ export class AVCanvas {
       this.#sprMapAudioNode.set(vs, audioNode);
     }
     await this.#spriteManager.addSprite(vs);
-    vs.preFrame(0);
   };
   /**
    * 删除 {@link VisibleSprite}

packages/av-cliper/src/sprite/visible-sprite.ts

@@ -1,7 +1,7 @@
-import { BaseSprite } from './base-sprite';
-import { IClip } from '../clips';
 import { Log } from '@webav/internal-utils';
 import { changePCMPlaybackRate } from '../av-utils';
+import { IClip } from '../clips';
+import { BaseSprite } from './base-sprite';
 
 /**
  * 包装 {@link IClip} 给素材扩展坐标、层级、透明度等信息，用于 {@link [AVCanvas](../../av-canvas/classes/AVCanvas.html)} 响应用户交互
@@ -48,6 +48,10 @@ export class VisibleSprite extends BaseSprite {
   #update(time: number) {
     if (this.#ticking) return;
     this.#ticking = true;
+
+    this.#lastVf?.close();
+    this.#lastVf = null;
+    this.#lastAudio = [];
     this.#clip
       .tick(time * this.time.playbackRate)
       .then(({ video, audio }) => {