Add ffmpeg wasm

^ Conflicts:
^	frontend/viewer/src/lib/components/audio/wavesurfer/waveform.svelte

Author: myieye

frontend/package.json

@@ -109,10 +109,7 @@
     "svelte-intl-precompile": "^0.12.3",
     "sveltekit-search-params": "^3",
     "tus-js-client": "^4.3.1",
-    "viewer": "workspace:*",
-    "@ffmpeg/ffmpeg": "0.12.15",
-    "@ffmpeg/util": "0.12.2",
-    "@ffmpeg/core": "0.12.10"
+    "viewer": "workspace:*"
   },
   "pnpm": {
     "onlyBuiltDependencies": [

frontend/pnpm-lock.yaml

@@ -20,6 +20,7 @@
     "lexbox-dev": "vite build -m web-component",
     "build": "vite build -m web-component",
     "build-app": "vite build",
+    "build-ffmpeg-worker": "vite build --config vite.config.ffmpeg-worker.ts",
     "preview": "vite preview",
     "pretest:playwright": "playwright install",
     "test:playwright": "playwright test",
@@ -98,6 +99,9 @@
     "@microsoft/signalr": "^8.0.0",
     "autoprefixer": "^10.4.19",
     "fast-json-patch": "^3.1.1",
+    "@ffmpeg/ffmpeg": "0.12.15",
+    "@ffmpeg/util": "0.12.2",
+    "@ffmpeg/core": "0.12.10",
     "jsdom": "^26.1.0",
     "just-throttle": "^4.2.0",
     "postcss": "catalog:",

frontend/viewer/package.json

@@ -10,7 +10,7 @@
   import AudioEditor from './audio-editor.svelte';
   import Loading from '$lib/components/Loading.svelte';
 
-  let open = $state(false);
+  let open = $state(true);
   useBackHandler({addToStack: () => open, onBack: () => open = false, key: 'audio-dialog'});
   const dialogsService = useDialogsService();
   dialogsService.invokeAudioDialog = getAudio;

frontend/viewer/src/lib/components/audio/AudioDialog.svelte

@@ -6,6 +6,7 @@
   import {formatDigitalDuration} from '../ui/format/format-duration';
   import DevContent from '$lib/layout/DevContent.svelte';
   import {Label} from '../ui/label';
+  import {convertToWav, loadFFmpeg} from './ffmpeg';
 
   type Props = {
     audio: Blob;
@@ -21,6 +22,10 @@
   const mb = $derived((audio.size / 1024 / 1024).toFixed(2));
   const formatedDuration = $derived(duration ? formatDigitalDuration({ seconds: duration }) : 'unknown');
 
+  void loadFFmpeg().then(async (ffmpeg) => {
+    if (!audio) return;
+    audio = await convertToWav(ffmpeg, audio);
+  });
 </script>
 
 <div class="flex flex-col gap-4 items-center justify-center">
@@ -37,9 +42,11 @@
       <span class="col-span-4">{$t`${audio.type}`}</span>
     </DevContent>
   </span>
-  <!-- contain-inline-size prevents wavesurfer from freaking out inside a grid -->
+  <!-- contain-size prevents wavesurfer from freaking out inside a grid
+  contain-inline-size would improve the height reactivity of the waveform, but
+  results in the waveform sometimes change its height unexpectedly -->
   <!-- pb-8 ensures the timeline is in the bounds of the container -->
-  <div class="w-full grow max-h-32 pb-3 contain-inline-size border-y">
+  <div class="w-full grow max-h-32 pb-3 contain-size border-y">
     <Waveform {audio} bind:playing bind:audioApi bind:duration showTimeline autoplay class="size-full" />
   </div>
   <div class="flex gap-2">

frontend/viewer/src/lib/components/audio/audio-editor.svelte

@@ -0,0 +1,104 @@
+/* This file generated by Vite / pnpm build-ffmpeg-worker. */
+// @ts-nocheck
+const f = "0.12.9", R = `https://unpkg.com/@ffmpeg/core@${f}/dist/umd/ffmpeg-core.js`;
+var E;
+(function(t) {
+  t.LOAD = "LOAD", t.EXEC = "EXEC", t.FFPROBE = "FFPROBE", t.WRITE_FILE = "WRITE_FILE", t.READ_FILE = "READ_FILE", t.DELETE_FILE = "DELETE_FILE", t.RENAME = "RENAME", t.CREATE_DIR = "CREATE_DIR", t.LIST_DIR = "LIST_DIR", t.DELETE_DIR = "DELETE_DIR", t.ERROR = "ERROR", t.DOWNLOAD = "DOWNLOAD", t.PROGRESS = "PROGRESS", t.LOG = "LOG", t.MOUNT = "MOUNT", t.UNMOUNT = "UNMOUNT";
+})(E || (E = {}));
+const u = new Error("unknown message type"), O = new Error("ffmpeg is not loaded, call `await ffmpeg.load()` first"), m = new Error("failed to import ffmpeg-core.js");
+let r;
+const l = async ({ coreURL: t, wasmURL: n, workerURL: e }) => {
+  const o = !r;
+  try {
+    t || (t = R), importScripts(t);
+  } catch {
+    if ((!t || t === R) && (t = R.replace("/umd/", "/esm/")), self.createFFmpegCore = (await import(
+      /* @vite-ignore */
+      t
+    )).default, !self.createFFmpegCore)
+      throw m;
+  }
+  const s = t, c = n || t.replace(/.js$/g, ".wasm"), a = e || t.replace(/.js$/g, ".worker.js");
+  return r = await self.createFFmpegCore({
+    // Fix `Overload resolution failed.` when using multi-threaded ffmpeg-core.
+    // Encoded wasmURL and workerURL in the URL as a hack to fix locateFile issue.
+    mainScriptUrlOrBlob: `${s}#${btoa(JSON.stringify({ wasmURL: c, workerURL: a }))}`
+  }), r.setLogger((i) => self.postMessage({ type: E.LOG, data: i })), r.setProgress((i) => self.postMessage({
+    type: E.PROGRESS,
+    data: i
+  })), o;
+}, D = ({ args: t, timeout: n = -1 }) => {
+  r.setTimeout(n), r.exec(...t);
+  const e = r.ret;
+  return r.reset(), e;
+}, S = ({ args: t, timeout: n = -1 }) => {
+  r.setTimeout(n), r.ffprobe(...t);
+  const e = r.ret;
+  return r.reset(), e;
+}, I = ({ path: t, data: n }) => (r.FS.writeFile(t, n), !0), L = ({ path: t, encoding: n }) => r.FS.readFile(t, { encoding: n }), N = ({ path: t }) => (r.FS.unlink(t), !0), A = ({ oldPath: t, newPath: n }) => (r.FS.rename(t, n), !0), k = ({ path: t }) => (r.FS.mkdir(t), !0), w = ({ path: t }) => {
+  const n = r.FS.readdir(t), e = [];
+  for (const o of n) {
+    const s = r.FS.stat(`${t}/${o}`), c = r.FS.isDir(s.mode);
+    e.push({ name: o, isDir: c });
+  }
+  return e;
+}, b = ({ path: t }) => (r.FS.rmdir(t), !0), p = ({ fsType: t, options: n, mountPoint: e }) => {
+  const o = t, s = r.FS.filesystems[o];
+  return s ? (r.FS.mount(s, n, e), !0) : !1;
+}, d = ({ mountPoint: t }) => (r.FS.unmount(t), !0);
+self.onmessage = async ({ data: { id: t, type: n, data: e } }) => {
+  const o = [];
+  let s;
+  try {
+    if (n !== E.LOAD && !r)
+      throw O;
+    switch (n) {
+      case E.LOAD:
+        s = await l(e);
+        break;
+      case E.EXEC:
+        s = D(e);
+        break;
+      case E.FFPROBE:
+        s = S(e);
+        break;
+      case E.WRITE_FILE:
+        s = I(e);
+        break;
+      case E.READ_FILE:
+        s = L(e);
+        break;
+      case E.DELETE_FILE:
+        s = N(e);
+        break;
+      case E.RENAME:
+        s = A(e);
+        break;
+      case E.CREATE_DIR:
+        s = k(e);
+        break;
+      case E.LIST_DIR:
+        s = w(e);
+        break;
+      case E.DELETE_DIR:
+        s = b(e);
+        break;
+      case E.MOUNT:
+        s = p(e);
+        break;
+      case E.UNMOUNT:
+        s = d(e);
+        break;
+      default:
+        throw u;
+    }
+  } catch (c) {
+    self.postMessage({
+      id: t,
+      type: E.ERROR,
+      data: c.toString()
+    });
+    return;
+  }
+  s instanceof Uint8Array && o.push(s.buffer), self.postMessage({ id: t, type: n, data: s }, o);
+};

frontend/viewer/src/lib/components/audio/ffmpeg-loader.ts

@@ -0,0 +1,57 @@
+import {FFmpeg} from '@ffmpeg/ffmpeg';
+import coreURL from '@ffmpeg/core?url';
+import {fetchFile} from '@ffmpeg/util';
+import inlineDataUrlWorker from './bundled-ffmpeg-worker.js?url&inline';
+import wasmUrl from '@ffmpeg/core/wasm?url';
+
+let loadingPromise: Promise<FFmpeg> | null = null;
+
+export async function loadFFmpeg(): Promise<FFmpeg> {
+  if (loadingPromise) {
+    return loadingPromise;
+  }
+
+  try {
+    loadingPromise = loadFFmpegInternal();
+  } catch (error) {
+    loadingPromise = null;
+    throw error;
+  }
+
+  return loadingPromise;
+}
+
+async function loadFFmpegInternal(): Promise<FFmpeg> {
+  const ffmpeg = new FFmpeg();
+
+  await ffmpeg.load({
+    coreURL: coreURL,
+    wasmURL: wasmUrl,
+    classWorkerURL: import.meta.env.DEV ? inlineDataUrlWorker : undefined,
+  });
+
+  return ffmpeg;
+}
+
+export async function convertToWav(ffmpeg: FFmpeg, blob: Blob): Promise<Blob> {
+  // Load input into ffmpeg FS
+  const inputName = 'input.mp3';
+  const outputName = 'output.wav';
+
+  await ffmpeg.writeFile(inputName, await fetchFile(blob));
+
+  // Normalize volume and convert to WAV
+  await ffmpeg.exec([
+    '-i', inputName,
+    '-af', 'loudnorm',
+    '-ar', '44100',
+    '-ac', '2',
+    '-c:a', 'pcm_s16le',
+    outputName
+  ]);
+
+  // Read output file
+  const data = await ffmpeg.readFile(outputName) as Uint8Array;;
+  return new Blob([data], {type: 'audio/wav'});
+}
+

frontend/viewer/src/lib/components/audio/ffmpeg/bundled-ffmpeg-worker.js

@@ -0,0 +1 @@
+export * from './ffmpeg-api';

frontend/viewer/src/lib/components/audio/ffmpeg/ffmpeg-api.ts

@@ -33,8 +33,7 @@
   import {formatDate, FormatDate, formatNumber} from '$lib/components/ui/format';
   import {SvelteDate} from 'svelte/reactivity';
   import {RichTextToggle} from '$lib/dotnet-types/generated-types/MiniLcm/Models/RichTextToggle';
-  import {loadFFmpeg} from '$lib/components/audio/ffmpeg-loader';
-
+  import {loadFFmpeg} from '$lib/components/audio/ffmpeg';
 
   const testingService = tryUseService(DotnetService.TestingService);