fix(engine): sample-accurate volume automation so dense fades keep their audio (#1117)

Animated media volume (GSAP/JS fades) dropped the audio track entirely for dense
fades. The 60 Hz timeline probe emits 100-300 keyframes for a multi-second fade,
which were folded into an FFmpeg `volume` expression nesting one `if(lt(t,...))`
per keyframe. Past ~95 nested levels (build-dependent, lower on some Linux ffmpeg
builds) the expression overflows FFmpeg's evaluator, fails filter-graph init,
fails the whole mix, and the muxer omits audio — so a `data-volume="0"` fade-in
rendered with no audio at all (follow-up to #1066; this is why #1064's own
scenario regressed once the fade was dense enough).

Apply volume automation as sample-accurate gain, layered so audio is never lost:

1. Primary: bake the envelope into the prepared PCM samples in-process
   (audioVolumeEnvelope.ts). The track WAV is always pcm_s16le/48k/stereo;
   multiply its samples by the interpolated envelope and atomically rename the
   result into place, then mix at unity. No expression, no keyframe ceiling,
   exact at every sample, and the downstream ffmpeg amix/AAC encode is untouched
   so golden baselines only change where a fade is applied. The RIFF parser
   scans chunks order-independently and accepts only 16-bit PCM, falling back
   otherwise. The output is written to a random-named sibling and renamed, so a
   crash can't leave a truncated WAV and there's no predictable-path write.
2. Fallback: RDP-bounded ffmpeg `volume` expression (0.5% tolerance, capped at
   32 segments) for the rare case a WAV is not 16-bit PCM. 0.5% keeps the
   rendered envelope within ~0.2 dB of the source curve.
3. Backstop: if an automated mix still fails, retry once at base volume and
   surface the degradation rather than dropping the track.

This mirrors how OSS NLEs render automation (sample-level gain): MoviePy,
Kdenlive/Shotcut (MLT), Remotion.

Verified end-to-end: a 297-keyframe fade that rendered with no audio now bakes
all 297 keyframes sample-accurately. Adds unit tests for sample-accurate gain,
track-start offset, base/tail holds, thousands of keyframes, order-independent
chunk parsing, and format rejection, plus mixer regression tests for bounded
nesting and the base-volume backstop.

Author: miguel-heygen

packages/engine/src/services/audioMixer.test.ts

@@ -108,6 +108,121 @@ describe("processCompositionAudio", () => {
     expect(filter).toContain("adelay=2000|2000");
   });
 
+  it("bounds expression nesting for dense keyframe automation without dropping the envelope", async () => {
+    const baseDir = mkdtempSync(join(tmpdir(), "hf-audio-base-"));
+    const workDir = mkdtempSync(join(tmpdir(), "hf-audio-work-"));
+    tempDirs.push(baseDir, workDir);
+
+    writeFileSync(join(baseDir, "bgm.wav"), "stub");
+
+    // Mirrors the 60 Hz timeline probe: a 10s eased fade emits hundreds of
+    // keyframes. The nested-if volume expression must not grow one level per
+    // keyframe — past ~95 levels FFmpeg fails filter-graph init and the audio
+    // track is dropped entirely (GH #1066 follow-up).
+    const keyframes = Array.from({ length: 300 }, (_, i) => {
+      const time = (i / 299) * 10;
+      const volume =
+        time < 3 ? 0.8 * (time / 3) ** 2 : time < 7 ? 0.8 : 0.8 * (1 - (time - 7) / 3) ** 2;
+      return { time, volume };
+    });
+
+    const result = await processCompositionAudio(
+      [
+        {
+          id: "bgm",
+          src: "bgm.wav",
+          start: 0,
+          end: 10,
+          mediaStart: 0,
+          layer: 0,
+          volume: 0,
+          volumeKeyframes: keyframes,
+          type: "audio",
+        },
+      ],
+      baseDir,
+      workDir,
+      join(baseDir, "out.m4a"),
+      10,
+    );
+
+    expect(result.success).toBe(true);
+
+    const mixArgs = runFfmpegMock.mock.calls[1]?.[0];
+    const filterIndex = mixArgs.indexOf("-filter_complex");
+    const filter = mixArgs[filterIndex + 1];
+
+    // One nested `if(lt(...))` is emitted per segment; cap it well under the
+    // FFmpeg evaluator's nesting limit (MAX_VOLUME_SEGMENTS = 32).
+    const nestingDepth = (filter.match(/if\(lt\(t/g) ?? []).length;
+    expect(nestingDepth).toBeGreaterThan(1);
+    expect(nestingDepth).toBeLessThan(32);
+
+    // The simplified envelope still spans the clip: silent start, audible peak.
+    expect(filter).toContain(":eval=frame");
+    expect(filter).toMatch(/volume=if\(lt\(t\\,[0-9.]+\)\\,0\+/);
+  });
+
+  it("falls back to a static-volume mix instead of dropping audio when the automated mix fails", async () => {
+    const baseDir = mkdtempSync(join(tmpdir(), "hf-audio-base-"));
+    const workDir = mkdtempSync(join(tmpdir(), "hf-audio-work-"));
+    tempDirs.push(baseDir, workDir);
+
+    writeFileSync(join(baseDir, "bgm.wav"), "stub");
+
+    // Simulate an ffmpeg build that rejects the automation expression: the
+    // first mix attempt fails, the static-volume retry succeeds. (prepare =
+    // call 0, automated mix = call 1, fallback mix = call 2.)
+    runFfmpegMock
+      .mockImplementationOnce(async () => ({
+        success: true,
+        durationMs: 1,
+        stderr: "",
+        exitCode: 0,
+      }))
+      .mockImplementationOnce(async () => ({
+        success: false,
+        durationMs: 1,
+        stderr: "Error initializing filters",
+        exitCode: 234,
+      }));
+
+    const result = await processCompositionAudio(
+      [
+        {
+          id: "bgm",
+          src: "bgm.wav",
+          start: 0,
+          end: 5,
+          mediaStart: 0,
+          layer: 0,
+          volume: 0.8,
+          volumeKeyframes: [
+            { time: 0, volume: 0.8 },
+            { time: 5, volume: 0 },
+          ],
+          type: "audio",
+        },
+      ],
+      baseDir,
+      workDir,
+      join(baseDir, "out.m4a"),
+      5,
+    );
+
+    expect(result.success).toBe(true);
+    expect(result.tracksProcessed).toBe(1);
+    expect(runFfmpegMock).toHaveBeenCalledTimes(3);
+    // Degradation is surfaced, not silent — the track rendered at base volume.
+    expect(result.error).toMatch(/base volume/i);
+
+    // The fallback mix omits the automation expression (base volume only).
+    const fallbackArgs = runFfmpegMock.mock.calls[2]?.[0];
+    const fallbackFilter = fallbackArgs[fallbackArgs.indexOf("-filter_complex") + 1];
+    expect(fallbackFilter).not.toContain(":eval=frame");
+    expect(fallbackFilter).toContain("volume=0.8");
+  });
+
   it("prepares percent-encoded non-Latin audio srcs from decoded filesystem paths", async () => {
     const baseDir = mkdtempSync(join(tmpdir(), "hf-audio-base-"));
     const workDir = mkdtempSync(join(tmpdir(), "hf-audio-work-"));

packages/engine/src/services/audioMixer.ts

@@ -14,6 +14,7 @@ import { runFfmpeg } from "../utils/runFfmpeg.js";
 import { unwrapTemplate } from "../utils/htmlTemplate.js";
 import { resolveProjectRelativeSrc } from "./videoFrameExtractor.js";
 import type { AudioElement, AudioTrack, MixResult } from "./audioMixer.types.js";
+import { applyVolumeEnvelopeToWav } from "./audioVolumeEnvelope.js";
 
 export type { AudioElement, MixResult } from "./audioMixer.types.js";
 
@@ -30,10 +31,89 @@ function escapeExpressionCommas(expression: string): string {
   return expression.replace(/\\/g, "\\\\").replace(/,/g, "\\,");
 }
 
-function buildVolumeExpression(track: AudioTrack): string {
+/**
+ * Upper bound on volume-automation keyframes folded into the FFmpeg `volume`
+ * expression. The expression nests one `if(lt(...))` per keyframe, and
+ * FFmpeg's expression evaluator has a finite nesting depth: past ~95 levels
+ * (build-dependent — lower on some Linux ffmpeg builds) `volume=...:eval=frame`
+ * fails filter-graph init, which fails the whole mix and drops the audio track
+ * entirely. The 60 Hz timeline probe routinely emits 100–300 keyframes for a
+ * multi-second fade (GH #1066 follow-up: a 171-keyframe GSAP fade rendered with
+ * no audio). 32 segments keeps a wide safety margin and is far more resolution
+ * than a piecewise-linear volume envelope needs.
+ */
+const MAX_VOLUME_SEGMENTS = 32;
+
+/**
+ * Volume delta below which a keyframe is collinear enough to drop. Kept tight
+ * (0.5% linear) so the rendered piecewise-linear envelope tracks the GSAP curve
+ * the browser plays in preview to within ~0.2 dB across the audible range — well
+ * under the ~1 dB loudness JND, so render stays WYSIWYG with preview. A full
+ * ease-in/ease-out fade still reduces to ~25 segments, inside MAX_VOLUME_SEGMENTS.
+ */
+const VOLUME_SIMPLIFY_EPSILON = 0.005;
+
+/**
+ * Reduce a sorted keyframe list to a perceptually-equivalent piecewise-linear
+ * envelope with a bounded segment count.
+ *
+ * Ramer–Douglas–Peucker drops control points lying within
+ * `VOLUME_SIMPLIFY_EPSILON` of the line through their neighbours (a linear fade
+ * collapses to its two endpoints; an eased fade to a handful). A uniform
+ * downsample backstop then bounds pathological inputs (e.g. audio-rate volume
+ * oscillation) to `MAX_VOLUME_SEGMENTS`. Endpoints are always preserved so the
+ * envelope still spans the full clip.
+ */
+function simplifyVolumeKeyframes(
+  keyframes: { time: number; volume: number }[],
+): { time: number; volume: number }[] {
+  if (keyframes.length < 3) return keyframes;
+
+  const keep = new Array<boolean>(keyframes.length).fill(false);
+  keep[0] = true;
+  keep[keyframes.length - 1] = true;
+  const stack: [number, number][] = [[0, keyframes.length - 1]];
+  while (stack.length > 0) {
+    const [startIndex, endIndex] = stack.pop()!;
+    const start = keyframes[startIndex]!;
+    const end = keyframes[endIndex]!;
+    const span = end.time - start.time;
+    let maxDistance = VOLUME_SIMPLIFY_EPSILON;
+    let splitIndex = -1;
+    for (let i = startIndex + 1; i < endIndex; i += 1) {
+      const point = keyframes[i]!;
+      const interpolated =
+        span === 0
+          ? start.volume
+          : start.volume + ((end.volume - start.volume) * (point.time - start.time)) / span;
+      const distance = Math.abs(point.volume - interpolated);
+      if (distance > maxDistance) {
+        maxDistance = distance;
+        splitIndex = i;
+      }
+    }
+    if (splitIndex !== -1) {
+      keep[splitIndex] = true;
+      stack.push([startIndex, splitIndex], [splitIndex, endIndex]);
+    }
+  }
+
+  const simplified = keyframes.filter((_, i) => keep[i]);
+  if (simplified.length <= MAX_VOLUME_SEGMENTS) return simplified;
+
+  const step = (simplified.length - 1) / (MAX_VOLUME_SEGMENTS - 1);
+  const sampled: { time: number; volume: number }[] = [];
+  for (let i = 0; i < MAX_VOLUME_SEGMENTS; i += 1) {
+    const point = simplified[Math.round(i * step)]!;
+    if (sampled.length === 0 || point.time > sampled.at(-1)!.time) sampled.push(point);
+  }
+  return sampled;
+}
+
+function buildVolumeExpression(track: AudioTrack, ignoreKeyframes = false): string {
   const trimDuration = track.end - track.start;
   const staticVolume = clampVolume(track.volume);
-  const keyframes = (track.volumeKeyframes ?? [])
+  const keyframes = (ignoreKeyframes ? [] : (track.volumeKeyframes ?? []))
     .filter((keyframe) => Number.isFinite(keyframe.time) && Number.isFinite(keyframe.volume))
     .map((keyframe) => ({
       time: Math.max(0, Math.min(trimDuration, keyframe.time - track.start)),
@@ -57,14 +137,19 @@ function buildVolumeExpression(track: AudioTrack): string {
     }
   }
 
-  if (deduped.length === 1) {
-    return `volume=${formatFilterNumber(deduped[0]!.volume)}`;
+  // Collapse the densely-sampled probe output to a bounded piecewise-linear
+  // envelope. Without this, the nested-if expression below grows one level per
+  // keyframe and overflows FFmpeg's expression evaluator (see MAX_VOLUME_SEGMENTS).
+  const simplified = simplifyVolumeKeyframes(deduped);
+
+  if (simplified.length === 1) {
+    return `volume=${formatFilterNumber(simplified[0]!.volume)}`;
   }
 
-  let expression = formatFilterNumber(deduped.at(-1)!.volume);
-  for (let i = deduped.length - 2; i >= 0; i -= 1) {
-    const current = deduped[i]!;
-    const next = deduped[i + 1]!;
+  let expression = formatFilterNumber(simplified.at(-1)!.volume);
+  for (let i = simplified.length - 2; i >= 0; i -= 1) {
+    const current = simplified[i]!;
+    const next = simplified[i + 1]!;
     const currentTime = formatFilterNumber(current.time);
     const nextTime = formatFilterNumber(next.time);
     const currentVolume = formatFilterNumber(current.volume);
@@ -299,42 +384,58 @@ async function mixAudioTracks(
   const outputDir = dirname(outputPath);
   if (!existsSync(outputDir)) mkdirSync(outputDir, { recursive: true });
 
-  const inputs: string[] = [];
-  const filterParts: string[] = [];
-
-  tracks.forEach((track, i) => {
-    inputs.push("-i", track.srcPath);
-    const delayMs = Math.round(track.start * 1000);
-    const trimDuration = track.end - track.start;
-    const volumeFilter = buildVolumeExpression(track);
-    filterParts.push(
-      `[${i}:a]atrim=0:${trimDuration},${volumeFilter},adelay=${delayMs}|${delayMs},apad=whole_dur=${totalDuration}[a${i}]`,
-    );
-  });
-
-  const mixInputs = tracks.map((_, i) => `[a${i}]`).join("");
-  const weights = tracks.map(() => "1").join(" ");
-  const mixFilter = `${mixInputs}amix=inputs=${tracks.length}:duration=longest:dropout_transition=0:normalize=0:weights='${weights}'[mixed]`;
-  const postMixGainFilter = `[mixed]volume=${masterOutputGain}[out]`;
-  const fullFilter = [...filterParts, mixFilter, postMixGainFilter].join(";");
+  const buildArgs = (ignoreAutomation: boolean): string[] => {
+    const inputs: string[] = [];
+    const filterParts: string[] = [];
+    tracks.forEach((track, i) => {
+      inputs.push("-i", track.srcPath);
+      const delayMs = Math.round(track.start * 1000);
+      const trimDuration = track.end - track.start;
+      const volumeFilter = buildVolumeExpression(track, ignoreAutomation);
+      filterParts.push(
+        `[${i}:a]atrim=0:${trimDuration},${volumeFilter},adelay=${delayMs}|${delayMs},apad=whole_dur=${totalDuration}[a${i}]`,
+      );
+    });
 
-  const args = [
-    ...inputs,
-    "-filter_complex",
-    fullFilter,
-    "-map",
-    "[out]",
-    "-acodec",
-    "aac",
-    "-b:a",
-    "192k",
-    "-t",
-    String(totalDuration),
-    "-y",
-    outputPath,
-  ];
+    const mixInputs = tracks.map((_, i) => `[a${i}]`).join("");
+    const weights = tracks.map(() => "1").join(" ");
+    const mixFilter = `${mixInputs}amix=inputs=${tracks.length}:duration=longest:dropout_transition=0:normalize=0:weights='${weights}'[mixed]`;
+    const postMixGainFilter = `[mixed]volume=${masterOutputGain}[out]`;
+    const fullFilter = [...filterParts, mixFilter, postMixGainFilter].join(";");
+
+    return [
+      ...inputs,
+      "-filter_complex",
+      fullFilter,
+      "-map",
+      "[out]",
+      "-acodec",
+      "aac",
+      "-b:a",
+      "192k",
+      "-t",
+      String(totalDuration),
+      "-y",
+      outputPath,
+    ];
+  };
 
-  const result = await runFfmpeg(args, { signal, timeout: ffmpegProcessTimeout });
+  let result = await runFfmpeg(buildArgs(false), { signal, timeout: ffmpegProcessTimeout });
+
+  // Defense in depth: volume automation is folded into an FFmpeg `volume`
+  // expression whose evaluator limits are build-dependent (see
+  // MAX_VOLUME_SEGMENTS). If that ever fails the mix, retry once without the
+  // automation so the track renders at its base volume rather than being
+  // dropped from the output entirely — a missing fade beats missing audio.
+  let degradedAutomation = false;
+  const hasAutomation = tracks.some((track) => (track.volumeKeyframes?.length ?? 0) > 0);
+  if (!result.success && !signal?.aborted && hasAutomation) {
+    const retry = await runFfmpeg(buildArgs(true), { signal, timeout: ffmpegProcessTimeout });
+    if (retry.success) {
+      result = retry;
+      degradedAutomation = true;
+    }
+  }
 
   if (signal?.aborted) {
     return {
@@ -360,6 +461,9 @@ async function mixAudioTracks(
     outputPath,
     durationMs: result.durationMs,
     tracksProcessed: tracks.length,
+    error: degradedAutomation
+      ? "Volume automation exceeded this ffmpeg build's expression limits; rendered at base volume"
+      : undefined,
   };
 }
 
@@ -452,15 +556,29 @@ export async function processCompositionAudio(
           audioSrcPath = trimmedPath;
         }
 
+        // Primary volume-automation path: bake the envelope into the PCM samples
+        // (sample-accurate, no keyframe ceiling). If the WAV isn't the expected
+        // 16-bit PCM, fall back to the ffmpeg expression path by leaving the
+        // keyframes on the track for buildVolumeExpression to handle.
+        let bakedEnvelope = false;
+        if (element.volumeKeyframes && element.volumeKeyframes.length > 0) {
+          bakedEnvelope = applyVolumeEnvelopeToWav(
+            audioSrcPath,
+            element.volumeKeyframes,
+            element.start,
+            element.volume ?? 1.0,
+          );
+        }
         tracks.push({
           id: element.id,
           srcPath: audioSrcPath,
           start: element.start,
           end: element.end,
           mediaStart: element.mediaStart,
           duration: element.end - element.start,
-          volume: element.volume ?? 1.0,
-          volumeKeyframes: element.volumeKeyframes,
+          // Gain is already in the samples when baked, so mix at unity.
+          volume: bakedEnvelope ? 1.0 : (element.volume ?? 1.0),
+          volumeKeyframes: bakedEnvelope ? undefined : element.volumeKeyframes,
         });
       } catch (err: unknown) {
         errors.push(`Error: ${element.id} — ${err instanceof Error ? err.message : String(err)}`);