fix(engine): degrade volume automation gracefully instead of dropping audio

Defense in depth on top of the keyframe bound: the volume envelope is folded
into an FFmpeg `volume` expression whose evaluator limits are build-dependent,
so a future edge case (an even stricter ffmpeg build, an escaping quirk) could
still fail the mix. Rather than let that drop the audio track entirely, retry
the mix once without the automation so the track renders at its base volume —
a missing fade beats missing audio. The degradation is surfaced via MixResult.

This mirrors how NLEs render automation (a sparse keyframe set with sample-
accurate interpolation — which `volume=...:eval=frame` already provides), so a
sample-level PCM rewrite would add a large new subsystem without changing the
result for timeline-driven fades.

Author: miguel-heygen

packages/engine/src/services/audioMixer.test.ts

@@ -163,6 +163,64 @@ describe("processCompositionAudio", () => {
     expect(filter).toMatch(/volume=if\(lt\(t\\,[0-9.]+\)\\,0\+/);
   });
 
+  it("falls back to a static-volume mix instead of dropping audio when the automated mix fails", async () => {
+    const baseDir = mkdtempSync(join(tmpdir(), "hf-audio-base-"));
+    const workDir = mkdtempSync(join(tmpdir(), "hf-audio-work-"));
+    tempDirs.push(baseDir, workDir);
+
+    writeFileSync(join(baseDir, "bgm.wav"), "stub");
+
+    // Simulate an ffmpeg build that rejects the automation expression: the
+    // first mix attempt fails, the static-volume retry succeeds. (prepare =
+    // call 0, automated mix = call 1, fallback mix = call 2.)
+    runFfmpegMock
+      .mockImplementationOnce(async () => ({
+        success: true,
+        durationMs: 1,
+        stderr: "",
+        exitCode: 0,
+      }))
+      .mockImplementationOnce(async () => ({
+        success: false,
+        durationMs: 1,
+        stderr: "Error initializing filters",
+        exitCode: 234,
+      }));
+
+    const result = await processCompositionAudio(
+      [
+        {
+          id: "bgm",
+          src: "bgm.wav",
+          start: 0,
+          end: 5,
+          mediaStart: 0,
+          layer: 0,
+          volume: 0.8,
+          volumeKeyframes: [
+            { time: 0, volume: 0.8 },
+            { time: 5, volume: 0 },
+          ],
+          type: "audio",
+        },
+      ],
+      baseDir,
+      workDir,
+      join(baseDir, "out.m4a"),
+      5,
+    );
+
+    expect(result.success).toBe(true);
+    expect(result.tracksProcessed).toBe(1);
+    expect(runFfmpegMock).toHaveBeenCalledTimes(3);
+
+    // The fallback mix omits the automation expression (base volume only).
+    const fallbackArgs = runFfmpegMock.mock.calls[2]?.[0];
+    const fallbackFilter = fallbackArgs[fallbackArgs.indexOf("-filter_complex") + 1];
+    expect(fallbackFilter).not.toContain(":eval=frame");
+    expect(fallbackFilter).toContain("volume=0.8");
+  });
+
   it("prepares percent-encoded non-Latin audio srcs from decoded filesystem paths", async () => {
     const baseDir = mkdtempSync(join(tmpdir(), "hf-audio-base-"));
     const workDir = mkdtempSync(join(tmpdir(), "hf-audio-work-"));

packages/engine/src/services/audioMixer.ts

@@ -103,10 +103,10 @@ function simplifyVolumeKeyframes(
   return sampled;
 }
 
-function buildVolumeExpression(track: AudioTrack): string {
+function buildVolumeExpression(track: AudioTrack, ignoreKeyframes = false): string {
   const trimDuration = track.end - track.start;
   const staticVolume = clampVolume(track.volume);
-  const keyframes = (track.volumeKeyframes ?? [])
+  const keyframes = (ignoreKeyframes ? [] : (track.volumeKeyframes ?? []))
     .filter((keyframe) => Number.isFinite(keyframe.time) && Number.isFinite(keyframe.volume))
     .map((keyframe) => ({
       time: Math.max(0, Math.min(trimDuration, keyframe.time - track.start)),
@@ -377,42 +377,58 @@ async function mixAudioTracks(
   const outputDir = dirname(outputPath);
   if (!existsSync(outputDir)) mkdirSync(outputDir, { recursive: true });
 
-  const inputs: string[] = [];
-  const filterParts: string[] = [];
-
-  tracks.forEach((track, i) => {
-    inputs.push("-i", track.srcPath);
-    const delayMs = Math.round(track.start * 1000);
-    const trimDuration = track.end - track.start;
-    const volumeFilter = buildVolumeExpression(track);
-    filterParts.push(
-      `[${i}:a]atrim=0:${trimDuration},${volumeFilter},adelay=${delayMs}|${delayMs},apad=whole_dur=${totalDuration}[a${i}]`,
-    );
-  });
-
-  const mixInputs = tracks.map((_, i) => `[a${i}]`).join("");
-  const weights = tracks.map(() => "1").join(" ");
-  const mixFilter = `${mixInputs}amix=inputs=${tracks.length}:duration=longest:dropout_transition=0:normalize=0:weights='${weights}'[mixed]`;
-  const postMixGainFilter = `[mixed]volume=${masterOutputGain}[out]`;
-  const fullFilter = [...filterParts, mixFilter, postMixGainFilter].join(";");
+  const buildArgs = (ignoreAutomation: boolean): string[] => {
+    const inputs: string[] = [];
+    const filterParts: string[] = [];
+    tracks.forEach((track, i) => {
+      inputs.push("-i", track.srcPath);
+      const delayMs = Math.round(track.start * 1000);
+      const trimDuration = track.end - track.start;
+      const volumeFilter = buildVolumeExpression(track, ignoreAutomation);
+      filterParts.push(
+        `[${i}:a]atrim=0:${trimDuration},${volumeFilter},adelay=${delayMs}|${delayMs},apad=whole_dur=${totalDuration}[a${i}]`,
+      );
+    });
 
-  const args = [
-    ...inputs,
-    "-filter_complex",
-    fullFilter,
-    "-map",
-    "[out]",
-    "-acodec",
-    "aac",
-    "-b:a",
-    "192k",
-    "-t",
-    String(totalDuration),
-    "-y",
-    outputPath,
-  ];
+    const mixInputs = tracks.map((_, i) => `[a${i}]`).join("");
+    const weights = tracks.map(() => "1").join(" ");
+    const mixFilter = `${mixInputs}amix=inputs=${tracks.length}:duration=longest:dropout_transition=0:normalize=0:weights='${weights}'[mixed]`;
+    const postMixGainFilter = `[mixed]volume=${masterOutputGain}[out]`;
+    const fullFilter = [...filterParts, mixFilter, postMixGainFilter].join(";");
+
+    return [
+      ...inputs,
+      "-filter_complex",
+      fullFilter,
+      "-map",
+      "[out]",
+      "-acodec",
+      "aac",
+      "-b:a",
+      "192k",
+      "-t",
+      String(totalDuration),
+      "-y",
+      outputPath,
+    ];
+  };
 
-  const result = await runFfmpeg(args, { signal, timeout: ffmpegProcessTimeout });
+  let result = await runFfmpeg(buildArgs(false), { signal, timeout: ffmpegProcessTimeout });
+
+  // Defense in depth: volume automation is folded into an FFmpeg `volume`
+  // expression whose evaluator limits are build-dependent (see
+  // MAX_VOLUME_SEGMENTS). If that ever fails the mix, retry once without the
+  // automation so the track renders at its base volume rather than being
+  // dropped from the output entirely — a missing fade beats missing audio.
+  let degradedAutomation = false;
+  const hasAutomation = tracks.some((track) => (track.volumeKeyframes?.length ?? 0) > 0);
+  if (!result.success && !signal?.aborted && hasAutomation) {
+    const retry = await runFfmpeg(buildArgs(true), { signal, timeout: ffmpegProcessTimeout });
+    if (retry.success) {
+      result = retry;
+      degradedAutomation = true;
+    }
+  }
 
   if (signal?.aborted) {
     return {
@@ -438,6 +454,9 @@ async function mixAudioTracks(
     outputPath,
     durationMs: result.durationMs,
     tracksProcessed: tracks.length,
+    error: degradedAutomation
+      ? "Volume automation exceeded this ffmpeg build's expression limits; rendered at base volume"
+      : undefined,
   };
 }