feat(engine): bake volume automation into PCM samples (sample-accurate, no ceiling)

Make sample-level gain the primary path for time-varying volume: after a track's
WAV is prepared (always pcm_s16le/48k/stereo), multiply its samples by the
interpolated envelope in-house, then mix at unity. This is exact at every sample,
has no keyframe ceiling, and leaves the downstream ffmpeg amix/AAC encode
untouched — so output and golden baselines only change where a fade is applied.
It's the MoviePy/MLT model done in-process, with no new dependency.

The RDP-bounded ffmpeg `volume` expression remains as a fallback for the rare
case where a WAV isn't the expected 16-bit PCM, and the static-volume retry
still backstops that. Layered: exact PCM gain -> bounded expression -> base
volume, so the audio track is never dropped.

Verified: the 297-keyframe fade that rendered with no audio now bakes all 297
keyframes sample-accurately (confirmed baked=true at runtime). Adds unit tests
for sample-accurate gain, track-start offset, base/tail holds, thousands of
keyframes, and format rejection.

Author: miguel-heygen

packages/engine/src/services/audioMixer.ts

@@ -14,6 +14,7 @@ import { runFfmpeg } from "../utils/runFfmpeg.js";
 import { unwrapTemplate } from "../utils/htmlTemplate.js";
 import { resolveProjectRelativeSrc } from "./videoFrameExtractor.js";
 import type { AudioElement, AudioTrack, MixResult } from "./audioMixer.types.js";
+import { applyVolumeEnvelopeToWav } from "./audioVolumeEnvelope.js";
 
 export type { AudioElement, MixResult } from "./audioMixer.types.js";
 
@@ -555,15 +556,29 @@ export async function processCompositionAudio(
           audioSrcPath = trimmedPath;
         }
 
+        // Primary volume-automation path: bake the envelope into the PCM samples
+        // (sample-accurate, no keyframe ceiling). If the WAV isn't the expected
+        // 16-bit PCM, fall back to the ffmpeg expression path by leaving the
+        // keyframes on the track for buildVolumeExpression to handle.
+        let bakedEnvelope = false;
+        if (element.volumeKeyframes && element.volumeKeyframes.length > 0) {
+          bakedEnvelope = applyVolumeEnvelopeToWav(
+            audioSrcPath,
+            element.volumeKeyframes,
+            element.start,
+            element.volume ?? 1.0,
+          );
+        }
         tracks.push({
           id: element.id,
           srcPath: audioSrcPath,
           start: element.start,
           end: element.end,
           mediaStart: element.mediaStart,
           duration: element.end - element.start,
-          volume: element.volume ?? 1.0,
-          volumeKeyframes: element.volumeKeyframes,
+          // Gain is already in the samples when baked, so mix at unity.
+          volume: bakedEnvelope ? 1.0 : (element.volume ?? 1.0),
+          volumeKeyframes: bakedEnvelope ? undefined : element.volumeKeyframes,
         });
       } catch (err: unknown) {
         errors.push(`Error: ${element.id} — ${err instanceof Error ? err.message : String(err)}`);

packages/engine/src/services/audioVolumeEnvelope.test.ts

@@ -0,0 +1,150 @@
+import { afterEach, describe, expect, it } from "vitest";
+import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import { tmpdir } from "node:os";
+import { applyVolumeEnvelopeToWav } from "./audioVolumeEnvelope.js";
+
+const SAMPLE_RATE = 48000;
+const CHANNELS = 2;
+
+/** Build a PCM s16le stereo WAV whose every sample equals `value`. */
+function writeConstantWav(path: string, frames: number, value: number): void {
+  const bytesPerSample = 2;
+  const dataSize = frames * CHANNELS * bytesPerSample;
+  const buffer = Buffer.alloc(44 + dataSize);
+  buffer.write("RIFF", 0, "ascii");
+  buffer.writeUInt32LE(36 + dataSize, 4);
+  buffer.write("WAVE", 8, "ascii");
+  buffer.write("fmt ", 12, "ascii");
+  buffer.writeUInt32LE(16, 16);
+  buffer.writeUInt16LE(1, 20); // PCM
+  buffer.writeUInt16LE(CHANNELS, 22);
+  buffer.writeUInt32LE(SAMPLE_RATE, 24);
+  buffer.writeUInt32LE(SAMPLE_RATE * CHANNELS * bytesPerSample, 28);
+  buffer.writeUInt16LE(CHANNELS * bytesPerSample, 32);
+  buffer.writeUInt16LE(16, 34);
+  buffer.write("data", 36, "ascii");
+  buffer.writeUInt32LE(dataSize, 40);
+  for (let i = 0; i < frames * CHANNELS; i += 1) buffer.writeInt16LE(value, 44 + i * 2);
+  writeFileSync(path, buffer);
+}
+
+function sampleAt(path: string, frame: number, channel = 0): number {
+  const buffer = readFileSync(path);
+  return buffer.readInt16LE(44 + (frame * CHANNELS + channel) * 2);
+}
+
+describe("applyVolumeEnvelopeToWav", () => {
+  const dirs: string[] = [];
+  const tmp = () => {
+    const d = mkdtempSync(join(tmpdir(), "hf-env-"));
+    dirs.push(d);
+    return d;
+  };
+  afterEach(() => {
+    for (const d of dirs.splice(0)) rmSync(d, { recursive: true, force: true });
+  });
+
+  it("applies a linear fade sample-accurately", () => {
+    const path = join(tmp(), "a.wav");
+    const frames = SAMPLE_RATE; // 1 second
+    writeConstantWav(path, frames, 10000);
+
+    // Fade 0 -> 1 over the full second.
+    const applied = applyVolumeEnvelopeToWav(
+      path,
+      [
+        { time: 0, volume: 0 },
+        { time: 1, volume: 1 },
+      ],
+      0,
+      0,
+    );
+    expect(applied).toBe(true);
+
+    expect(sampleAt(path, 0)).toBe(0); // gain 0
+    expect(sampleAt(path, frames / 2)).toBeCloseTo(5000, -2); // gain ~0.5
+    expect(sampleAt(path, frames - 1)).toBeGreaterThan(9900); // gain ~1
+  });
+
+  it("offsets keyframes by the track start (composition time -> track-relative)", () => {
+    const path = join(tmp(), "b.wav");
+    const frames = SAMPLE_RATE;
+    writeConstantWav(path, frames, 10000);
+
+    // Track starts at 5s; the fade runs from comp-time 5s..6s -> wav 0s..1s.
+    applyVolumeEnvelopeToWav(
+      path,
+      [
+        { time: 5, volume: 0 },
+        { time: 6, volume: 1 },
+      ],
+      5,
+      0,
+    );
+
+    expect(sampleAt(path, 0)).toBe(0);
+    expect(sampleAt(path, frames / 2)).toBeCloseTo(5000, -2);
+  });
+
+  it("holds base volume before the first keyframe and the last value after", () => {
+    const path = join(tmp(), "c.wav");
+    const frames = SAMPLE_RATE * 3; // 3 seconds
+    writeConstantWav(path, frames, 10000);
+
+    // Base 0.8 held until a fade-out begins at 2s.
+    applyVolumeEnvelopeToWav(
+      path,
+      [
+        { time: 2, volume: 0.8 },
+        { time: 3, volume: 0 },
+      ],
+      0,
+      0.8,
+    );
+
+    expect(sampleAt(path, SAMPLE_RATE)).toBeCloseTo(8000, -2); // 1s: base 0.8
+    expect(sampleAt(path, frames - 1)).toBeLessThan(200); // 3s: faded to ~0
+  });
+
+  it("handles thousands of keyframes without failing (no expression ceiling)", () => {
+    const path = join(tmp(), "d.wav");
+    const frames = SAMPLE_RATE * 2;
+    writeConstantWav(path, frames, 10000);
+
+    const keyframes = Array.from({ length: 5000 }, (_, i) => ({
+      time: (i / 4999) * 2,
+      volume: Math.abs(Math.sin(i / 50)),
+    }));
+    expect(applyVolumeEnvelopeToWav(path, keyframes, 0, 0)).toBe(true);
+  });
+
+  it("rejects non-16-bit PCM so the caller can fall back", () => {
+    const path = join(tmp(), "e.wav");
+    // 24-bit PCM header (bitsPerSample = 24); body contents are irrelevant.
+    const buffer = Buffer.alloc(44);
+    buffer.write("RIFF", 0, "ascii");
+    buffer.write("WAVE", 8, "ascii");
+    buffer.write("fmt ", 12, "ascii");
+    buffer.writeUInt32LE(16, 16);
+    buffer.writeUInt16LE(1, 20);
+    buffer.writeUInt16LE(CHANNELS, 22);
+    buffer.writeUInt32LE(SAMPLE_RATE, 24);
+    buffer.writeUInt16LE(24, 34);
+    buffer.write("data", 36, "ascii");
+    buffer.writeUInt32LE(0, 40);
+    writeFileSync(path, buffer);
+
+    expect(
+      applyVolumeEnvelopeToWav(
+        path,
+        [
+          { time: 0, volume: 0 },
+          { time: 1, volume: 1 },
+        ],
+        0,
+        0,
+      ),
+    ).toBe(false);
+  });
+});

packages/engine/src/services/audioVolumeEnvelope.ts

@@ -0,0 +1,152 @@
+/**
+ * Sample-accurate volume automation.
+ *
+ * The audio mixer's primary path for time-varying volume bakes the envelope
+ * directly into the prepared PCM rather than encoding it as an FFmpeg `volume`
+ * expression. The expression approach nests one `if(lt(t,...))` per keyframe and
+ * overflows FFmpeg's expression evaluator past ~95 levels (a dense GSAP fade
+ * emits hundreds of keyframes), which fails the whole mix and drops the audio
+ * track. Multiplying the samples in-house has no such ceiling, is exact at every
+ * sample, and keeps the downstream ffmpeg `amix`/AAC encode untouched — so the
+ * output (and the golden baselines) only change where a fade is actually applied.
+ *
+ * The prepared tracks are always `pcm_s16le`, 48 kHz, stereo (see
+ * `prepareAudioTrack` / `extractAudioFromVideo`). Anything else is rejected so
+ * the caller can fall back to the expression path rather than corrupting audio.
+ */
+
+import { readFileSync, writeFileSync } from "fs";
+import type { AudioVolumeKeyframe } from "./audioMixer.types.js";
+
+const PCM_FORMAT = 1; // WAVE_FORMAT_PCM
+const SUPPORTED_BITS = 16;
+
+interface WavLayout {
+  numChannels: number;
+  sampleRate: number;
+  dataOffset: number;
+  dataSize: number;
+}
+
+/** Locate the `fmt ` and `data` chunks and validate the format we know how to edit. */
+function parseWavLayout(buffer: Buffer): WavLayout | null {
+  if (buffer.length < 12 || buffer.toString("ascii", 0, 4) !== "RIFF") return null;
+  if (buffer.toString("ascii", 8, 12) !== "WAVE") return null;
+
+  let offset = 12;
+  let fmt: { numChannels: number; sampleRate: number; bitsPerSample: number } | null = null;
+  let data: { offset: number; size: number } | null = null;
+
+  while (offset + 8 <= buffer.length) {
+    const chunkId = buffer.toString("ascii", offset, offset + 4);
+    const chunkSize = buffer.readUInt32LE(offset + 4);
+    const body = offset + 8;
+    if (chunkId === "fmt " && body + 16 <= buffer.length) {
+      if (buffer.readUInt16LE(body) !== PCM_FORMAT) return null;
+      fmt = {
+        numChannels: buffer.readUInt16LE(body + 2),
+        sampleRate: buffer.readUInt32LE(body + 4),
+        bitsPerSample: buffer.readUInt16LE(body + 14),
+      };
+    } else if (chunkId === "data") {
+      data = { offset: body, size: Math.min(chunkSize, buffer.length - body) };
+      break; // sample data follows; no need to scan further
+    }
+    // Chunks are word-aligned: an odd size carries a trailing pad byte.
+    offset = body + chunkSize + (chunkSize % 2);
+  }
+
+  if (!fmt || !data) return null;
+  if (fmt.bitsPerSample !== SUPPORTED_BITS || fmt.numChannels < 1) return null;
+  return {
+    numChannels: fmt.numChannels,
+    sampleRate: fmt.sampleRate,
+    dataOffset: data.offset,
+    dataSize: data.size,
+  };
+}
+
+/**
+ * Normalise keyframes to track-relative seconds, sorted and de-duplicated, with
+ * `baseVolume` filling any gap before the first keyframe. Returns the breakpoints
+ * the gain envelope is linearly interpolated between.
+ */
+function toRelativeEnvelope(
+  keyframes: AudioVolumeKeyframe[],
+  trackStart: number,
+  baseVolume: number,
+): { time: number; volume: number }[] {
+  const points = keyframes
+    .filter((k) => Number.isFinite(k.time) && Number.isFinite(k.volume))
+    .map((k) => ({
+      time: Math.max(0, k.time - trackStart),
+      volume: Math.max(0, Math.min(1, k.volume)),
+    }))
+    .sort((a, b) => a.time - b.time);
+
+  const deduped: { time: number; volume: number }[] = [];
+  for (const point of points) {
+    const previous = deduped.at(-1);
+    if (previous && Math.abs(previous.time - point.time) < 1e-9) previous.volume = point.volume;
+    else deduped.push(point);
+  }
+
+  if (deduped.length === 0) return deduped;
+  if (deduped[0]!.time > 0) {
+    deduped.unshift({ time: 0, volume: Math.max(0, Math.min(1, baseVolume)) });
+  }
+  return deduped;
+}
+
+/**
+ * Multiply a prepared WAV's samples by a time-varying gain envelope in place.
+ *
+ * @returns `true` if the envelope was applied; `false` if the file isn't the
+ *   expected 16-bit PCM (caller should fall back to the expression path).
+ */
+export function applyVolumeEnvelopeToWav(
+  wavPath: string,
+  keyframes: AudioVolumeKeyframe[],
+  trackStart: number,
+  baseVolume: number,
+): boolean {
+  const envelope = toRelativeEnvelope(keyframes, trackStart, baseVolume);
+  if (envelope.length === 0) return false;
+
+  try {
+    const buffer = readFileSync(wavPath);
+    const layout = parseWavLayout(buffer);
+    if (!layout) return false;
+
+    const { numChannels, sampleRate, dataOffset, dataSize } = layout;
+    const bytesPerSample = SUPPORTED_BITS / 8;
+    const frameBytes = numChannels * bytesPerSample;
+    const frameCount = Math.floor(dataSize / frameBytes);
+
+    let segment = 0;
+    for (let frame = 0; frame < frameCount; frame += 1) {
+      const time = frame / sampleRate;
+      while (segment < envelope.length - 2 && time >= envelope[segment + 1]!.time) segment += 1;
+
+      const a = envelope[segment]!;
+      const b = envelope[segment + 1] ?? a;
+      const span = b.time - a.time;
+      const progress = span <= 0 ? 0 : Math.min(1, Math.max(0, (time - a.time) / span));
+      const gain = a.volume + (b.volume - a.volume) * progress;
+
+      const base = dataOffset + frame * frameBytes;
+      for (let channel = 0; channel < numChannels; channel += 1) {
+        const at = base + channel * bytesPerSample;
+        const scaled = Math.round(buffer.readInt16LE(at) * gain);
+        buffer.writeInt16LE(scaled < -32768 ? -32768 : scaled > 32767 ? 32767 : scaled, at);
+      }
+    }
+
+    writeFileSync(wavPath, buffer);
+    return true;
+  } catch {
+    // Any read/parse/write failure → leave the file untouched and let the
+    // caller fall back to the ffmpeg expression path rather than losing audio.
+    return false;
+  }
+}