fix: support animated media volume (#1066)

## Problem

Audio-capable media could only use a static `data-volume` value during preview/render. Issue #1064 reports this for `<audio>`: GSAP/JS attempts to fade an element from `volume: 0` to a non-zero value still rendered as silent because HyperFrames only respected the initial `data-volume` value.

The same root cause applies to `<video data-has-audio="true">`: preview/runtime can animate the media element, but the offline audio mix previously only had one static volume number for the extracted audio track.

Closes #1064.

## What this fixes

- Preserves authored `HTMLMediaElement.volume` changes made by GSAP/JS between runtime media sync ticks instead of clobbering them back to `data-volume`.
- Updates the WebAudio transport gain for active media sources when the element volume changes.
- Probes scripted media timelines in the producer browser pass and records sampled volume keyframes for both `<audio>` elements and video-derived audio tracks.
- Maps extracted video audio IDs such as `bg-video-audio` back to their source `<video id="bg-video">` so video volume automation is sampled from the actual element.
- Converts sampled keyframes into a frame-evaluated FFmpeg `volume` expression during audio mixing, so rendered output includes fades and other timeline-driven media volume changes.

## Root cause

There were two static-volume paths:

1. Runtime media sync treated `clip.volume` parsed from `data-volume` as authoritative on every tick and rewrote `el.volume = clip.volume * userVolume`, undoing GSAP/JS updates after timeline seeks.
2. The producer audio stage mixed audio from compile/probe metadata, where `volume` was a single number. The FFmpeg filter used `volume=<initial value>`, so a clip starting at `data-volume="0"` stayed silent in the muxed output even if the browser timeline had changed `audio.volume` or `video.volume`.

The fix makes the browser/runtime media volume the source of truth when authors animate it, then carries that time-varying signal into the offline mix.

## Verification

### Local checks

- `bun install` in the clean worktree
- `bun run build:hyperframes-runtime`
- `bun run --filter @hyperframes/core test -- src/runtime/media.test.ts`
- `bun run --filter @hyperframes/engine test -- src/services/audioMixer.test.ts`
- `bun run --filter @hyperframes/core typecheck && bun run --filter @hyperframes/engine typecheck && bun run --filter @hyperframes/producer typecheck`
- `bunx oxfmt --check <10 touched files>`
- `bunx oxlint <10 touched files>`
- `bun run --filter @hyperframes/producer build`

### Audio repro and render proof

Created `/tmp/hf-1064` with `<audio id="bg-audio" ... data-volume="0">` and `gsap.to(audio, { volume: 1 })`.

Before fix:
- `RMS level dB: -inf`

After fix:
- `[Probe] Runtime audio volume automation: bg-audio {"keyframeCount":12}`
- `RMS level dB: -24.133461`

### Video repro and render proof

Created `/tmp/hf-1064-video` with `<video id="bg-video" data-has-audio="true" data-volume="0">` and `gsap.to(video, { volume: 1 })`.

After widening the probe mapping:
- `[Probe] Runtime audio volume automation: bg-video-audio {"keyframeCount":12}`
- `RMS level dB: -24.203664`

### Preview verification

Used `hyperframes preview` from the PR branch and `agent-browser` against Studio:

- Audio preview direct seek through `window.__player.seek(...)` showed `#bg-audio.volume`: `0 -> 0.5 -> 1`.
- Audio preview playback advanced to ~0.92s with `#bg-audio.volume` ~0.917 and the audio element unpaused.
- Video preview direct seek showed `#bg-video.volume`: `0 -> 0.5 -> 1`.
- Video preview playback sampled during the fade at ~0.68s showed `#bg-video.volume` around `0.664`.

### Browser verification

Used `agent-browser` against the rendered MP4 from the PR branch:

- Opened `file:///tmp/hf-1064/out-pr.mp4`
- Verified the browser loaded a playable `<video>`: `{"hasVideo":true,"duration":3.020996,"readyState":4,"paused":false}`
- Screenshot artifact: `/Users/miguel07code/dev/hyperframes-oss/qa-artifacts/issue-1064/pr-rendered-video.png`
- Recording artifact: `/Users/miguel07code/dev/hyperframes-oss/qa-artifacts/issue-1064/pr-rendered-video.webm`

## Notes

- The commit was prepared in a clean worktree based on `origin/main` to avoid mixing in the existing `fix/nonlatin-media-src-resolution` checkout changes.
- Pre-commit lint, format, and typecheck passed. The hook's Fallow audit still exits non-zero on inherited duplication/complexity in the touched large runtime/probe files; the dead-code issue from the new type export was fixed before committing.
- Volume automation is sampled from the browser timeline and approximated as piecewise-linear FFmpeg volume expressions. This is intended for timeline-driven GSAP/JS fades, not audio-rate DSP.

Author: miguel-heygen

packages/core/src/runtime/init.ts

@@ -1241,6 +1241,10 @@ export function initSandboxRuntimeModular(): void {
     for (const mediaEl of mediaEls) {
       if (metadataBoundMedia.has(mediaEl)) continue;
       metadataBoundMedia.add(mediaEl);
+      const parsedVolume = Number.parseFloat(mediaEl.dataset.volume ?? "");
+      if (Number.isFinite(parsedVolume)) {
+        mediaEl.volume = Math.max(0, Math.min(1, parsedVolume));
+      }
       mediaEl.addEventListener("loadedmetadata", scheduleMetadataDurationHydration);
       mediaEl.addEventListener("durationchange", scheduleMetadataDurationHydration);
 
@@ -1312,6 +1316,7 @@ export function initSandboxRuntimeModular(): void {
       userMuted: state.bridgeMuted,
       userVolume: state.bridgeVolume,
       forceSync,
+      onElementVolume: (el, volume) => webAudio.setElementVolume(el, volume),
       onAutoplayBlocked: () => {
         if (state.mediaAutoplayBlockedPosted) return;
         state.mediaAutoplayBlockedPosted = true;
@@ -1461,8 +1466,8 @@ export function initSandboxRuntimeModular(): void {
         externalCompositionsReady = true;
         bindRootTimelineIfAvailable();
         window.__renderReady = true;
-        runAdapters("discover", state.currentTime);
         bindMediaMetadataListeners();
+        runAdapters("discover", state.currentTime);
         installAssetFailureDiagnostics();
         applyCaptionOverrides();
         postTimeline();
@@ -1667,8 +1672,8 @@ export function initSandboxRuntimeModular(): void {
   ] as RuntimeDeterministicAdapter[];
   patchVideoTextureCompat();
   installRuntimeErrorDiagnostics();
-  runAdapters("discover");
   bindMediaMetadataListeners();
+  runAdapters("discover");
   // ── Single-clock transport ──
   //
   // TransportClock is the sole time authority. GSAP is always paused —

packages/core/src/runtime/media.test.ts

@@ -343,6 +343,41 @@ describe("syncRuntimeMedia", () => {
     expect(clip.el.volume).toBeCloseTo(0.3);
   });
 
+  it("preserves authored volume changes made between sync ticks", () => {
+    const clip = createMockClip({ start: 0, end: 10, volume: 0 });
+    syncRuntimeMedia({ clips: [clip], timeSeconds: 0, playing: false, playbackRate: 1 });
+    expect(clip.el.volume).toBe(0);
+
+    clip.el.volume = 0.5;
+    syncRuntimeMedia({ clips: [clip], timeSeconds: 0.5, playing: false, playbackRate: 1 });
+
+    expect(clip.el.volume).toBe(0.5);
+  });
+
+  it("reports the effective element volume to external audio transports", () => {
+    const clip = createMockClip({ start: 0, end: 10, volume: 0 });
+    const onElementVolume = vi.fn();
+    syncRuntimeMedia({
+      clips: [clip],
+      timeSeconds: 0,
+      playing: false,
+      playbackRate: 1,
+      onElementVolume,
+    });
+    clip.el.volume = 0.75;
+    syncRuntimeMedia({
+      clips: [clip],
+      timeSeconds: 1,
+      playing: false,
+      playbackRate: 1,
+      userVolume: 0.5,
+      onElementVolume,
+    });
+
+    expect(clip.el.volume).toBeCloseTo(0.375);
+    expect(onElementVolume).toHaveBeenLastCalledWith(clip.el, 0.375);
+  });
+
   it("hard-syncs on the first active tick (sub-composition activation, mediaStart offsets)", () => {
     const clip = createMockClip({ start: 0, end: 10, mediaStart: 0 });
     Object.defineProperty(clip.el, "currentTime", { value: 0, writable: true });

packages/core/src/runtime/media.ts

@@ -103,6 +103,13 @@ function markPlayRequested(el: HTMLMediaElement): void {
   el.addEventListener("error", clear, { once: true });
 }
 
+const lastRuntimeAppliedVolume = new WeakMap<HTMLMediaElement, number>();
+
+function clampVolume(volume: number): number {
+  if (!Number.isFinite(volume)) return 1;
+  return Math.max(0, Math.min(1, volume));
+}
+
 export function syncRuntimeMedia(params: {
   clips: RuntimeMediaClip[];
   timeSeconds: number;
@@ -132,6 +139,7 @@ export function syncRuntimeMedia(params: {
    * outbound message; further invocations are suppressed by the caller.
    */
   onAutoplayBlocked?: () => void;
+  onElementVolume?: (el: HTMLMediaElement, volume: number) => void;
   forceSync?: boolean;
 }): void {
   // Either flag silences output. Combined up front so the per-clip loop is
@@ -151,8 +159,19 @@ export function syncRuntimeMedia(params: {
           relTime = clip.mediaStart + ((relTime - clip.mediaStart) % loopLength);
         }
       }
-      const userVol = params.userVolume ?? 1;
-      el.volume = (clip.volume ?? 1) * userVol;
+      const userVol = clampVolume(params.userVolume ?? 1);
+      const fallbackAuthorVolume = clampVolume(clip.volume ?? 1);
+      const previousRuntimeVolume = lastRuntimeAppliedVolume.get(el);
+      const currentElementVolume = clampVolume(el.volume);
+      const authorVolume =
+        previousRuntimeVolume !== undefined &&
+        Math.abs(currentElementVolume - previousRuntimeVolume) > 0.0001
+          ? currentElementVolume
+          : fallbackAuthorVolume;
+      const effectiveVolume = clampVolume(authorVolume * userVol);
+      el.volume = effectiveVolume;
+      lastRuntimeAppliedVolume.set(el, effectiveVolume);
+      params.onElementVolume?.(el, effectiveVolume);
       if (shouldMute) el.muted = true;
       // Ensure full preload for every active media element. Streaming
       // formats (MP3) may arrive with preload="metadata", which only
@@ -283,6 +302,7 @@ export function syncRuntimeMedia(params: {
     lastOffset.delete(el);
     strictDriftSamples.delete(el);
     seekLoadRetried.delete(el);
+    lastRuntimeAppliedVolume.delete(el);
     if (!el.paused) el.pause();
   }
 }

packages/core/src/runtime/webAudioTransport.ts

@@ -191,6 +191,18 @@ export class WebAudioTransport {
     }
   }
 
+  setElementVolume(el: HTMLMediaElement, volume: number): void {
+    const safeVolume = Math.max(0, Math.min(1, volume));
+    for (const source of this._activeSources) {
+      if (source.el !== el) continue;
+      try {
+        source.gainNode.gain.value = safeVolume;
+      } catch (err) {
+        swallow("webAudioTransport.setElementVolume", err);
+      }
+    }
+  }
+
   setMuted(muted: boolean): void {
     if (this._masterGain) {
       this._masterGain.gain.value = muted ? 0 : 1;

packages/engine/src/index.ts

@@ -135,7 +135,12 @@ export {
 export { createVideoFrameInjector } from "./services/videoFrameInjector.js";
 
 export { parseAudioElements, processCompositionAudio } from "./services/audioMixer.js";
-export type { AudioElement, AudioTrack, MixResult } from "./services/audioMixer.types.js";
+export type {
+  AudioElement,
+  AudioTrack,
+  AudioVolumeKeyframe,
+  MixResult,
+} from "./services/audioMixer.types.js";
 
 // ── Parallel rendering ─────────────────────────────────────────────────────────
 export {

packages/engine/src/services/audioMixer.test.ts

@@ -65,6 +65,49 @@ describe("processCompositionAudio", () => {
     expect(filter).toContain("[mixed]volume=1[out]");
   });
 
+  it("uses frame-evaluated volume automation when keyframes are present", async () => {
+    const baseDir = mkdtempSync(join(tmpdir(), "hf-audio-base-"));
+    const workDir = mkdtempSync(join(tmpdir(), "hf-audio-work-"));
+    tempDirs.push(baseDir, workDir);
+
+    writeFileSync(join(baseDir, "voice.wav"), "stub");
+
+    const result = await processCompositionAudio(
+      [
+        {
+          id: "voice",
+          src: "voice.wav",
+          start: 2,
+          end: 5,
+          mediaStart: 0,
+          layer: 0,
+          volume: 0,
+          volumeKeyframes: [
+            { time: 2, volume: 0 },
+            { time: 3, volume: 1 },
+            { time: 5, volume: 0.5 },
+          ],
+          type: "audio",
+        },
+      ],
+      baseDir,
+      workDir,
+      join(baseDir, "out.m4a"),
+      5,
+    );
+
+    expect(result.success).toBe(true);
+
+    const mixArgs = runFfmpegMock.mock.calls[1]?.[0];
+    const filterIndex = mixArgs.indexOf("-filter_complex");
+    const filter = mixArgs[filterIndex + 1];
+
+    expect(filter).toContain("volume=");
+    expect(filter).toContain(":eval=frame");
+    expect(filter).toContain("lt(t\\,1)");
+    expect(filter).toContain("adelay=2000|2000");
+  });
+
   it("prepares percent-encoded non-Latin audio srcs from decoded filesystem paths", async () => {
     const baseDir = mkdtempSync(join(tmpdir(), "hf-audio-base-"));
     const workDir = mkdtempSync(join(tmpdir(), "hf-audio-work-"));

packages/engine/src/services/audioMixer.ts

@@ -15,7 +15,67 @@ import { unwrapTemplate } from "../utils/htmlTemplate.js";
 import { resolveProjectRelativeSrc } from "./videoFrameExtractor.js";
 import type { AudioElement, AudioTrack, MixResult } from "./audioMixer.types.js";
 
-export type { AudioElement, AudioTrack, MixResult } from "./audioMixer.types.js";
+export type { AudioElement, MixResult } from "./audioMixer.types.js";
+
+function clampVolume(volume: number): number {
+  if (!Number.isFinite(volume)) return 1;
+  return Math.max(0, Math.min(1, volume));
+}
+
+function formatFilterNumber(value: number): string {
+  return Number(value.toFixed(6)).toString();
+}
+
+function escapeExpressionCommas(expression: string): string {
+  return expression.replace(/\\/g, "\\\\").replace(/,/g, "\\,");
+}
+
+function buildVolumeExpression(track: AudioTrack): string {
+  const trimDuration = track.end - track.start;
+  const staticVolume = clampVolume(track.volume);
+  const keyframes = (track.volumeKeyframes ?? [])
+    .filter((keyframe) => Number.isFinite(keyframe.time) && Number.isFinite(keyframe.volume))
+    .map((keyframe) => ({
+      time: Math.max(0, Math.min(trimDuration, keyframe.time - track.start)),
+      volume: clampVolume(keyframe.volume),
+    }))
+    .sort((a, b) => a.time - b.time);
+
+  if (keyframes.length === 0) return `volume=${formatFilterNumber(staticVolume)}`;
+
+  if (keyframes[0]!.time > 0) {
+    keyframes.unshift({ time: 0, volume: staticVolume });
+  }
+
+  const deduped: typeof keyframes = [];
+  for (const keyframe of keyframes) {
+    const previous = deduped.at(-1);
+    if (previous && Math.abs(previous.time - keyframe.time) < 0.000001) {
+      previous.volume = keyframe.volume;
+    } else {
+      deduped.push(keyframe);
+    }
+  }
+
+  if (deduped.length === 1) {
+    return `volume=${formatFilterNumber(deduped[0]!.volume)}`;
+  }
+
+  let expression = formatFilterNumber(deduped.at(-1)!.volume);
+  for (let i = deduped.length - 2; i >= 0; i -= 1) {
+    const current = deduped[i]!;
+    const next = deduped[i + 1]!;
+    const currentTime = formatFilterNumber(current.time);
+    const nextTime = formatFilterNumber(next.time);
+    const currentVolume = formatFilterNumber(current.volume);
+    const span = Math.max(0.000001, next.time - current.time);
+    const slope = formatFilterNumber((next.volume - current.volume) / span);
+    const segment = `${currentVolume}+(${slope})*(t-${currentTime})`;
+    expression = `if(lt(t,${nextTime}),${segment},${expression})`;
+  }
+
+  return `volume=${escapeExpressionCommas(expression)}:eval=frame`;
+}
 
 interface ExtractResult {
   success: boolean;
@@ -246,8 +306,9 @@ async function mixAudioTracks(
     inputs.push("-i", track.srcPath);
     const delayMs = Math.round(track.start * 1000);
     const trimDuration = track.end - track.start;
+    const volumeFilter = buildVolumeExpression(track);
     filterParts.push(
-      `[${i}:a]atrim=0:${trimDuration},volume=${track.volume},adelay=${delayMs}|${delayMs},apad=whole_dur=${totalDuration}[a${i}]`,
+      `[${i}:a]atrim=0:${trimDuration},${volumeFilter},adelay=${delayMs}|${delayMs},apad=whole_dur=${totalDuration}[a${i}]`,
     );
   });
 
@@ -399,6 +460,7 @@ export async function processCompositionAudio(
           mediaStart: element.mediaStart,
           duration: element.end - element.start,
           volume: element.volume ?? 1.0,
+          volumeKeyframes: element.volumeKeyframes,
         });
       } catch (err: unknown) {
         errors.push(`Error: ${element.id} — ${err instanceof Error ? err.message : String(err)}`);

packages/engine/src/services/audioMixer.types.ts

@@ -1,3 +1,8 @@
+export interface AudioVolumeKeyframe {
+  time: number;
+  volume: number;
+}
+
 export interface AudioElement {
   id: string;
   src: string;
@@ -6,6 +11,7 @@ export interface AudioElement {
   mediaStart: number;
   layer: number;
   volume?: number;
+  volumeKeyframes?: AudioVolumeKeyframe[];
   type: "audio" | "video";
 }
 
@@ -17,6 +23,7 @@ export interface AudioTrack {
   mediaStart: number;
   duration: number;
   volume: number;
+  volumeKeyframes?: AudioVolumeKeyframe[];
 }
 
 export interface MixResult {