feat(producer): add Rio-style residual-RMS check to regression harness (#882)

* feat(producer): add Rio-style residual-RMS check to regression harness

The existing audio comparison in the regression harness measures the
Pearson correlation between RMS envelopes of the rendered and snapshot
streams. That catches shape-level drift but is insensitive to level
shifts, phase offsets, or codec-quantization noise — two streams can
correlate >0.9 while differing audibly.

Rio's approach (rio/tests/checksum.py:compare_audio_files_ffmpeg) is
sample-level: subtract the snapshot from the rendered stream, run
`astats`, read the residual Overall RMS in dBFS. Identical streams
cancel to silence (-inf, or sub -90 dBFS for AAC-vs-AAC); anything
>= -50 dBFS is considered drift.

This commit adds the same check as an optional secondary gate:

  - utils/audioRegression.ts: new `computeAudioResidualRmsDb()` that
    spawns ffmpeg with the same filter graph Rio uses (aresample +
    pan + volume=-1 + amix + astats) and returns the parsed Overall
    RMS plus a pass/fail flag.
  - utils/audioRegression.test.ts: 3 new tests covering identical
    streams (-inf result), drifted streams (440Hz vs 880Hz sine),
    and missing-audio-stream input.
  - regression-harness.ts: optional `maxAudioResidualRmsDb` field in
    meta.json. Default is undefined (skip the check) so legacy
    fixtures aren't retroactively gated; new fixtures opt in by
    setting a threshold (e.g. -50). Harness emits `residualRmsDb` in
    the audio_comparison_complete JSON event and the pretty log line.

The existing correlation check stays in place; the new residual check
is independent. They measure complementary properties (shape vs
sample-cancellation) and both should hold for a faithful render.

* fix(producer): harden residual-RMS check (parser, duration guard, error surfacing)

Addresses review feedback on PR #882:

- Stateful astats parse: modern ffmpeg emits `Overall` on its own line
  followed by per-stat lines, so the single-line `Overall RMS level dB:`
  regex never fires on 6.x/7.x/8.x. Find the `Overall` header, take the
  next `RMS level dB:` line. Single-line fallback preserved for 4.x.
- Pre-probe both inputs' audio durations and fail up-front if they differ
  by >5 ms — `amix=duration=shortest` was silently masking trailing
  audio differences.
- Surface ffmpeg/ffprobe spawn errors, signal kills, and non-zero exits
  with a stderr tail. Previously every failure mode collapsed into
  "NaN, fail" with no diagnostic.
- Extend `TestResult.audio` with `residualRmsDb` + `residualError`,
  propagate to `audio-failures.json`.
- Fix `residualSuffix` formatter: NaN (real failure) was being rendered
  as "-inf dBFS" (perfect match). Split the branch on `Number.isNaN`
  separately from `Number.isFinite` and add an explicit error label.

Author: jrusso1020

packages/producer/src/regression-harness.ts

@@ -18,7 +18,11 @@ import { createRenderJob, executeRenderJob } from "./services/renderOrchestrator
 import { compileForRender } from "./services/htmlCompiler.js";
 import { validateCompilation } from "./services/compilationTester.js";
 import { extractMediaMetadata } from "./utils/ffprobe.js";
-import { buildRmsEnvelope, compareAudioEnvelopes } from "./utils/audioRegression.js";
+import {
+  buildRmsEnvelope,
+  compareAudioEnvelopes,
+  computeAudioResidualRmsDb,
+} from "./utils/audioRegression.js";
 import { parseFps, fpsToNumber } from "@hyperframes/core";
 import {
   checkDistributedSupport,
@@ -38,6 +42,15 @@ type TestMetadata = {
   maxFrameFailures: number;
   minAudioCorrelation: number;
   maxAudioLagWindows: number;
+  /**
+   * Optional residual-RMS check. Subtracts the rendered audio from the
+   * baseline and reads the residual Overall RMS via `astats`. A value
+   * of `-50` treats residuals at-or-below -50 dBFS as effectively-
+   * silent — i.e. the streams are sample-level equivalent. Omit
+   * (undefined) to skip the check; fixtures authored before this field
+   * was introduced have implicit `undefined`.
+   */
+  maxAudioResidualRmsDb?: number;
   renderConfig: {
     /**
      * Frame rate. Stored on disk as a JSON number (integer fps, e.g. `30`)
@@ -140,6 +153,15 @@ type TestResult = {
     passed: boolean;
     correlation: number;
     lagWindows: number;
+    /**
+     * Residual Overall RMS (dBFS) of `rendered - snapshot`. Present only
+     * when the fixture opts in via `meta.maxAudioResidualRmsDb`.
+     * `Number.NEGATIVE_INFINITY` ⇒ perfect cancellation. `NaN` ⇒ residual
+     * check could not run (missing ffmpeg, duration mismatch, ...); see
+     * `audio.residualError` for the reason.
+     */
+    residualRmsDb?: number;
+    residualError?: string;
   };
   renderedOutputPath?: string;
 };
@@ -153,6 +175,28 @@ function logPretty(message: string, emoji = "•") {
   console.error(`${emoji} ${message}`);
 }
 
+/**
+ * Format the residual-RMS suffix used in the audio-quality log line.
+ *
+ * Three states must surface distinctly:
+ *   • `null`            → fixture didn't opt into residual RMS         → "" (no suffix)
+ *   • `NaN`             → check ran but produced no parseable reading  → "(error: ...)"
+ *   • `-Infinity`       → perfect cancellation (identical streams)     → "-inf dBFS"
+ *   • finite number     → measured residual                            → "<value> dBFS"
+ *
+ * Pre-fix this branched on `Number.isFinite()` only, collapsing NaN
+ * (a real-failure signal) into the `-inf` label (a perfect-match signal).
+ */
+function formatResidualSuffix(residualRmsDb: number | null, error: string | undefined): string {
+  if (residualRmsDb === null && !error) return "";
+  if (error) return `, residualRMS: error (${error})`;
+  if (residualRmsDb === null || Number.isNaN(residualRmsDb)) {
+    return ", residualRMS: error (no parseable reading)";
+  }
+  if (!Number.isFinite(residualRmsDb)) return ", residualRMS: -inf dBFS";
+  return `, residualRMS: ${residualRmsDb.toFixed(2)} dBFS`;
+}
+
 function parseArgs(argv: string[]): CliOptions {
   const testNames: string[] = [];
   const excludeTags: string[] = [];
@@ -229,6 +273,12 @@ function validateMetadata(meta: unknown): TestMetadata {
   if (typeof m.maxAudioLagWindows !== "number" || m.maxAudioLagWindows < 1) {
     throw new Error("meta.json: 'maxAudioLagWindows' must be >= 1");
   }
+  if (
+    m.maxAudioResidualRmsDb !== undefined &&
+    (typeof m.maxAudioResidualRmsDb !== "number" || !Number.isFinite(m.maxAudioResidualRmsDb))
+  ) {
+    throw new Error("meta.json: 'maxAudioResidualRmsDb' must be a finite number when present");
+  }
   if (!m.renderConfig || typeof m.renderConfig !== "object") {
     throw new Error("meta.json: 'renderConfig' must be an object");
   }
@@ -671,16 +721,29 @@ function saveFailureDetails(
 
   // Save audio failures
   if (result.audio && !result.audio.passed) {
+    const residualRmsDb = result.audio.residualRmsDb;
+    const residualError = result.audio.residualError;
+    const residualThreshold = suite.meta.maxAudioResidualRmsDb;
+    const residualExceeds =
+      residualThreshold !== undefined &&
+      typeof residualRmsDb === "number" &&
+      Number.isFinite(residualRmsDb) &&
+      residualRmsDb > residualThreshold;
     const audioReport = {
       summary: {
         correlation: result.audio.correlation,
         lagWindows: result.audio.lagWindows,
         threshold: suite.meta.minAudioCorrelation,
         maxLagWindows: suite.meta.maxAudioLagWindows,
+        ...(residualRmsDb !== undefined ? { residualRmsDb } : {}),
+        ...(residualThreshold !== undefined ? { residualThreshold } : {}),
+        ...(residualError ? { residualError } : {}),
       },
       analysis: {
         correlationBelowThreshold: result.audio.correlation < suite.meta.minAudioCorrelation,
         lagExceedsLimit: Math.abs(result.audio.lagWindows) > suite.meta.maxAudioLagWindows,
+        residualExceedsThreshold: residualExceeds,
+        residualCheckFailed: residualError !== undefined,
       },
     };
 
@@ -1051,6 +1114,8 @@ async function runTestSuite(
     let audioPassed = true;
     let audioCorrelation = 1;
     let audioLagWindows = 0;
+    let audioResidualRmsDb: number | null = null;
+    let audioResidualError: string | undefined;
 
     if (!isPngSequence) {
       logPretty("Comparing audio quality...", "🔊");
@@ -1068,13 +1133,35 @@ async function runTestSuite(
         audioCorrelation = audio.correlation;
         audioLagWindows = audio.lagWindows;
         audioPassed = audio.correlation >= suite.meta.minAudioCorrelation;
+
+        // Sample-level residual-RMS check (complementary to the
+        // envelope-correlation gate above). Only runs when the fixture
+        // opts in via `maxAudioResidualRmsDb`; the correlation gate
+        // stays in place either way for legacy fixtures. Correlation
+        // measures shape similarity at envelope granularity; residual
+        // RMS measures sample-level cancellation — both surface
+        // different drift classes.
+        if (suite.meta.maxAudioResidualRmsDb !== undefined) {
+          const residual = computeAudioResidualRmsDb(
+            renderedOutputPath,
+            snapshotVideoPath,
+            suite.meta.maxAudioResidualRmsDb,
+          );
+          audioResidualRmsDb = residual.overallDb;
+          audioResidualError = residual.error;
+          if (!residual.ok) {
+            audioPassed = false;
+          }
+        }
       }
     }
 
     result.audio = {
       passed: audioPassed,
       correlation: audioCorrelation,
       lagWindows: audioLagWindows,
+      ...(audioResidualRmsDb !== null ? { residualRmsDb: audioResidualRmsDb } : {}),
+      ...(audioResidualError ? { residualError: audioResidualError } : {}),
     };
 
     console.log(
@@ -1084,17 +1171,20 @@ async function runTestSuite(
         passed: audioPassed,
         correlation: audioCorrelation,
         lagWindows: audioLagWindows,
+        residualRmsDb: audioResidualRmsDb,
+        residualError: audioResidualError,
       }),
     );
 
+    const residualSuffix = formatResidualSuffix(audioResidualRmsDb, audioResidualError);
     if (audioPassed) {
       logPretty(
-        `Audio quality: PASSED (correlation: ${audioCorrelation.toFixed(3)}, lag: ${audioLagWindows})`,
+        `Audio quality: PASSED (correlation: ${audioCorrelation.toFixed(3)}, lag: ${audioLagWindows}${residualSuffix})`,
         "✓",
       );
     } else {
       logPretty(
-        `Audio quality: FAILED (correlation: ${audioCorrelation.toFixed(3)}, threshold: ${suite.meta.minAudioCorrelation})`,
+        `Audio quality: FAILED (correlation: ${audioCorrelation.toFixed(3)}, threshold: ${suite.meta.minAudioCorrelation}${residualSuffix})`,
         "✗",
       );
     }

packages/producer/src/utils/audioRegression.test.ts

@@ -1,5 +1,13 @@
-import { describe, expect, it } from "vitest";
-import { buildRmsEnvelope, compareAudioEnvelopes } from "./audioRegression.js";
+import { spawnSync } from "node:child_process";
+import { mkdtempSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { afterAll, beforeAll, describe, expect, it } from "vitest";
+import {
+  buildRmsEnvelope,
+  compareAudioEnvelopes,
+  computeAudioResidualRmsDb,
+} from "./audioRegression.js";
 
 describe("compareAudioEnvelopes", () => {
   it("treats silent-vs-silent audio as a perfect match", () => {
@@ -14,3 +22,83 @@ describe("compareAudioEnvelopes", () => {
     });
   });
 });
+
+// Skip the spawn-based tests entirely on hosts without ffmpeg. The
+// regression harness only runs in environments where ffmpeg is present
+// (`Dockerfile.test`, dev boxes with apt's ffmpeg), so an absent ffmpeg
+// is a developer-laptop fact, not a producer regression.
+const HAS_FFMPEG = spawnSync("ffmpeg", ["-version"], { encoding: "utf-8" }).status === 0;
+
+describe.skipIf(!HAS_FFMPEG)("computeAudioResidualRmsDb", () => {
+  let tmp: string;
+
+  beforeAll(() => {
+    tmp = mkdtempSync(join(tmpdir(), "hf-audio-residual-test-"));
+    // Two test wavs: identical 1-second 440 Hz sine, and a 880 Hz sine
+    // that's audibly different from the 440 reference.
+    for (const [name, freq] of [
+      ["sine-440-a.wav", 440],
+      ["sine-440-b.wav", 440],
+      ["sine-880.wav", 880],
+    ] as const) {
+      const result = spawnSync(
+        "ffmpeg",
+        [
+          "-nostdin",
+          "-v",
+          "error",
+          "-f",
+          "lavfi",
+          "-i",
+          `sine=frequency=${freq}:duration=1:sample_rate=48000`,
+          "-ac",
+          "2",
+          "-c:a",
+          "pcm_s16le",
+          join(tmp, name),
+        ],
+        { encoding: "utf-8" },
+      );
+      if (result.status !== 0) {
+        throw new Error(`ffmpeg setup failed for ${name}: ${result.stderr}`);
+      }
+    }
+  });
+
+  afterAll(() => {
+    rmSync(tmp, { recursive: true, force: true });
+  });
+
+  it("returns -inf (or very low dBFS) for two identical streams", () => {
+    const result = computeAudioResidualRmsDb(
+      join(tmp, "sine-440-a.wav"),
+      join(tmp, "sine-440-b.wav"),
+    );
+    expect(result.ok).toBe(true);
+    // 440-vs-440 PCM cancels to silence; ffmpeg reports -inf which we
+    // normalize to NEGATIVE_INFINITY, OR a value well below -90 if the
+    // resampler introduces sub-bit-quantization noise.
+    expect(result.overallDb).toBeLessThan(-80);
+  });
+
+  it("fails when streams are audibly different (440 Hz vs 880 Hz)", () => {
+    const result = computeAudioResidualRmsDb(
+      join(tmp, "sine-440-a.wav"),
+      join(tmp, "sine-880.wav"),
+    );
+    expect(result.ok).toBe(false);
+    // The residual of two uncorrelated unit-amplitude sines is roughly
+    // the sum of both signals at near-full level — typically around
+    // -3 dBFS in this resampled-stereo configuration.
+    expect(result.overallDb).toBeGreaterThan(-30);
+  });
+
+  it("reports ok=false when an input has no audio stream", () => {
+    // A bare empty file: ffmpeg can't probe it, so the function reports
+    // a parse failure (ok=false, NaN). Callers decide whether to treat
+    // that as a pass (no-audio fixture) or a fail (audio expected).
+    const result = computeAudioResidualRmsDb("/dev/null", join(tmp, "sine-440-a.wav"));
+    expect(result.ok).toBe(false);
+    expect(Number.isNaN(result.overallDb)).toBe(true);
+  });
+});