feat: add threshold/onset-triggered recording with configurable options

DavidMenting · aider-chat-bot · DavidMenting · commit 801de42c3fb7 · 2025-10-09T17:31:04.000+02:00
Co-authored-by: aider (openrouter/openai/gpt-5) &lt;aider@aider.chat&gt;
diff --git a/src/lib/components/Voice.svelte b/src/lib/components/Voice.svelte
@@ -18,7 +18,7 @@
   let uploadError = $state<string | null>(null);
 
   // Recording state
-  let recordingStatus = $state<'idle' | 'requesting-permission' | 'recording' | 'processing' | 'error'>('idle');
+  let recordingStatus = $state<'idle' | 'requesting-permission' | 'waiting' | 'recording' | 'processing' | 'error'>('idle');
   let recordingProgress = $state<number>(0);
   let recordingError = $state<string | null>(null);
 
@@ -266,6 +266,9 @@
           if (progress.stage === 'requesting') {
             recordingStatus = 'requesting-permission';
             recordingProgress = 0;
+          } else if (progress.stage === 'waiting') {
+            recordingStatus = 'waiting';
+            recordingProgress = 0;
           } else if (progress.stage === 'recording') {
             recordingStatus = 'recording';
             recordingProgress = progress.percentage;
@@ -337,10 +340,12 @@
     <img src={imageSrc} alt="Voice" class="w-20 h-20" />
 
     <!-- Recording progress indicator -->
-    {#if recordingStatus === 'recording' || recordingStatus === 'processing'}
+    {#if recordingStatus === 'waiting' || recordingStatus === 'recording' || recordingStatus === 'processing'}
       <div class="absolute inset-0 flex items-center justify-center bg-yellow-500 bg-opacity-70 rounded-lg">
         <div class="text-white text-xs font-bold">
-          {#if recordingStatus === 'recording'}
+          {#if recordingStatus === 'waiting'}
+            🎧 Waiting...
+          {:else if recordingStatus === 'recording'}
             🎤 {Math.round(recordingProgress)}%
           {:else}
             ⚙️ {Math.round(recordingProgress)}%
diff --git a/src/lib/services/audioRecorder.ts b/src/lib/services/audioRecorder.ts
@@ -14,6 +14,7 @@
 
 import { createLogger } from '$lib/utils/logger';
 import type { ProcessedAudio } from './audioProcessor';
+import { captureOnset } from './onsetDetector';
 
 const logger = createLogger('AudioRecorder');
 
@@ -29,17 +30,43 @@ export interface RecordingOptions {
   deviceId?: string;
 
   /**
-   * Optional threshold level (0-1) for automatic recording trigger.
-   * Not implemented yet, reserved for future use.
+   * Optional threshold level (0-1) for automatic onset-triggered recording.
+   * When provided, recording will arm and wait until the level crosses this threshold,
+   * then capture exactly 1 second including optional pre-roll.
    */
   threshold?: number;
+
+  /**
+   * Optional pre-roll time in milliseconds included before the trigger point.
+   * Defaults to 120ms. The post-trigger capture length is reduced accordingly
+   * so the total captured length remains 1 second.
+   */
+  preRollMs?: number;
+
+  /**
+   * Minimum time in milliseconds the signal must stay above threshold to trigger.
+   * Defaults to 12ms.
+   */
+  holdMs?: number;
+
+  /**
+   * Timeout in milliseconds while waiting for a trigger before aborting.
+   * Defaults to 10000ms.
+   */
+  timeoutMs?: number;
+
+  /**
+   * Optional high-pass filter cutoff in Hz to reduce low-frequency rumble.
+   * Defaults to 80Hz.
+   */
+  highpassHz?: number;
 }
 
 export interface RecordingProgress {
   /**
-   * Recording stage: 'requesting' | 'recording' | 'processing'
+   * Recording stage: 'requesting' | 'waiting' | 'recording' | 'processing'
    */
-  stage: 'requesting' | 'recording' | 'processing';
+  stage: 'requesting' | 'waiting' | 'recording' | 'processing';
 
   /**
    * Progress percentage (0-100)
@@ -82,25 +109,85 @@ export async function recordAudio(
     stream = await navigator.mediaDevices.getUserMedia(constraints);
     logger.debug('Microphone access granted');
 
-    // Start recording
-    onProgress?.({ stage: 'recording', percentage: 0 });
+    if (typeof options.threshold === 'number') {
+      // Onset-triggered recording path (wait for threshold then capture 1s incl. pre-roll)
+      onProgress?.({ stage: 'waiting', percentage: 0 });
 
-    const audioBlob = await recordForDuration(stream, RECORDING_DURATION_MS, onProgress);
-    logger.debug(`Recorded ${audioBlob.size} bytes`);
+      const preRollMs = Math.max(0, Math.floor(options.preRollMs ?? 120));
+      const captureMs = Math.max(1, 1000 - preRollMs);
 
-    // Stop all tracks
-    stream.getTracks().forEach(track => track.stop());
-    stream = null;
+      const { samples, sampleRate } = await captureOnset(stream, {
+        threshold: options.threshold,
+        preRollMs,
+        holdMs: Math.max(1, Math.floor(options.holdMs ?? 12)),
+        timeoutMs: Math.max(1000, Math.floor(options.timeoutMs ?? 10000)),
+        highpassHz: options.highpassHz ?? 80,
+        captureMs
+      });
+
+      // Stop all tracks
+      stream.getTracks().forEach(track => track.stop());
+      stream = null;
+
+      // Convert to DRUM-compatible format
+      onProgress?.({ stage: 'processing', percentage: 80 });
+
+      let mono = samples;
+
+      if (sampleRate !== TARGET_SAMPLE_RATE) {
+        mono = resample(mono, sampleRate, TARGET_SAMPLE_RATE);
+      }
 
-    // Process the recorded audio
-    onProgress?.({ stage: 'processing', percentage: 90 });
+      // Ensure exactly 1 second (keep leading pre-roll)
+      if (mono.length > MAX_SAMPLES) {
+        mono = mono.slice(0, MAX_SAMPLES);
+      } else if (mono.length < MAX_SAMPLES) {
+        const padded = new Float32Array(MAX_SAMPLES);
+        padded.set(mono, 0);
+        mono = padded;
+      }
+
+      // Convert to 16-bit PCM
+      const pcmBuffer = new ArrayBuffer(mono.length * 2);
+      const pcmView = new DataView(pcmBuffer);
+      for (let i = 0; i < mono.length; i++) {
+        const sample = Math.max(-1, Math.min(1, mono[i]));
+        const intSample = Math.round(sample * 32767);
+        pcmView.setInt16(i * 2, intSample, true);
+      }
+
+      const processedAudio: ProcessedAudio = {
+        pcmData: new Uint8Array(pcmBuffer),
+        sampleRate: TARGET_SAMPLE_RATE,
+        duration: mono.length / TARGET_SAMPLE_RATE,
+        originalFileName: `recording-${Date.now()}.wav`
+      };
+
+      logger.info('Recording completed successfully');
+      onProgress?.({ stage: 'processing', percentage: 100 });
+
+      return processedAudio;
+    } else {
+      // Start fixed-duration recording path
+      onProgress?.({ stage: 'recording', percentage: 0 });
 
-    const processedAudio = await processRecording(audioBlob);
-    logger.info('Recording completed successfully');
+      const audioBlob = await recordForDuration(stream, RECORDING_DURATION_MS, onProgress);
+      logger.debug(`Recorded ${audioBlob.size} bytes`);
 
-    onProgress?.({ stage: 'processing', percentage: 100 });
+      // Stop all tracks
+      stream.getTracks().forEach(track => track.stop());
+      stream = null;
+
+      // Process the recorded audio
+      onProgress?.({ stage: 'processing', percentage: 90 });
+
+      const processedAudio = await processRecording(audioBlob);
+      logger.info('Recording completed successfully');
 
-    return processedAudio;
+      onProgress?.({ stage: 'processing', percentage: 100 });
+
+      return processedAudio;
+    }
 
   } catch (error) {
     // Cleanup on error
diff --git a/src/lib/services/onsetDetector.ts b/src/lib/services/onsetDetector.ts
@@ -0,0 +1,88 @@
+import { createLogger } from '$lib/utils/logger';
+
+const logger = createLogger('OnsetDetector');
+
+export interface OnsetConfig {
+  threshold: number;
+  preRollMs: number;
+  holdMs: number;
+  timeoutMs: number;
+  highpassHz?: number;
+  captureMs: number; // total capture length AFTER preroll; total output = preRollMs + captureMs
+}
+
+/**
+ * Capture a 1-channel Float32Array when an onset crosses a threshold,
+ * including an optional pre-roll. Uses an AudioWorklet for low-latency detection.
+ */
+export async function captureOnset(
+  stream: MediaStream,
+  cfg: OnsetConfig
+): Promise<{ samples: Float32Array; sampleRate: number }> {
+  const AudioCtx: typeof AudioContext = (window as any).AudioContext || (window as any).webkitAudioContext;
+  const ac = new AudioCtx();
+
+  await ac.audioWorklet.addModule(new URL('./worklets/onset-processor.js', import.meta.url));
+
+  const src = ac.createMediaStreamSource(stream);
+  const node = new AudioWorkletNode(ac, 'onset-detector', { numberOfInputs: 1, numberOfOutputs: 0 });
+
+  src.connect(node);
+
+  node.port.postMessage({
+    threshold: cfg.threshold,
+    preRollMs: cfg.preRollMs,
+    holdMs: cfg.holdMs,
+    captureMs: cfg.captureMs,
+    highpassHz: cfg.highpassHz ?? 80
+  });
+
+  return new Promise((resolve, reject) => {
+    let settled = false;
+
+    const cleanup = () => {
+      try { src.disconnect(); } catch {}
+      try { node.disconnect(); } catch {}
+      node.port.onmessage = null;
+      ac.close().catch(() => {});
+    };
+
+    const timer = window.setTimeout(() => {
+      if (settled) return;
+      settled = true;
+      logger.warn('Onset detection timed out');
+      cleanup();
+      reject(new Error('Onset detection timed out'));
+    }, cfg.timeoutMs);
+
+    node.port.onmessage = (e: MessageEvent) => {
+      const data: any = (e as any).data;
+      if (!data) return;
+
+      if (data.type === 'trigger') {
+        // could emit progress here if needed
+        return;
+      }
+
+      if (data.type === 'data') {
+        if (settled) return;
+        settled = true;
+        window.clearTimeout(timer);
+
+        let samples: Float32Array;
+        if (data.samples instanceof Float32Array) {
+          samples = data.samples as Float32Array;
+        } else if (data.samples instanceof ArrayBuffer) {
+          samples = new Float32Array(data.samples as ArrayBuffer);
+        } else if (data.buffer) {
+          samples = new Float32Array(data.buffer as ArrayBuffer);
+        } else {
+          samples = Float32Array.from(data.samples);
+        }
+
+        cleanup();
+        resolve({ samples, sampleRate: data.sampleRate as number });
+      }
+    };
+  });
+}
diff --git a/src/lib/services/worklets/onset-processor.js b/src/lib/services/worklets/onset-processor.js
@@ -0,0 +1,110 @@
+/* AudioWorkletProcessor: onset-triggered capture with optional pre-roll.
+   Captures exactly preRollMs + captureMs samples after trigger.
+*/
+class DCBlocker {
+  constructor(cutHz, sr) {
+    const twoPi = 2 * Math.PI;
+    this.R = Math.exp(-twoPi * (cutHz / sr));
+    this.x1 = 0;
+    this.y1 = 0;
+  }
+  process(x) {
+    const y = x - this.x1 + this.R * this.y1;
+    this.x1 = x;
+    this.y1 = y;
+    return y;
+  }
+}
+
+registerProcessor('onset-detector', class extends AudioWorkletProcessor {
+  constructor() {
+    super();
+    this.cfg = {
+      threshold: 0.15,
+      preRoll: 0,
+      holdSamples: 1,
+      capLen: Math.round(48000),
+      hpHz: 80
+    };
+    this.dc = new DCBlocker(this.cfg.hpHz, sampleRate);
+
+    this.ring = new Float32Array(this.cfg.preRoll);
+    this.ringIdx = 0;
+
+    this.above = 0;
+    this.capturing = false;
+
+    this.buf = new Float32Array(this.cfg.preRoll + this.cfg.capLen);
+    this.writeIdx = 0;
+
+    this.port.onmessage = (e) => {
+      const c = e.data || {};
+      this.cfg.threshold = typeof c.threshold === 'number' ? c.threshold : this.cfg.threshold;
+      const pre = Math.max(0, Math.round(((c.preRollMs ?? 0) * sampleRate) / 1000));
+      const hold = Math.max(1, Math.round(((c.holdMs ?? 1) * sampleRate) / 1000));
+      const cap = Math.max(1, Math.round(((c.captureMs ?? 1000) * sampleRate) / 1000));
+      const hp = typeof c.highpassHz === 'number' ? c.highpassHz : this.cfg.hpHz;
+
+      this.cfg.preRoll = pre;
+      this.cfg.holdSamples = hold;
+      this.cfg.capLen = cap;
+      this.cfg.hpHz = hp;
+
+      this.dc = new DCBlocker(this.cfg.hpHz, sampleRate);
+      this.ring = new Float32Array(this.cfg.preRoll);
+      this.ringIdx = 0;
+      this.above = 0;
+      this.capturing = false;
+      this.buf = new Float32Array(this.cfg.preRoll + this.cfg.capLen);
+      this.writeIdx = 0;
+    };
+  }
+
+  process(inputs) {
+    const ch = inputs[0] && inputs[0][0];
+    if (!ch) return true;
+
+    for (let i = 0; i < ch.length; i++) {
+      const s = this.dc.process(ch[i]);
+
+      if (this.ring.length > 0) {
+        this.ring[this.ringIdx] = s;
+        this.ringIdx = (this.ringIdx + 1) % this.ring.length;
+      }
+
+      if (!this.capturing) {
+        if (Math.abs(s) >= this.cfg.threshold) {
+          this.above++;
+          if (this.above >= this.cfg.holdSamples) {
+            // Trigger: copy preroll into buffer oldest->newest
+            const pre = this.ring.length;
+            if (pre > 0) {
+              for (let k = 0; k < pre; k++) {
+                const idx = (this.ringIdx + k) % pre;
+                this.buf[k] = this.ring[idx];
+              }
+            }
+            this.writeIdx = pre;
+            this.capturing = true;
+            this.port.postMessage({ type: 'trigger' });
+          }
+        } else {
+          this.above = 0;
+        }
+      }
+
+      if (this.capturing) {
+        if (this.writeIdx < this.buf.length) {
+          this.buf[this.writeIdx++] = s;
+          if (this.writeIdx >= this.buf.length) {
+            // Done: transfer the buffer
+            this.port.postMessage({ type: 'data', sampleRate, samples: this.buf }, [this.buf.buffer]);
+            return false; // stop processor
+          }
+        }
+      }
+    }
+
+    return true;
+  }
+});
