Merge pull request #256 from esokullu/main

bugfix with video record with no sound

Author: esokullu

package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "webbrain",
-  "version": "20.1.1",
+  "version": "20.1.3",
   "description": "Open-source AI browser agent — chat with pages, automate tasks, multi-provider LLM support.",
   "private": true,
   "type": "module",

package.json

@@ -1,6 +1,6 @@
 # WebBrain Chrome/Edge Extension — Architecture
 
-> Version 20.1.1 · Manifest V3 · Service Worker background
+> Version 20.1.3 · Manifest V3 · Service Worker background
 
 ## High-Level Overview
 
@@ -192,8 +192,8 @@ offscreen/recorder.js
       │     captured audio ─→ mixDestination
       │     mic            ─→ mixDestination
       │     tab audio only ─→ audioContext.destination   (passthrough so user hears the call)
-      ├─ MediaStream(video tracks ∪ mixDestination audio tracks)
-      └─ MediaRecorder(mimeType: 'video/webm;codecs=vp9,opus')
+      ├─ MediaStream(video tracks ∪ mixDestination audio tracks when audio exists)
+      └─ MediaRecorder(mimeType selected from actual final video/audio tracks)
               └─ ondataavailable → chunks[]
                                   → on stop, Blob → dataURL → background
 

src/chrome/ARCHITECTURE.md

@@ -1,7 +1,7 @@
 {
   "manifest_version": 3,
   "name": "WebBrain",
-  "version": "20.1.1",
+  "version": "20.1.3",
   "description": "Open-source AI browser agent — chat with pages, automate tasks, multi-provider LLM support.",
   "permissions": [
     "sidePanel",

src/chrome/manifest.json

@@ -53,6 +53,32 @@
     });
   }
 
+  function pickRecorderMimeType({ hasVideo, hasAudio, requestedMime }) {
+    const candidates = hasVideo
+      ? (hasAudio
+          ? [
+              'video/webm;codecs=vp9,opus',
+              'video/webm;codecs=vp8,opus',
+              'video/webm',
+            ]
+          : [
+              'video/webm;codecs=vp9',
+              'video/webm;codecs=vp8',
+              'video/webm',
+            ])
+      : [
+          'audio/webm;codecs=opus',
+          'audio/webm',
+        ];
+    const chosenMime = (requestedMime && MediaRecorder.isTypeSupported(requestedMime))
+      ? requestedMime
+      : candidates.find(m => MediaRecorder.isTypeSupported(m));
+    if (!chosenMime) {
+      throw new Error('No supported MediaRecorder mimeType found.');
+    }
+    return chosenMime;
+  }
+
   async function start(message) {
     let { streamId, tabId, options } = message || {};
     if (session) {
@@ -74,26 +100,6 @@
       mimeType: requestedMime,
     } = options || {};
 
-    // Pick a MediaRecorder mimeType the browser actually supports. VP9 is
-    // best quality-per-byte; VP8 is the wide-compat fallback. Audio-only
-    // gets webm/opus.
-    const candidates = video
-      ? [
-          'video/webm;codecs=vp9,opus',
-          'video/webm;codecs=vp8,opus',
-          'video/webm',
-        ]
-      : [
-          'audio/webm;codecs=opus',
-          'audio/webm',
-        ];
-    const chosenMime = (requestedMime && MediaRecorder.isTypeSupported(requestedMime))
-      ? requestedMime
-      : candidates.find(m => MediaRecorder.isTypeSupported(m));
-    if (!chosenMime) {
-      throw new Error('No supported MediaRecorder mimeType found.');
-    }
-
     // 1. Capture stream — chrome.tabCapture exposes the active tab as a
     // MediaStream when we pass the streamId we got from
     // chrome.tabCapture.getMediaStreamId() on the service-worker side. For
@@ -172,41 +178,63 @@
     // them. Release everything we acquired before rethrowing.
     let audioContext = null;
     let recorder;
+    let chosenMime;
     try {
-      audioContext = new AudioContext();
-      const mixDest = audioContext.createMediaStreamDestination();
-
       const capturedAudioTracks = captureStream.getAudioTracks();
-      if (capturedAudioTracks.length) {
-        const capturedAudioSource = audioContext.createMediaStreamSource(
-          new MediaStream(capturedAudioTracks)
-        );
-        capturedAudioSource.connect(mixDest); // into the recording
-        if (source === 'tab') {
-          capturedAudioSource.connect(audioContext.destination);
+      const hasAudioSources = capturedAudioTracks.length > 0 || !!micStream;
+      let mixDest = null;
+      if (hasAudioSources) {
+        audioContext = new AudioContext();
+        mixDest = audioContext.createMediaStreamDestination();
+
+        if (capturedAudioTracks.length) {
+          const capturedAudioSource = audioContext.createMediaStreamSource(
+            new MediaStream(capturedAudioTracks)
+          );
+          capturedAudioSource.connect(mixDest); // into the recording
+          if (source === 'tab') {
+            capturedAudioSource.connect(audioContext.destination);
+          }
         }
-      }
 
-      if (micStream) {
-        const micSource = audioContext.createMediaStreamSource(micStream);
-        micSource.connect(mixDest); // into the recording (do NOT loop to speaker — feedback)
+        if (micStream) {
+          const micSource = audioContext.createMediaStreamSource(micStream);
+          micSource.connect(mixDest); // into the recording (do NOT loop to speaker — feedback)
+        }
       }
 
       // 4. Build the final stream the recorder consumes.
       const finalStream = new MediaStream();
       if (video) {
         for (const t of captureStream.getVideoTracks()) finalStream.addTrack(t);
       }
-      for (const t of mixDest.stream.getAudioTracks()) finalStream.addTrack(t);
+      if (mixDest) {
+        for (const t of mixDest.stream.getAudioTracks()) finalStream.addTrack(t);
+      }
+      const finalHasVideo = finalStream.getVideoTracks().length > 0;
+      const finalHasAudio = finalStream.getAudioTracks().length > 0;
+      if (!finalHasVideo && !finalHasAudio) {
+        throw new Error('No media tracks available to record.');
+      }
+      chosenMime = pickRecorderMimeType({
+        hasVideo: finalHasVideo,
+        hasAudio: finalHasAudio,
+        requestedMime,
+      });
 
       // 5. MediaRecorder. Collect dataavailable chunks. Pass a timeslice so
       // partial data survives a crash and gives us progress estimates.
-      recorder = new MediaRecorder(finalStream, {
+      const recorderOptions = {
         mimeType: chosenMime,
-        // ~256 kbps audio is plenty for speech; the rest is video budget.
-        audioBitsPerSecond: 192_000,
-        videoBitsPerSecond: 2_500_000,
-      });
+      };
+      if (finalHasAudio) {
+        // ~192 kbps audio is plenty for speech; the rest is video budget.
+        recorderOptions.audioBitsPerSecond = 192_000;
+      }
+      if (finalHasVideo) {
+        recorderOptions.videoBitsPerSecond = 2_500_000;
+      }
+      recorder = new MediaRecorder(finalStream, recorderOptions);
     } catch (e) {
       try { for (const t of captureStream.getTracks()) t.stop(); } catch {}
       if (micStream) { try { for (const t of micStream.getTracks()) t.stop(); } catch {} }
@@ -215,7 +243,9 @@
     }
     const chunks = [];
     let bytes = 0;
+    let dataEventCount = 0;
     recorder.ondataavailable = (e) => {
+      dataEventCount += 1;
       if (e.data && e.data.size > 0) {
         chunks.push(e.data);
         bytes += e.data.size;
@@ -241,6 +271,7 @@
       stopPromise: null,
       stopEventObserved: false,
       get bytes() { return bytes; },
+      get dataEventCount() { return dataEventCount; },
     };
     const activeSession = session;
     recorder.addEventListener('stop', () => {
@@ -339,30 +370,59 @@
   function waitForRecorderStop(s) {
     if (!s?.recorder) return Promise.resolve();
     if (s.stopPromise) return s.stopPromise;
-    if (s.recorder.state === 'inactive' && s.stopEventObserved) return Promise.resolve();
     s.stopPromise = new Promise((resolve) => {
       let done = false;
       let timeout = null;
+      let stopped = s.recorder.state === 'inactive';
+      let finalDataSettled = false;
+      const initialBytes = Number(s.bytes || 0);
       const finish = () => {
         if (done) return;
         done = true;
         s.stopEventObserved = true;
         if (timeout) clearTimeout(timeout);
-        try { s.recorder.removeEventListener('stop', finish); } catch {}
+        try { s.recorder.removeEventListener('stop', onStop); } catch {}
+        try { s.recorder.removeEventListener('dataavailable', onFinalDataAvailable); } catch {}
         resolve();
       };
-      try { s.recorder.addEventListener('stop', finish); } catch {}
+      const maybeFinish = () => {
+        if (stopped && finalDataSettled) finish();
+      };
+      const onStop = () => {
+        stopped = true;
+        maybeFinish();
+      };
+      const onFinalDataAvailable = (e) => {
+        if (e?.data && e.data.size > 0) {
+          finalDataSettled = true;
+        } else if (initialBytes > 0 || Number(s.bytes || 0) > 0) {
+          // A final dataavailable event can be empty when prior timeslices
+          // already flushed usable bytes. Treat that as settled, but never
+          // accept a zero-byte recording until the timeout fallback fires.
+          finalDataSettled = true;
+        }
+        maybeFinish();
+      };
+      try { s.recorder.addEventListener('stop', onStop); } catch {}
+      try { s.recorder.addEventListener('dataavailable', onFinalDataAvailable); } catch {}
       try { s.recorder.requestData(); } catch {}
       timeout = setTimeout(() => {
-        log('timed out waiting for MediaRecorder stop event; finalizing with collected chunks');
+        log('timed out waiting for MediaRecorder final data; finalizing with collected chunks', {
+          bytes: s.bytes,
+          dataEvents: s.dataEventCount,
+          stopped,
+        });
+        stopped = true;
+        finalDataSettled = true;
         finish();
-      }, 5000);
+      }, 2000);
       try {
-        if (s.recorder.state === 'inactive') finish();
+        if (s.recorder.state === 'inactive') stopped = true;
         else s.recorder.stop();
       } catch {
-        finish();
+        stopped = true;
       }
+      maybeFinish();
     });
     return s.stopPromise;
   }

src/chrome/src/offscreen/recorder.js

@@ -19,7 +19,7 @@ import {
 
 // Version shown in the subtitle. Kept here so it only needs one update per
 // release; the subtitle string itself is translated.
-const EXT_VERSION = '20.1.1';
+const EXT_VERSION = '20.1.3';
 
 const providersContainer = document.getElementById('providers');
 const displaySettings = document.getElementById('display-settings');

src/chrome/src/ui/settings.js

@@ -1,6 +1,6 @@
 # WebBrain Firefox Extension — Architecture
 
-> Version 20.1.1 · Manifest V2 · Background Page
+> Version 20.1.3 · Manifest V2 · Background Page
 
 ## How Firefox Differs from Chrome
 

src/firefox/ARCHITECTURE.md

@@ -1,7 +1,7 @@
 {
   "manifest_version": 2,
   "name": "WebBrain",
-  "version": "20.1.1",
+  "version": "20.1.3",
   "description": "Open-source AI browser agent — chat with pages, automate tasks, multi-provider LLM support.",
   "permissions": [
     "activeTab",

src/firefox/manifest.json

@@ -19,7 +19,7 @@ import {
 
 // Version shown in the subtitle. Kept here so it only needs one update per
 // release; the subtitle string itself is translated.
-const EXT_VERSION = '20.1.1';
+const EXT_VERSION = '20.1.3';
 
 const providersContainer = document.getElementById('providers');
 const displaySettings = document.getElementById('display-settings');

src/firefox/src/ui/settings.js

@@ -4857,11 +4857,17 @@ test('chrome /record-full-screen is slash-only, Chrome-only, and hidden from the
   assert.match(host, /export async function startDisplayRecording\(options = \{\}/, 'chrome: recorder host should expose display recording for slash command routing');
   assert.match(host, /function scheduleRecordingSafetyWatchdog[\s\S]*?chrome\.alarms\?\.create/, 'chrome: hidden recordings should have a background-owned safety cap');
   assert.match(offscreen, /navigator\.mediaDevices\.getDisplayMedia\(\{[\s\S]*?audio: audio !== false,[\s\S]*?video: true,[\s\S]*?\}\)/, 'chrome: offscreen recorder should open the screen/window picker through getDisplayMedia');
+  assert.match(offscreen, /function pickRecorderMimeType\(\{ hasVideo, hasAudio, requestedMime \}\)[\s\S]*?'video\/webm;codecs=vp9,opus'[\s\S]*?'video\/webm;codecs=vp9'[\s\S]*?'audio\/webm;codecs=opus'/, 'chrome: recorder should choose MIME after it knows whether the final stream has audio');
+  assert.match(offscreen, /const hasAudioSources = capturedAudioTracks\.length > 0 \|\| !!micStream;[\s\S]*?if \(hasAudioSources\) \{[\s\S]*?mixDest = audioContext\.createMediaStreamDestination\(\);[\s\S]*?if \(mixDest\) \{[\s\S]*?mixDest\.stream\.getAudioTracks\(\)/, 'chrome: display video-only recordings should not add an empty WebAudio track');
+  assert.match(offscreen, /if \(finalHasAudio\) \{[\s\S]*?recorderOptions\.audioBitsPerSecond = 192_000;[\s\S]*?\}[\s\S]*?if \(finalHasVideo\) \{[\s\S]*?recorderOptions\.videoBitsPerSecond = 2_500_000;/, 'chrome: recorder bitrate hints should match the actual final stream tracks');
   assert.doesNotMatch(offscreen, /chrome\.desktopCapture/, 'chrome: offscreen recorder cannot use extension APIs beyond chrome.runtime');
   assert.match(offscreen, /addEventListener\('ended', \(\) => \{[\s\S]*?s\.captureEndedCleanupStarted = true;[\s\S]*?finalizeCaptureEnded\(s\)\.catch/, 'chrome: capture-ended cleanup should use the async finalize path');
   assert.match(offscreen, /async function finalizeCaptureEnded\(s\) \{[\s\S]*?s\.stopping = true;[\s\S]*?await waitForRecorderStop\(s\);[\s\S]*?notifyCaptureEnded\(s\);[\s\S]*?await releaseSession\(s\);[\s\S]*?\}/, 'chrome: capture-ended finalize should wait for final recorder data before notifying background');
   assert.match(offscreen, /function waitForRecorderStop\(s\) \{[\s\S]*?if \(s\.stopPromise\) return s\.stopPromise;[\s\S]*?s\.stopPromise = new Promise/, 'chrome: recorder stop waits should share one in-progress stop promise');
-  assert.doesNotMatch(offscreen, /s\.recorder\.state === 'inactive'\) return Promise\.resolve/, 'chrome: inactive recorder state alone must not skip waiting for queued stop/dataavailable events');
+  assert.match(offscreen, /let dataEventCount = 0;[\s\S]*?recorder\.ondataavailable = \(e\) => \{[\s\S]*?dataEventCount \+= 1;[\s\S]*?get dataEventCount\(\) \{ return dataEventCount; \}/, 'chrome: recorder should track dataavailable events for finalization');
+  assert.match(offscreen, /const onFinalDataAvailable = \(e\) => \{[\s\S]*?finalDataSettled = true;[\s\S]*?maybeFinish\(\);[\s\S]*?\};[\s\S]*?s\.recorder\.addEventListener\('dataavailable', onFinalDataAvailable\)/, 'chrome: recorder stop must wait for final dataavailable before resolving');
+  assert.match(offscreen, /timed out waiting for MediaRecorder final data[\s\S]*?\}, 2000\);/, 'chrome: final data fallback should stay short enough for visible stop UI');
+  assert.doesNotMatch(offscreen, /s\.recorder\.state === 'inactive'\) (?:return Promise\.resolve|finish\(\))/, 'chrome: inactive recorder state alone must not skip waiting for queued stop/dataavailable events');
   assert.match(offscreen, /function notifyCaptureEnded\(s\) \{[\s\S]*?target: 'background'[\s\S]*?action: 'recording_capture_ended'[\s\S]*?tabId: s\.tabId/, 'chrome: capture-ended notify should ask background to persist the stopped recording');
   assert.match(background, /case 'recording_capture_ended':[\s\S]*?stopTabRecording\(\{ reason: 'capture_ended' \}\)/, 'chrome: background should finalize and save recordings when the shared stream ends');
   assert.match(offscreen, /async function releaseSession\(s\) \{[\s\S]*?s\.captureStream = null;[\s\S]*?s\.micStream = null;[\s\S]*?s\.audioContext = null;[\s\S]*?micStream\?\.getTracks/, 'chrome: releaseSession should be idempotent and stop the separately-acquired mic stream');