Merge pull request #286 from dozro/fix/voice-message-mess

Fix audio codec handling and enhance voice recorder functionality

Author: Just-Insane

.changeset/fix_voice_message_element_compat.md

@@ -0,0 +1,5 @@
+---
+default: patch
+---
+
+fix of compatibility of voice messages with element clients and style misshaps

src/app/features/room/AudioMessageRecorder.tsx

@@ -8,6 +8,7 @@ type AudioMessageRecorderProps = {
   onRequestClose: () => void;
   onWaveformUpdate: (waveform: number[]) => void;
   onAudioLengthUpdate: (length: number) => void;
+  onAudioCodecUpdate?: (codec: string) => void;
 };
 
 // We use a react voice recorder library to handle the recording of audio messages, as it provides a simple API and handles the complexities of recording audio in the browser.
@@ -19,6 +20,7 @@ export function AudioMessageRecorder({
   onRequestClose,
   onWaveformUpdate,
   onAudioLengthUpdate,
+  onAudioCodecUpdate,
 }: AudioMessageRecorderProps) {
   const containerRef = useRef<HTMLDivElement>(null);
   const isDismissedRef = useRef(false);
@@ -50,7 +52,7 @@ export function AudioMessageRecorder({
             borderRadius: config.radii.R400,
             boxShadow: config.shadow.E200,
             padding: config.space.S400,
-            minWidth: 300,
+            width: 300,
           }}
         >
           <Text size="H4">Audio Message Recorder</Text>
@@ -60,16 +62,20 @@ export function AudioMessageRecorder({
               audioFile,
               waveform,
               audioLength,
+              audioCodec,
             }: {
               audioFile: Blob;
               waveform: number[];
               audioLength: number;
+              audioCodec: string;
             }) => {
               if (isDismissedRef.current) return;
               // closes the recorder and sends the audio file back to the parent component to be uploaded and sent as a message
               onRecordingComplete(audioFile);
               onWaveformUpdate(waveform);
               onAudioLengthUpdate(audioLength);
+              // Pass the audio codec to the parent component
+              if (onAudioCodecUpdate) onAudioCodecUpdate(audioCodec);
             }}
             buttonBackgroundColor={color.SurfaceVariant.Container}
             buttonHoverBackgroundColor={color.SurfaceVariant.ContainerHover}

src/app/features/room/RoomInput.tsx

@@ -151,6 +151,7 @@ import { usePowerLevelsContext } from '$hooks/usePowerLevels';
 import { useRoomCreators } from '$hooks/useRoomCreators';
 import { useRoomPermissions } from '$hooks/useRoomPermissions';
 import { AutocompleteNotice } from '$components/editor/autocomplete/AutocompleteNotice';
+import { getSupportedAudioExtension } from '$plugins/voice-recorder-kit/supportedCodec';
 import { SchedulePickerDialog } from './schedule-send';
 import * as css from './schedule-send/SchedulePickerDialog.css';
 import {
@@ -1131,7 +1132,7 @@ export const RoomInput = forwardRef<HTMLDivElement, RoomInputProps>(
                       onRecordingComplete={(audioBlob) => {
                         const file = new File(
                           [audioBlob],
-                          `sable-audio-message-${Date.now()}.ogg`,
+                          `sable-audio-message-${Date.now()}.${getSupportedAudioExtension(audioBlob.type)}`,
                           {
                             type: audioBlob.type,
                           }

src/app/features/room/msgContent.ts

@@ -155,28 +155,60 @@ export const getAudioMsgContent = (
   audioLength?: number
 ): AudioMsgContent => {
   const { file, encInfo } = item;
-  const content: IContent = {
+  let content: IContent = {
     msgtype: MsgType.Audio,
     filename: file.name,
-    body: file.name,
+    body: item.body && item.body.length > 0 ? item.body : 'a voice message',
     format: 'org.matrix.custom.html',
-    formatted_body: file.name,
+    formatted_body: item.body && item.body.length > 0 ? item.body : '<em>a voice message</em>',
     info: {
       mimetype: file.type,
       size: file.size,
+      duration: item.metadata.markedAsSpoiler || !audioLength ? 0 : audioLength * 1000,
     },
+
+    // Element-compatible unstable extensible-event keys
     'org.matrix.msc1767.audio': {
       waveform: waveform?.map((v) => Math.round(v * 1024)), // scale waveform values to fit in 10 bits (0-1024) for more efficient storage, as per MSC1767 spec
       duration: item.metadata.markedAsSpoiler || !audioLength ? 0 : audioLength * 1000, // if marked as spoiler, set duration to 0 to hide it in clients that support msc1767
     },
+    'org.matrix.msc1767.text': item.body && item.body.length > 0 ? item.body : 'a voice message',
+    'org.matrix.msc3245.voice.v2': {
+      duration: !audioLength ? 0 : audioLength,
+      waveform: waveform?.map((v) => Math.round(v * 1024)),
+    },
+    // for element compat
+    'org.matrix.msc3245.voice': {},
   };
   if (encInfo) {
     content.file = {
       ...encInfo,
       url: mxc,
     };
+    content = {
+      ...content,
+
+      // Element-compatible unstable extensible-event keys
+      'org.matrix.msc1767.file': {
+        name: file.name,
+        mimetype: file.type,
+        size: file.size,
+        file: content.file,
+      },
+    };
   } else {
     content.url = mxc;
+    content = {
+      ...content,
+
+      // Element-compatible unstable extensible-event keys
+      'org.matrix.msc1767.file': {
+        name: file.name,
+        mimetype: file.type,
+        size: file.size,
+        url: content.url,
+      },
+    };
   }
   if (item.body && item.body.length > 0) content.body = item.body;
   if (item.formatted_body && item.formatted_body.length > 0) {

src/app/plugins/voice-recorder-kit/supportedCodec.ts

@@ -0,0 +1,80 @@
+const safariPreferredCodecs = [
+  // Safari works best with MP4/AAC but fails when strict codecs are defined on iOS.
+  // Prioritize the plain container to avoid NotSupportedError during MediaRecorder initialization.
+  'audio/mp4',
+  'audio/mp4;codecs=mp4a.40.2',
+  'audio/mp4;codecs=mp4a.40.5',
+  'audio/mp4;codecs=aac',
+  'audio/aac',
+  // Fallbacks
+  'audio/wav;codecs=1',
+  'audio/wav',
+  'audio/mpeg',
+];
+
+const defaultPreferredCodecs = [
+  // Chromium / Firefox stable path.
+  'audio/webm;codecs=opus',
+  'audio/webm',
+  // Firefox
+  'audio/ogg;codecs=opus',
+  'audio/ogg;codecs=vorbis',
+  'audio/ogg',
+  // Fallbacks
+  'audio/wav;codecs=1',
+  'audio/wav',
+  'audio/mpeg',
+  // Keep MP4/AAC as late fallback for non-Safari browsers.
+  'audio/mp4;codecs=mp4a.40.2',
+  'audio/mp4;codecs=mp4a.40.5',
+  'audio/mp4;codecs=aac',
+  'audio/mp4',
+  'audio/aac',
+  'audio/ogg;codecs=speex',
+  'audio/webm;codecs=vorbis',
+];
+
+/**
+ * Checks for supported audio codecs in the current browser and returns the first supported codec.
+ * If no supported codec is found, it returns null.
+ */
+export function getSupportedAudioCodec(): string | null {
+  if (!('MediaRecorder' in globalThis) || !globalThis.MediaRecorder) {
+    return null;
+  }
+
+  const userAgent = globalThis.navigator?.userAgent ?? '';
+  const isIOS =
+    /iPad|iPhone|iPod/.test(userAgent) ||
+    // eslint-disable-next-line @typescript-eslint/no-deprecated
+    (globalThis.navigator?.platform === 'MacIntel' && globalThis.navigator?.maxTouchPoints > 1);
+  const isSafari = /^((?!chrome|android|crios|fxios|edgios).)*safari/i.test(userAgent) || isIOS;
+
+  const preferredCodecs = isSafari ? safariPreferredCodecs : defaultPreferredCodecs;
+  const supportedCodec = preferredCodecs.find((codec) => MediaRecorder.isTypeSupported(codec));
+  return supportedCodec || null;
+}
+
+/**
+ * Returns the appropriate file extension for a given audio codec.
+ * This is used to ensure that the recorded audio file has the correct extension based on the codec used for recording.
+ */
+export function getSupportedAudioExtension(codec: string): string {
+  const baseType = codec.split(';')[0].trim();
+  switch (baseType) {
+    case 'audio/ogg':
+      return 'ogg';
+    case 'audio/webm':
+      return 'webm';
+    case 'audio/mp4':
+      return 'm4a';
+    case 'audio/mpeg':
+      return 'mp3';
+    case 'audio/wav':
+      return 'wav';
+    case 'audio/aac':
+      return 'aac';
+    default:
+      return 'dat'; // default extension for unknown codecs
+  }
+}

src/app/plugins/voice-recorder-kit/types.ts

@@ -7,6 +7,7 @@ export type VoiceRecorderStopPayload = {
   audioUrl: string;
   waveform: number[];
   audioLength: number;
+  audioCodec: string;
 };
 
 export type UseVoiceRecorderOptions = {