feat(a11y): PoC Audio Description

app/containers/message/Components/Attachments/Audio.tsx

@@ -7,6 +7,7 @@ import AudioPlayer from '../../../AudioPlayer';
 import Markdown from '../../../markdown';
 import MessageContext from '../../Context';
 import { useMediaAutoDownload } from '../../hooks/useMediaAutoDownload';
+import AudioTranscribe from './AudioTranscribe';
 
 interface IMessageAudioProps {
 	file: IAttachment;
@@ -25,6 +26,7 @@ const MessageAudio = ({ file, getCustomEmoji, author, msg }: IMessageAudioProps)
 		<View style={{ gap: 4 }}>
 			{msg ? <Markdown msg={msg} username={user.username} getCustomEmoji={getCustomEmoji} /> : null}
 			<AudioPlayer msgId={id} fileUri={url} downloadState={status} onPlayButtonPress={onPress} rid={rid} />
+			<AudioTranscribe uri={url} enabled={status === 'downloaded'} />
 		</View>
 	);
 };

app/containers/message/Components/Attachments/AudioTranscribe.tsx

@@ -0,0 +1,77 @@
+import React, { Suspense, lazy, useState } from 'react';
+import { ActivityIndicator, StyleSheet, Text, TouchableOpacity, View } from 'react-native';
+
+import I18n from '../../../../i18n';
+import { useTheme } from '../../../../theme';
+import sharedStyles from '../../../../views/Styles';
+
+// Lazy-load the runner so `react-native-executorch` (and its native JSI
+// bindings — see SECURITY_REVIEW_REACT_NATIVE_EXECUTORCH.md §11.1) is only
+// pulled into the JS bundle and initialised when the user actually opts in.
+const TranscriptionRunner = lazy(() => import('./TranscriptionRunner'));
+
+interface IAudioTranscribeProps {
+	uri: string;
+	enabled: boolean;
+}
+
+const styles = StyleSheet.create({
+	button: {
+		flexDirection: 'row',
+		alignItems: 'center',
+		alignSelf: 'flex-start',
+		paddingVertical: 6,
+		paddingHorizontal: 12,
+		borderRadius: 4,
+		gap: 8
+	},
+	buttonLabel: {
+		fontSize: 14,
+		...sharedStyles.textSemibold
+	},
+	loading: {
+		flexDirection: 'row',
+		alignItems: 'center',
+		alignSelf: 'flex-start',
+		paddingVertical: 6,
+		paddingHorizontal: 12,
+		gap: 8
+	}
+});
+
+const RunnerFallback = () => {
+	const { colors } = useTheme();
+	return (
+		<View style={styles.loading}>
+			<ActivityIndicator size='small' color={colors.fontDefault} />
+			<Text style={[styles.buttonLabel, { color: colors.fontDefault }]}>{I18n.t('Translating')}</Text>
+		</View>
+	);
+};
+
+const AudioTranscribe = ({ uri, enabled }: IAudioTranscribeProps) => {
+	const { colors } = useTheme();
+	const [started, setStarted] = useState(false);
+
+	if (!enabled || !uri) return null;
+
+	if (started) {
+		return (
+			<Suspense fallback={<RunnerFallback />}>
+				<TranscriptionRunner uri={uri} />
+			</Suspense>
+		);
+	}
+
+	return (
+		<TouchableOpacity
+			accessibilityRole='button'
+			accessibilityLabel={I18n.t('Translate')}
+			onPress={() => setStarted(true)}
+			style={[styles.button, { backgroundColor: colors.buttonBackgroundPrimaryDefault }]}>
+			<Text style={[styles.buttonLabel, { color: colors.fontWhite }]}>{I18n.t('Translate')}</Text>
+		</TouchableOpacity>
+	);
+};
+
+export default AudioTranscribe;

app/containers/message/Components/Attachments/TranscriptionRunner.tsx

@@ -0,0 +1,103 @@
+import React, { useEffect, useRef } from 'react';
+import { AccessibilityInfo, ActivityIndicator, StyleSheet, Text, View } from 'react-native';
+import { initExecutorch } from 'react-native-executorch';
+import { ExpoResourceFetcher } from 'react-native-executorch-expo-resource-fetcher';
+
+import I18n from '../../../../i18n';
+import { useTheme } from '../../../../theme';
+import sharedStyles from '../../../../views/Styles';
+import { useAudioTranscription } from '../../hooks/useAudioTranscription';
+
+// Installing the JSI bindings here (instead of at app boot) keeps the native
+// `fetchUrlFunc` gadget out of the process for users who never trigger
+// transcription. See SECURITY_REVIEW_REACT_NATIVE_EXECUTORCH.md §11.1.
+initExecutorch({ resourceFetcher: ExpoResourceFetcher });
+
+const styles = StyleSheet.create({
+	container: {
+		gap: 6
+	},
+	button: {
+		flexDirection: 'row',
+		alignItems: 'center',
+		alignSelf: 'flex-start',
+		paddingVertical: 6,
+		paddingHorizontal: 12,
+		borderRadius: 4,
+		gap: 8
+	},
+	buttonLabel: {
+		fontSize: 14,
+		...sharedStyles.textSemibold
+	},
+	transcript: {
+		fontSize: 14,
+		...sharedStyles.textRegular
+	},
+	error: {
+		fontSize: 13,
+		...sharedStyles.textRegular
+	}
+});
+
+const TranscriptionRunner = ({ uri }: { uri: string }) => {
+	const { colors } = useTheme();
+	const { status, text, downloadProgress } = useAudioTranscription(uri);
+	const announcedStartRef = useRef(false);
+
+	const isWorking = status === 'loading-model' || status === 'transcribing';
+	let label = I18n.t('Translating');
+	if (status === 'loading-model' && downloadProgress > 0 && downloadProgress < 1) {
+		label = `${I18n.t('Translating')} ${Math.round(downloadProgress * 100)}%`;
+	}
+
+	useEffect(() => {
+		if (isWorking && !announcedStartRef.current) {
+			announcedStartRef.current = true;
+			AccessibilityInfo.announceForAccessibility(I18n.t('Translating'));
+		}
+	}, [isWorking]);
+
+	useEffect(() => {
+		if (status === 'done' && text) {
+			AccessibilityInfo.announceForAccessibility(text);
+		} else if (status === 'error') {
+			AccessibilityInfo.announceForAccessibility(I18n.t('Translation_failed'));
+		}
+	}, [status, text]);
+
+	return (
+		<View style={styles.container}>
+			{isWorking ? (
+				<View
+					accessible
+					accessibilityLiveRegion='polite'
+					accessibilityLabel={label}
+					style={[styles.button, { backgroundColor: colors.buttonBackgroundPrimaryDisabled }]}>
+					<ActivityIndicator size='small' color={colors.fontWhite} />
+					<Text style={[styles.buttonLabel, { color: colors.fontWhite }]}>{label}</Text>
+				</View>
+			) : null}
+			{status === 'done' && text ? (
+				<Text
+					accessible
+					accessibilityLiveRegion='polite'
+					accessibilityLabel={text}
+					style={[styles.transcript, { color: colors.fontDefault }]}>
+					{text}
+				</Text>
+			) : null}
+			{status === 'error' ? (
+				<Text
+					accessible
+					accessibilityLiveRegion='assertive'
+					accessibilityLabel={I18n.t('Translation_failed')}
+					style={[styles.error, { color: colors.fontDanger }]}>
+					{I18n.t('Translation_failed')}
+				</Text>
+			) : null}
+		</View>
+	);
+};
+
+export default TranscriptionRunner;

app/containers/message/hooks/useAudioTranscription.tsx

@@ -0,0 +1,44 @@
+import { useEffect, useRef, useState } from 'react';
+import { type SpeechToTextLanguage, useSpeechToText, WHISPER_TINY } from 'react-native-executorch';
+import { decodeAudioData } from 'react-native-audio-api';
+
+import i18n from '../../../i18n';
+import log from '../../../lib/methods/helpers/log';
+
+export type TTranscriptionStatus = 'loading-model' | 'transcribing' | 'done' | 'error';
+
+const TARGET_SAMPLE_RATE = 16000;
+
+const getTranscriptionLanguage = (): SpeechToTextLanguage => {
+	const base = (i18n.locale || 'en').toLowerCase().split('-')[0];
+	return base as SpeechToTextLanguage;
+};
+
+export const useAudioTranscription = (uri: string) => {
+	const model = useSpeechToText({ model: WHISPER_TINY });
+
+	const [status, setStatus] = useState<TTranscriptionStatus>('loading-model');
+	const [text, setText] = useState<string>('');
+	const startedRef = useRef(false);
+
+	useEffect(() => {
+		if (!uri || startedRef.current) return;
+		if (!model.isReady) return;
+		startedRef.current = true;
+		(async () => {
+			try {
+				setStatus('transcribing');
+				const decoded = await decodeAudioData(uri, TARGET_SAMPLE_RATE);
+				const waveform = decoded.getChannelData(0);
+				const result = await model.transcribe(waveform, { language: getTranscriptionLanguage() });
+				setText(result?.text ?? '');
+				setStatus('done');
+			} catch (e) {
+				log(e);
+				setStatus('error');
+			}
+		})();
+	}, [uri, model.isReady, model]);
+
+	return { status, text, downloadProgress: model.downloadProgress };
+};

app/i18n/locales/en.json

@@ -917,6 +917,8 @@
   "topic": "topic",
   "totp-invalid": "Code or password invalid",
   "Translate": "Translate",
+  "Translating": "Translating…",
+  "Translation_failed": "Translation failed",
   "Travel_and_places": "Travel and places",
   "Troubleshooting": "Troubleshooting",
   "Try_again": "Try again",