From b7ec35b639e939965ca6d1ac40d5d89bb703509a Mon Sep 17 00:00:00 2001 From: theInfiTualEr Date: Sun, 24 Sep 2023 14:02:28 +0330 Subject: [PATCH 1/3] play functionality, typescript errors fixed, initial warning fixed --- README.md | 14 +++++++ .../RNLiveAudioStreamModule.java | 40 +++++++++++++++++++ index.d.ts | 25 +++++++++++- index.ios.js | 27 +++++++++++++ index.js | 3 ++ 5 files changed, 107 insertions(+), 2 deletions(-) create mode 100644 index.ios.js diff --git a/README.md b/README.md index 7ce0036..f347939 100644 --- a/README.md +++ b/README.md @@ -33,6 +33,8 @@ Add the following line to ```android/app/src/main/AndroidManifest.xml``` ## Usage ```javascript import LiveAudioStream from 'react-native-live-audio-stream'; +import { PermissionsAndroid } from "react-native"; + const options = { sampleRate: 32000, // default is 44100 but 32000 is adequate for accurate voice recognition @@ -42,14 +44,26 @@ const options = { bufferSize: 4096 // default is 2048 }; +await PermissionsAndroid.request( + PermissionsAndroid.PERMISSIONS.RECORD_AUDIO +); + LiveAudioStream.init(options); LiveAudioStream.on('data', data => { // base64-encoded audio data chunks + + // below line plays the received audio data + // NOTE: this DOES NOT WORK on iOS + LiveAudioStream.addPlay(data); }); ... +// NOTE: `startPlay` is not necessary on iOS +LiveAudioStream.startPlay(); LiveAudioStream.start(); ... LiveAudioStream.stop(); +// NOTE: `stopPlay` is not necessary on iOS +LiveAudioStream.stopPlay(); ... ``` diff --git a/android/src/main/java/com/imxiqi/rnliveaudiostream/RNLiveAudioStreamModule.java b/android/src/main/java/com/imxiqi/rnliveaudiostream/RNLiveAudioStreamModule.java index 20f454f..ca612ff 100644 --- a/android/src/main/java/com/imxiqi/rnliveaudiostream/RNLiveAudioStreamModule.java +++ b/android/src/main/java/com/imxiqi/rnliveaudiostream/RNLiveAudioStreamModule.java @@ -2,6 +2,8 @@ import android.media.AudioFormat; import android.media.AudioRecord; +import android.media.AudioTrack; +import android.media.AudioManager; import android.media.MediaRecorder.AudioSource; import android.util.Base64; import android.util.Log; @@ -25,6 +27,7 @@ public class RNLiveAudioStreamModule extends ReactContextBaseJavaModule { private int audioFormat; private int audioSource; + private AudioTrack player; private AudioRecord recorder; private int bufferSize; private boolean isRecording; @@ -76,6 +79,14 @@ public void init(ReadableMap options) { int recordingBufferSize = bufferSize * 3; recorder = new AudioRecord(audioSource, sampleRateInHz, channelConfig, audioFormat, recordingBufferSize); + + // more info: https://stackoverflow.com/questions/9413998/live-audio-recording-and-playing-in-android-and-thread-callback-handling + // TODO channelConfig or AudioFormat.CHANNEL_OUT_MONO ? + // TODO recordingBufferSize or bufferSize ? + player = new AudioTrack(AudioManager.STREAM_MUSIC, + sampleRateInHz, AudioFormat.CHANNEL_OUT_MONO, audioFormat, + recordingBufferSize, AudioTrack.MODE_STREAM); + player.setPlaybackRate(sampleRateInHz); } @ReactMethod @@ -114,4 +125,33 @@ public void run() { public void stop(Promise promise) { isRecording = false; } + + @ReactMethod + public void startPlay() { + player.play(); + } + + @ReactMethod + public void addPlay(String audioBufferBase64) { + byte[] audioBuffer = Base64.decode(audioBufferBase64, Base64.NO_WRAP); + // player.flush(); + player.write(audioBuffer, 0, audioBuffer.length); + } + + @ReactMethod + public void stopPlay() { + player.stop(); + // TODO should not release? considering we might play again, or not. + player.release(); + } + + @ReactMethod + public void addListener(String eventName) { + // Keep: Required for RN built in Event Emitter Calls. + } + + @ReactMethod + public void removeListeners(Integer count) { + // Keep: Required for RN built in Event Emitter Calls. + } } diff --git a/index.d.ts b/index.d.ts index b846438..e9fd10c 100644 --- a/index.d.ts +++ b/index.d.ts @@ -1,8 +1,30 @@ declare module "react-native-live-audio-stream" { export interface IAudioRecord { init: (options: Options) => void + /** + * make sure to call `init` before this + */ start: () => void - stop: () => Promise + stop: () => void + /** + * make sure to call `init` before this + * NOTE: this DOES NOT WORK on iOS + */ + startPlay: () => void; + /** + * NOTE: this DOES NOT WORK on iOS + * @param audioBufferBase64 same data that you got on `data` event + */ + addPlay: (audioBufferBase64: string) => void; + /** + * NOTE: this DOES NOT WORK on iOS + */ + stopPlay: () => void; + /** + * + * @param event + * @param callback provides data as base64 header-less wave audio + */ on: (event: "data", callback: (data: string) => void) => void } @@ -20,7 +42,6 @@ declare module "react-native-live-audio-stream" { * - `6` */ audioSource?: number - wavFile: string bufferSize?: number } diff --git a/index.ios.js b/index.ios.js new file mode 100644 index 0000000..163770e --- /dev/null +++ b/index.ios.js @@ -0,0 +1,27 @@ +import { NativeModules, NativeEventEmitter } from 'react-native'; +const { RNLiveAudioStream } = NativeModules; +const EventEmitter = new NativeEventEmitter(RNLiveAudioStream); + +const AudioRecord = {}; + +AudioRecord.init = options => RNLiveAudioStream.init(options); +AudioRecord.start = () => RNLiveAudioStream.start(); +AudioRecord.stop = () => RNLiveAudioStream.stop(); +AudioRecord.startPlay = () => {}; +AudioRecord.addPlay = (audioBufferBase64) => {}; +AudioRecord.stopPlay = () => {}; + +const eventsMap = { + data: 'data' +}; + +AudioRecord.on = (event, callback) => { + const nativeEvent = eventsMap[event]; + if (!nativeEvent) { + throw new Error('Invalid event'); + } + EventEmitter.removeAllListeners(nativeEvent); + return EventEmitter.addListener(nativeEvent, callback); +}; + +export default AudioRecord; diff --git a/index.js b/index.js index c5968b2..600b2f9 100644 --- a/index.js +++ b/index.js @@ -7,6 +7,9 @@ const AudioRecord = {}; AudioRecord.init = options => RNLiveAudioStream.init(options); AudioRecord.start = () => RNLiveAudioStream.start(); AudioRecord.stop = () => RNLiveAudioStream.stop(); +AudioRecord.startPlay = () => RNLiveAudioStream.startPlay(); +AudioRecord.addPlay = (audioBufferBase64) => RNLiveAudioStream.addPlay(audioBufferBase64); +AudioRecord.stopPlay = () => RNLiveAudioStream.stopPlay(); const eventsMap = { data: 'data' From 189ca6e8920eb19219879e7de6fed340a1910f91 Mon Sep 17 00:00:00 2001 From: theInfiTualEr Date: Tue, 3 Oct 2023 11:17:47 +0330 Subject: [PATCH 2/3] forgot to set channels in AudioTrack --- .../com/imxiqi/rnliveaudiostream/RNLiveAudioStreamModule.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/android/src/main/java/com/imxiqi/rnliveaudiostream/RNLiveAudioStreamModule.java b/android/src/main/java/com/imxiqi/rnliveaudiostream/RNLiveAudioStreamModule.java index ca612ff..b1e5fe7 100644 --- a/android/src/main/java/com/imxiqi/rnliveaudiostream/RNLiveAudioStreamModule.java +++ b/android/src/main/java/com/imxiqi/rnliveaudiostream/RNLiveAudioStreamModule.java @@ -81,10 +81,9 @@ public void init(ReadableMap options) { recorder = new AudioRecord(audioSource, sampleRateInHz, channelConfig, audioFormat, recordingBufferSize); // more info: https://stackoverflow.com/questions/9413998/live-audio-recording-and-playing-in-android-and-thread-callback-handling - // TODO channelConfig or AudioFormat.CHANNEL_OUT_MONO ? // TODO recordingBufferSize or bufferSize ? player = new AudioTrack(AudioManager.STREAM_MUSIC, - sampleRateInHz, AudioFormat.CHANNEL_OUT_MONO, audioFormat, + sampleRateInHz, channelConfig, audioFormat, recordingBufferSize, AudioTrack.MODE_STREAM); player.setPlaybackRate(sampleRateInHz); } From e2fd76bc3f10429d737b1341ee18cb774f21161f Mon Sep 17 00:00:00 2001 From: theInfiTualEr Date: Wed, 4 Oct 2023 12:30:32 +0330 Subject: [PATCH 3/3] load/unload, adding header to audio --- README.md | 19 ++- .../RNLiveAudioStreamModule.java | 134 +++++++++++++++--- index.d.ts | 30 +++- index.ios.js | 6 +- index.js | 4 + 5 files changed, 163 insertions(+), 30 deletions(-) diff --git a/README.md b/README.md index f347939..164195b 100644 --- a/README.md +++ b/README.md @@ -1,16 +1,13 @@ -# react-native-live-audio-stream -[![npm](https://img.shields.io/npm/v/react-native-live-audio-stream)](https://www.npmjs.com/package/react-native-live-audio-stream) - -Get live audio stream data for React Native. Ideal for live voice recognition (transcribing). +# theInfiTualEr/react-native-live-audio-stream -This module is modified from [react-native-audio-record](https://github.com/goodatlas/react-native-audio-record). Instead of saving to an audio file, it only emit events with live data. By doing this, it can reduce memory usage and eliminate file operation overheads in the case that an audio file is not necessary (e.g. live transcribing). +This package is a modified version of [react-native-live-audio-stream](https://github.com/xiqi/react-native-live-audio-stream) which that itself is a modification of [react-native-audio-record](https://github.com/goodatlas/react-native-audio-record) package. This package adds **play**, **unload** and **generating audio header** functionality to the `react-native-live-audio-stream` package, but **ONLY FOR ANDROID.** -Most of the code was written by the respective original authors. +[![npm](https://img.shields.io/npm/v/react-native-live-audio-stream)](https://www.npmjs.com/package/react-native-live-audio-stream) ## Install ``` -yarn add react-native-live-audio-stream +npm install theInfiTualEr/react-native-live-audio-stream cd ios pod install ``` @@ -42,6 +39,7 @@ const options = { bitsPerSample: 16, // 8 or 16, default 16 audioSource: 6, // android only (see below) bufferSize: 4096 // default is 2048 + hasAudioHeader: true // default is false, but you probably need it }; await PermissionsAndroid.request( @@ -57,13 +55,19 @@ LiveAudioStream.on('data', data => { LiveAudioStream.addPlay(data); }); ... +// NOTE: `loadPlayer` is not necessary on iOS +LiveAudioStream.loadPlayer(); // NOTE: `startPlay` is not necessary on iOS LiveAudioStream.startPlay(); +LiveAudioStream.loadRecorder(); LiveAudioStream.start(); ... LiveAudioStream.stop(); +LiveAudioStream.unloadRecorder(); // NOTE: `stopPlay` is not necessary on iOS LiveAudioStream.stopPlay(); +// NOTE: `unloadPlayer` is not necessary on iOS +LiveAudioStream.unloadPlayer(); ... ``` @@ -80,6 +84,7 @@ LiveAudioStream.on('data', data => { ``` ## Credits/References +- [react-native-live-audio-stream](https://github.com/xiqi/react-native-live-audio-stream) - [react-native-audio-record](https://github.com/goodatlas/react-native-audio-record) - iOS [Audio Queues](https://developer.apple.com/library/content/documentation/MusicAudio/Conceptual/AudioQueueProgrammingGuide) - Android [AudioRecord](https://developer.android.com/reference/android/media/AudioRecord.html) diff --git a/android/src/main/java/com/imxiqi/rnliveaudiostream/RNLiveAudioStreamModule.java b/android/src/main/java/com/imxiqi/rnliveaudiostream/RNLiveAudioStreamModule.java index b1e5fe7..420902f 100644 --- a/android/src/main/java/com/imxiqi/rnliveaudiostream/RNLiveAudioStreamModule.java +++ b/android/src/main/java/com/imxiqi/rnliveaudiostream/RNLiveAudioStreamModule.java @@ -6,7 +6,6 @@ import android.media.AudioManager; import android.media.MediaRecorder.AudioSource; import android.util.Base64; -import android.util.Log; import com.facebook.react.bridge.Promise; import com.facebook.react.bridge.ReactApplicationContext; @@ -26,12 +25,15 @@ public class RNLiveAudioStreamModule extends ReactContextBaseJavaModule { private int channelConfig; private int audioFormat; private int audioSource; + private boolean hasAudioHeader; private AudioTrack player; private AudioRecord recorder; private int bufferSize; private boolean isRecording; + private byte[] wavHeaders; + public RNLiveAudioStreamModule(ReactApplicationContext reactContext) { super(reactContext); this.reactContext = reactContext; @@ -50,17 +52,13 @@ public void init(ReadableMap options) { } channelConfig = AudioFormat.CHANNEL_IN_MONO; - if (options.hasKey("channels")) { - if (options.getInt("channels") == 2) { - channelConfig = AudioFormat.CHANNEL_IN_STEREO; - } + if (options.hasKey("channels") && options.getInt("channels") == 2) { + channelConfig = AudioFormat.CHANNEL_IN_STEREO; } audioFormat = AudioFormat.ENCODING_PCM_16BIT; - if (options.hasKey("bitsPerSample")) { - if (options.getInt("bitsPerSample") == 8) { - audioFormat = AudioFormat.ENCODING_PCM_8BIT; - } + if (options.hasKey("bitsPerSample") && options.getInt("bitsPerSample") == 8) { + audioFormat = AudioFormat.ENCODING_PCM_8BIT; } audioSource = AudioSource.VOICE_RECOGNITION; @@ -68,24 +66,24 @@ public void init(ReadableMap options) { audioSource = options.getInt("audioSource"); } + hasAudioHeader = false; + if (options.hasKey("hasAudioHeader")) { + hasAudioHeader = options.getBoolean("hasAudioHeader"); + } + isRecording = false; eventEmitter = reactContext.getJSModule(DeviceEventManagerModule.RCTDeviceEventEmitter.class); bufferSize = AudioRecord.getMinBufferSize(sampleRateInHz, channelConfig, audioFormat); - if (options.hasKey("bufferSize")) { bufferSize = Math.max(bufferSize, options.getInt("bufferSize")); } - int recordingBufferSize = bufferSize * 3; - recorder = new AudioRecord(audioSource, sampleRateInHz, channelConfig, audioFormat, recordingBufferSize); - - // more info: https://stackoverflow.com/questions/9413998/live-audio-recording-and-playing-in-android-and-thread-callback-handling // TODO recordingBufferSize or bufferSize ? - player = new AudioTrack(AudioManager.STREAM_MUSIC, - sampleRateInHz, channelConfig, audioFormat, - recordingBufferSize, AudioTrack.MODE_STREAM); - player.setPlaybackRate(sampleRateInHz); + // int recordingBufferSize = bufferSize * 3; + long totalAudioLen = bufferSize; + long totalDataLen = totalAudioLen + 36; + wavHeaders = genWavHeader(totalAudioLen, totalDataLen); } @ReactMethod @@ -106,7 +104,18 @@ public void run() { // skip first 2 buffers to eliminate "click sound" if (bytesRead > 0 && ++count > 2) { - base64Data = Base64.encodeToString(buffer, Base64.NO_WRAP); + if (!hasAudioHeader) { + base64Data = Base64.encodeToString(buffer, Base64.NO_WRAP); + eventEmitter.emit("data", base64Data); + continue; + } + + byte[] combined = new byte[wavHeaders.length + buffer.length]; + System.arraycopy(wavHeaders, 0, combined, 0, wavHeaders.length); + System.arraycopy(buffer, 0, combined, wavHeaders.length, buffer.length); + + base64Data = Base64.encodeToString(combined, Base64.NO_WRAP); + eventEmitter.emit("data", base64Data); } } @@ -132,18 +141,44 @@ public void startPlay() { @ReactMethod public void addPlay(String audioBufferBase64) { - byte[] audioBuffer = Base64.decode(audioBufferBase64, Base64.NO_WRAP); // player.flush(); + byte[] audioBuffer = Base64.decode(audioBufferBase64, Base64.NO_WRAP); player.write(audioBuffer, 0, audioBuffer.length); } @ReactMethod public void stopPlay() { player.stop(); - // TODO should not release? considering we might play again, or not. + } + + @ReactMethod + public void loadPlayer() { + // more info: + // https://stackoverflow.com/questions/9413998/live-audio-recording-and-playing-in-android-and-thread-callback-handling + // TODO recordingBufferSize or bufferSize ? + // int recordingBufferSize = bufferSize * 3; + player = new AudioTrack(AudioManager.STREAM_MUSIC, + sampleRateInHz, channelConfig, audioFormat, + bufferSize, AudioTrack.MODE_STREAM); + player.setPlaybackRate(sampleRateInHz); + } + + @ReactMethod + public void unloadPlayer() { player.release(); } + @ReactMethod + public void loadRecorder() { + // int recordingBufferSize = bufferSize * 3; + recorder = new AudioRecord(audioSource, sampleRateInHz, channelConfig, audioFormat, bufferSize); + } + + @ReactMethod + public void unloadRecorder() { + recorder.release(); + } + @ReactMethod public void addListener(String eventName) { // Keep: Required for RN built in Event Emitter Calls. @@ -153,4 +188,61 @@ public void addListener(String eventName) { public void removeListeners(Integer count) { // Keep: Required for RN built in Event Emitter Calls. } + + private byte[] genWavHeader(long totalAudioLen, long totalDataLen) { + long sampleRate = sampleRateInHz; + int channels = channelConfig == AudioFormat.CHANNEL_IN_MONO ? 1 : 2; + int bitsPerSample = audioFormat == AudioFormat.ENCODING_PCM_8BIT ? 8 : 16; + long byteRate = sampleRate * channels * bitsPerSample / 8; + int blockAlign = channels * bitsPerSample / 8; + + byte[] header = new byte[44]; + + header[0] = 'R'; // RIFF chunk + header[1] = 'I'; + header[2] = 'F'; + header[3] = 'F'; + header[4] = (byte) (totalDataLen & 0xff); // how big is the rest of this file + header[5] = (byte) ((totalDataLen >> 8) & 0xff); + header[6] = (byte) ((totalDataLen >> 16) & 0xff); + header[7] = (byte) ((totalDataLen >> 24) & 0xff); + header[8] = 'W'; // WAVE chunk + header[9] = 'A'; + header[10] = 'V'; + header[11] = 'E'; + header[12] = 'f'; // 'fmt ' chunk + header[13] = 'm'; + header[14] = 't'; + header[15] = ' '; + header[16] = 16; // 4 bytes: size of 'fmt ' chunk + header[17] = 0; + header[18] = 0; + header[19] = 0; + header[20] = 1; // format = 1 for PCM + header[21] = 0; + header[22] = (byte) channels; // mono or stereo + header[23] = 0; + header[24] = (byte) (sampleRate & 0xff); // samples per second + header[25] = (byte) ((sampleRate >> 8) & 0xff); + header[26] = (byte) ((sampleRate >> 16) & 0xff); + header[27] = (byte) ((sampleRate >> 24) & 0xff); + header[28] = (byte) (byteRate & 0xff); // bytes per second + header[29] = (byte) ((byteRate >> 8) & 0xff); + header[30] = (byte) ((byteRate >> 16) & 0xff); + header[31] = (byte) ((byteRate >> 24) & 0xff); + header[32] = (byte) blockAlign; // bytes in one sample, for all channels + header[33] = 0; + header[34] = (byte) bitsPerSample; // bits in a sample + header[35] = 0; + header[36] = 'd'; // beginning of the data chunk + header[37] = 'a'; + header[38] = 't'; + header[39] = 'a'; + header[40] = (byte) (totalAudioLen & 0xff); // how big is this data chunk + header[41] = (byte) ((totalAudioLen >> 8) & 0xff); + header[42] = (byte) ((totalAudioLen >> 16) & 0xff); + header[43] = (byte) ((totalAudioLen >> 24) & 0xff); + + return header; + } } diff --git a/index.d.ts b/index.d.ts index e9fd10c..ad56fe8 100644 --- a/index.d.ts +++ b/index.d.ts @@ -2,10 +2,34 @@ declare module "react-native-live-audio-stream" { export interface IAudioRecord { init: (options: Options) => void /** - * make sure to call `init` before this + * make sure to call `init` before this, + * starts recording. */ start: () => void + /** + * stops recording. + */ stop: () => void + /** + * has to be called before playing audio. + * NOTE: this DOES NOT WORK on iOS + */ + loadPlayer: () => void + /** + * unloads the player resources, can be called on unmount. + * NOTE: this DOES NOT WORK on iOS + */ + unloadPlayer: () => void + /** + * has to be called before playing recorder. + * NOTE: this DOES NOT WORK on iOS + */ + loadRecorder: () => void + /** + * unloads the recorder resources, can be called on unmount. + * NOTE: this DOES NOT WORK on iOS + */ + unloadRecorder: () => void /** * make sure to call `init` before this * NOTE: this DOES NOT WORK on iOS @@ -43,6 +67,10 @@ declare module "react-native-live-audio-stream" { */ audioSource?: number bufferSize?: number + /** + * you probably want this to be true if you want to play it elsewhere. + */ + hasAudioHeader?: boolean } const AudioRecord: IAudioRecord diff --git a/index.ios.js b/index.ios.js index 163770e..ad6640f 100644 --- a/index.ios.js +++ b/index.ios.js @@ -5,10 +5,14 @@ const EventEmitter = new NativeEventEmitter(RNLiveAudioStream); const AudioRecord = {}; AudioRecord.init = options => RNLiveAudioStream.init(options); +AudioRecord.loadPlayer = () => {}; +AudioRecord.unloadPlayer = () => {}; +AudioRecord.loadRecorder = () => {}; +AudioRecord.unloadRecorder = () => {}; AudioRecord.start = () => RNLiveAudioStream.start(); AudioRecord.stop = () => RNLiveAudioStream.stop(); AudioRecord.startPlay = () => {}; -AudioRecord.addPlay = (audioBufferBase64) => {}; +AudioRecord.addPlay = () => {}; AudioRecord.stopPlay = () => {}; const eventsMap = { diff --git a/index.js b/index.js index 600b2f9..3a29705 100644 --- a/index.js +++ b/index.js @@ -5,6 +5,10 @@ const EventEmitter = new NativeEventEmitter(RNLiveAudioStream); const AudioRecord = {}; AudioRecord.init = options => RNLiveAudioStream.init(options); +AudioRecord.loadPlayer = () => RNLiveAudioStream.loadPlayer(); +AudioRecord.unloadPlayer = () => RNLiveAudioStream.unloadPlayer(); +AudioRecord.loadRecorder = () => RNLiveAudioStream.loadRecorder(); +AudioRecord.unloadRecorder = () => RNLiveAudioStream.unloadRecorder(); AudioRecord.start = () => RNLiveAudioStream.start(); AudioRecord.stop = () => RNLiveAudioStream.stop(); AudioRecord.startPlay = () => RNLiveAudioStream.startPlay();