diff --git a/CHANGELOG.md b/CHANGELOG.md index ea03acc26..ee7cf53ff 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -50,6 +50,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **AudioWorklet**: `BufferSize::Fixed` now sets `renderSizeHint` on the `AudioContext`. - **CoreAudio**: Timestamps now include device latency and safety offset. - **CoreAudio**: Poisoned stream mutex in stream functions now propagate panics. +- **CoreAudio**: Physical stream format is now set directly on the hardware device. +- **CoreAudio**: Stream error callback now receives `StreamError::StreamInvalidated` on any sample + rate change on macOS, and on iOS on route changes that require a stream rebuild. +- **CoreAudio**: Stream error callback now receives `StreamError::DeviceNotAvailable` on iOS + when media services are lost. +- **CoreAudio**: User timeouts are now obeyed when building a stream. - **JACK**: Timestamps now use the precise hardware deadline. - **JACK**: Buffer size change no longer fires an error callback; internal buffers are resized without error. diff --git a/Cargo.toml b/Cargo.toml index 118b509da..5b0c74953 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -136,16 +136,29 @@ objc2-core-audio-types = { version = "0.3", default-features = false, features = "CoreAudioBaseTypes", ] } objc2-core-foundation = { version = "0.3" } -objc2-foundation = { version = "0.3" } +objc2-foundation = { version = "0.3", default-features = false, features = [ + "std", + "NSArray", + "NSString", + "NSValue", +] } objc2 = { version = "0.6" } [target.'cfg(target_os = "macos")'.dependencies] jack = { version = "0.13", optional = true } [target.'cfg(target_os = "ios")'.dependencies] +block2 = "0.6" +objc2-foundation = { version = "0.3", features = [ + "block2", + "NSDictionary", + "NSNotification", + "NSOperation", +] } objc2-avf-audio = { version = "0.3", default-features = false, features = [ "std", "AVAudioSession", + "AVAudioSessionTypes", ] } [target.'cfg(target_os = "emscripten")'.dependencies] diff --git a/src/host/coreaudio/ios/mod.rs b/src/host/coreaudio/ios/mod.rs index 713b6a3e3..04d9f4f0f 100644 --- a/src/host/coreaudio/ios/mod.rs +++ b/src/host/coreaudio/ios/mod.rs @@ -1,13 +1,15 @@ //! CoreAudio implementation for iOS using AVAudioSession and RemoteIO Audio Units. +use std::ptr::NonNull; +use std::sync::Arc; use std::sync::Mutex; +use std::time::Duration; use coreaudio::audio_unit::render_callback::data; use coreaudio::audio_unit::{render_callback, AudioUnit, Element, Scope}; use objc2_audio_toolbox::{kAudioOutputUnitProperty_EnableIO, kAudioUnitProperty_StreamFormat}; -use objc2_core_audio_types::AudioBuffer; - use objc2_avf_audio::AVAudioSession; +use objc2_core_audio_types::AudioBuffer; use super::{asbd_from_config, frames_to_duration, host_time_to_stream_instant}; use crate::traits::{DeviceTrait, HostTrait, StreamTrait}; @@ -25,10 +27,10 @@ use self::enumerate::{ default_input_device, default_output_device, Devices, SupportedInputConfigs, SupportedOutputConfigs, }; -use std::ptr::NonNull; -use std::time::Duration; pub mod enumerate; +mod session_event_manager; +use session_event_manager::{ErrorCallbackMutex, SessionEventManager}; // These days the default of iOS is now F32 and no longer I16 const SUPPORTED_SAMPLE_FORMAT: SampleFormat = SampleFormat::F32; @@ -169,6 +171,9 @@ impl DeviceTrait for Device { // Query device buffer size for latency calculation let device_buffer_frames = Some(get_device_buffer_frames()); + let error_callback: ErrorCallbackMutex = Arc::new(Mutex::new(Box::new(error_callback))); + let session_manager = SessionEventManager::new(error_callback.clone()); + // Set up input callback setup_input_callback( &mut audio_unit, @@ -176,15 +181,22 @@ impl DeviceTrait for Device { config.sample_rate, device_buffer_frames, data_callback, - error_callback, + move |e| { + if let Ok(mut cb) = error_callback.lock() { + cb(e); + } + }, )?; audio_unit.start()?; - Ok(Stream::new(StreamInner { - playing: true, - audio_unit, - })) + Ok(Stream::new( + StreamInner { + playing: true, + audio_unit, + }, + session_manager, + )) } /// Create an output stream. @@ -206,6 +218,9 @@ impl DeviceTrait for Device { // Query device buffer size for latency calculation let device_buffer_frames = Some(get_device_buffer_frames()); + let error_callback: ErrorCallbackMutex = Arc::new(Mutex::new(Box::new(error_callback))); + let session_manager = SessionEventManager::new(error_callback.clone()); + // Set up output callback setup_output_callback( &mut audio_unit, @@ -213,26 +228,35 @@ impl DeviceTrait for Device { config.sample_rate, device_buffer_frames, data_callback, - error_callback, + move |e| { + if let Ok(mut cb) = error_callback.lock() { + cb(e); + } + }, )?; audio_unit.start()?; - Ok(Stream::new(StreamInner { - playing: true, - audio_unit, - })) + Ok(Stream::new( + StreamInner { + playing: true, + audio_unit, + }, + session_manager, + )) } } pub struct Stream { inner: Mutex, + _session_manager: SessionEventManager, } impl Stream { - fn new(inner: StreamInner) -> Self { + fn new(inner: StreamInner, session_manager: SessionEventManager) -> Self { Self { inner: Mutex::new(inner), + _session_manager: session_manager, } } } diff --git a/src/host/coreaudio/ios/session_event_manager.rs b/src/host/coreaudio/ios/session_event_manager.rs new file mode 100644 index 000000000..7d1dd94e6 --- /dev/null +++ b/src/host/coreaudio/ios/session_event_manager.rs @@ -0,0 +1,116 @@ +//! Monitors AVAudioSession lifecycle events and reports them as stream errors. + +use std::ptr::NonNull; +use std::sync::{Arc, Mutex}; + +use block2::RcBlock; +use objc2::runtime::AnyObject; +use objc2_avf_audio::{ + AVAudioSessionMediaServicesWereLostNotification, + AVAudioSessionMediaServicesWereResetNotification, AVAudioSessionRouteChangeNotification, + AVAudioSessionRouteChangeReason, AVAudioSessionRouteChangeReasonKey, +}; +use objc2_foundation::{NSNotification, NSNotificationCenter, NSNumber, NSString}; + +use crate::StreamError; + +pub(super) type ErrorCallbackMutex = Arc>>; + +unsafe fn route_change_error(notification: &NSNotification) -> Option { + let user_info = notification.userInfo()?; + let key = AVAudioSessionRouteChangeReasonKey?; + let dict = unsafe { user_info.cast_unchecked::() }; + let value = dict.objectForKey(key)?; + let number = value.downcast_ref::()?; + let reason = AVAudioSessionRouteChangeReason(number.unsignedIntegerValue()); + match reason { + AVAudioSessionRouteChangeReason::OldDeviceUnavailable + | AVAudioSessionRouteChangeReason::CategoryChange + | AVAudioSessionRouteChangeReason::Override + | AVAudioSessionRouteChangeReason::RouteConfigurationChange => { + Some(StreamError::StreamInvalidated) + } + + AVAudioSessionRouteChangeReason::NoSuitableRouteForCategory => { + Some(StreamError::DeviceNotAvailable) + } + + _ => None, + } +} + +pub(super) struct SessionEventManager { + observers: Vec< + objc2::rc::Retained>, + >, +} + +// SAFETY: NSNotificationCenter is thread-safe on iOS. The observer tokens stored here are opaque +// handles used only to call removeObserver in Drop; no data is read or written through them. +unsafe impl Send for SessionEventManager {} +unsafe impl Sync for SessionEventManager {} + +impl SessionEventManager { + pub(super) fn new(error_callback: ErrorCallbackMutex) -> Self { + let nc = NSNotificationCenter::defaultCenter(); + let mut observers = Vec::new(); + + { + let cb = error_callback.clone(); + let block = RcBlock::new(move |notif: NonNull| { + if let Some(err) = unsafe { route_change_error(notif.as_ref()) } { + if let Ok(mut cb) = cb.lock() { + cb(err); + } + } + }); + if let Some(name) = unsafe { AVAudioSessionRouteChangeNotification } { + let observer = unsafe { + nc.addObserverForName_object_queue_usingBlock(Some(name), None, None, &block) + }; + observers.push(observer); + } + } + + { + let cb = error_callback.clone(); + let block = RcBlock::new(move |_: NonNull| { + if let Ok(mut cb) = cb.lock() { + cb(StreamError::DeviceNotAvailable); + } + }); + if let Some(name) = unsafe { AVAudioSessionMediaServicesWereLostNotification } { + let observer = unsafe { + nc.addObserverForName_object_queue_usingBlock(Some(name), None, None, &block) + }; + observers.push(observer); + } + } + + { + let cb = error_callback.clone(); + let block = RcBlock::new(move |_: NonNull| { + if let Ok(mut cb) = cb.lock() { + cb(StreamError::StreamInvalidated); + } + }); + if let Some(name) = unsafe { AVAudioSessionMediaServicesWereResetNotification } { + let observer = unsafe { + nc.addObserverForName_object_queue_usingBlock(Some(name), None, None, &block) + }; + observers.push(observer); + } + } + + Self { observers } + } +} + +impl Drop for SessionEventManager { + fn drop(&mut self) { + let nc = NSNotificationCenter::defaultCenter(); + for observer in &self.observers { + unsafe { nc.removeObserver(observer.as_ref()) }; + } + } +} diff --git a/src/host/coreaudio/macos/device.rs b/src/host/coreaudio/macos/device.rs index 33cd27bb8..c63e7fa01 100644 --- a/src/host/coreaudio/macos/device.rs +++ b/src/host/coreaudio/macos/device.rs @@ -10,8 +10,14 @@ use crate::{ OutputCallbackInfo, SampleFormat, SampleRate, StreamConfig, StreamError, SupportedBufferSize, SupportedStreamConfig, SupportedStreamConfigRange, SupportedStreamConfigsError, }; +use coreaudio::audio_unit::audio_format::LinearPcmFlags; +use coreaudio::audio_unit::macos_helpers::{ + find_matching_physical_format, set_device_physical_stream_format, RateListener, +}; use coreaudio::audio_unit::render_callback::{self, data}; -use coreaudio::audio_unit::{AudioUnit, Element, Scope}; +use coreaudio::audio_unit::{ + AudioUnit, Element, SampleFormat as CoreAudioSampleFormat, Scope, StreamFormat, +}; use objc2_audio_toolbox::{ kAudioOutputUnitProperty_CurrentDevice, kAudioOutputUnitProperty_EnableIO, kAudioUnitProperty_StreamFormat, @@ -46,14 +52,45 @@ use std::sync::{Arc, Mutex}; use std::time::{Duration, Instant}; use super::invoke_error_callback; -use super::property_listener::AudioObjectPropertyListener; use coreaudio::audio_unit::macos_helpers::get_device_name; -/// Attempt to set the device sample rate to the provided rate. -/// Return an error if the requested sample rate is not supported by the device. +/// Try to find a matching physical stream format on the device and apply it. +/// +/// Setting the physical format ensures the hardware runs at the requested bit depth and sample +/// rate without unnecessary conversions. +fn set_physical_format( + device_id: AudioDeviceID, + sample_rate: SampleRate, + channels: ChannelCount, + sample_format: SampleFormat, +) -> Result { + let core_format = match sample_format { + SampleFormat::I8 => CoreAudioSampleFormat::I8, + SampleFormat::I16 => CoreAudioSampleFormat::I16, + SampleFormat::I24 => CoreAudioSampleFormat::I24, + SampleFormat::I32 => CoreAudioSampleFormat::I32, + SampleFormat::F32 => CoreAudioSampleFormat::F32, + _ => return Err(coreaudio::Error::UnsupportedStreamFormat), + }; + let stream_format = StreamFormat { + sample_rate: sample_rate as f64, + sample_format: core_format, + flags: LinearPcmFlags::empty(), + channels: channels as u32, + }; + let asbd = find_matching_physical_format(device_id, stream_format) + .ok_or(coreaudio::Error::UnsupportedStreamFormat)?; + set_device_physical_stream_format(device_id, asbd).map(|_| asbd) +} + +/// Set the device's nominal sample rate via `kAudioDevicePropertyNominalSampleRate`. +/// +/// Unlike [`set_physical_format`], this only changes the device clock rate. The AudioUnit bridges +/// any remaining format difference to the virtual stream format seen by the callback. fn set_sample_rate( audio_device_id: AudioObjectID, target_sample_rate: SampleRate, + timeout: Option, ) -> Result<(), BuildStreamError> { // Get the current sample rate. let mut property_address = AudioObjectPropertyAddress { @@ -76,7 +113,7 @@ fn set_sample_rate( coreaudio::Error::from_os_status(status)?; // If the requested sample rate is different to the device sample rate, update the device. - if sample_rate as u32 != target_sample_rate { + if (sample_rate - target_sample_rate as f64).abs() >= 1.0 { // Get available sample rate ranges. property_address.mSelector = kAudioDevicePropertyAvailableNominalSampleRates; let mut data_size = 0u32; @@ -116,40 +153,13 @@ fn set_sample_rate( return Err(BuildStreamError::StreamConfigNotSupported); } - let (send, recv) = channel::>(); - let sample_rate_address = AudioObjectPropertyAddress { - mSelector: kAudioDevicePropertyNominalSampleRate, - mScope: kAudioObjectPropertyScopeGlobal, - mElement: kAudioObjectPropertyElementMaster, - }; - // Send sample rate updates back on a channel. - let sample_rate_handler = move || { - let mut rate: f64 = 0.0; - let mut data_size = mem::size_of::() as u32; - - let result = unsafe { - AudioObjectGetPropertyData( - audio_device_id, - NonNull::from(&sample_rate_address), - 0, - null(), - NonNull::from(&mut data_size), - NonNull::from(&mut rate).cast(), - ) - }; - send.send(coreaudio::Error::from_os_status(result).map(|_| rate)) - .ok(); - }; - - let listener = AudioObjectPropertyListener::new( - audio_device_id, - sample_rate_address, - sample_rate_handler, - )?; + // Register the listener before setting the property so we don't miss the notification. + let (sender, receiver) = channel::(); + let mut listener = RateListener::new(audio_device_id, Some(sender)); + listener.register()?; - // Finally, set the sample rate. + // Set the nominal sample rate. property_address.mSelector = kAudioDevicePropertyNominalSampleRate; - // Set the nominal sample rate using a single f64 as required by CoreAudio. let rate = sample_rate as f64; let data_size = mem::size_of::() as u32; let status = unsafe { @@ -166,40 +176,38 @@ fn set_sample_rate( // Wait for the reported_rate to change. // - // This should not take longer than a few ms, but we timeout after 1 sec just in case. - // We loop over potentially several events from the channel to ensure - // that we catch the expected change in sample rate. - let mut timeout = Duration::from_secs(1); + // This should not take longer than a few ms. Use the caller's timeout if provided, + // otherwise default to 1 second. We loop over potentially several events from the + // channel to ensure that we catch the expected change in sample rate. + let mut remaining = timeout.unwrap_or(Duration::from_secs(1)); let start = Instant::now(); - loop { - match recv.recv_timeout(timeout) { - Err(err) => { - let description = match err { - RecvTimeoutError::Disconnected => { - "sample rate listener channel disconnected unexpectedly" - } - RecvTimeoutError::Timeout => { - "timeout waiting for sample rate update for device" - } + match receiver.recv_timeout(remaining) { + Ok(reported_rate) => { + if (reported_rate - target_sample_rate as f64).abs() < 1.0 { + break; } - .to_string(); - return Err(BackendSpecificError { description }.into()); } - Ok(Ok(reported_sample_rate)) => { - if reported_sample_rate == target_sample_rate as f64 { - break; + Err(RecvTimeoutError::Timeout) => { + return Err(BackendSpecificError { + description: "timeout waiting for sample rate update for device" + .to_string(), } + .into()); } - Ok(Err(_)) => { - // TODO: should we consider collecting this error? + Err(RecvTimeoutError::Disconnected) => { + return Err(BackendSpecificError { + description: "sample rate listener channel disconnected unexpectedly" + .to_string(), + } + .into()); } - }; - timeout = timeout + } + remaining = remaining .checked_sub(start.elapsed()) .unwrap_or(Duration::ZERO); } - listener.remove()?; + // listener dropped here; its Drop impl calls unregister() automatically. } Ok(()) } @@ -738,7 +746,7 @@ impl Device { sample_format: SampleFormat, mut data_callback: D, error_callback: E, - _timeout: Option, + timeout: Option, ) -> Result where D: FnMut(&Data, &InputCallbackInfo) + Send + 'static, @@ -748,8 +756,19 @@ impl Device { let scope = Scope::Output; let element = Element::Input; - // Potentially change the device sample rate to match the config. - set_sample_rate(self.audio_device_id, config.sample_rate)?; + // Set the physical stream format (bit depth + sample rate) on the hardware device. + // This avoids unnecessary format conversions, which is especially important on aggregate + // devices. Falls back to sample-rate-only if no matching physical format is available. + if set_physical_format( + self.audio_device_id, + config.sample_rate, + config.channels, + sample_format, + ) + .is_err() + { + set_sample_rate(self.audio_device_id, config.sample_rate, timeout)?; + } let mut loopback_aggregate: Option = None; let mut audio_unit = if self.supports_input() { @@ -842,12 +861,26 @@ impl Device { sample_format: SampleFormat, mut data_callback: D, error_callback: E, - _timeout: Option, + timeout: Option, ) -> Result where D: FnMut(&mut Data, &OutputCallbackInfo) + Send + 'static, E: FnMut(StreamError) + Send + 'static, { + // Best-effort: set the physical stream format (bit depth + sample rate) on the hardware. + // This avoids unnecessary conversions, especially on aggregate devices. Not an error if + // it fails — the AudioUnit will handle format conversion as before. + if set_physical_format( + self.audio_device_id, + config.sample_rate, + config.channels, + sample_format, + ) + .is_err() + { + set_sample_rate(self.audio_device_id, config.sample_rate, timeout)?; + } + let mut audio_unit = audio_unit_from_device(self, false)?; // The scope and element for working with a device's output stream. diff --git a/src/host/coreaudio/macos/mod.rs b/src/host/coreaudio/macos/mod.rs index b48bc8c6e..dfc81f435 100644 --- a/src/host/coreaudio/macos/mod.rs +++ b/src/host/coreaudio/macos/mod.rs @@ -12,8 +12,8 @@ use std::sync::{mpsc, Arc, Mutex, Weak}; pub use self::enumerate::{default_input_device, default_output_device, Devices}; use objc2_core_audio::{ - kAudioDevicePropertyDeviceIsAlive, kAudioObjectPropertyElementMain, - kAudioObjectPropertyScopeGlobal, AudioObjectPropertyAddress, + kAudioDevicePropertyDeviceIsAlive, kAudioDevicePropertyNominalSampleRate, + kAudioObjectPropertyElementMain, kAudioObjectPropertyScopeGlobal, AudioObjectPropertyAddress, }; use property_listener::AudioObjectPropertyListener; @@ -103,29 +103,41 @@ impl DisconnectManager { error_callback: Arc>, ) -> Result { let (shutdown_tx, shutdown_rx) = mpsc::channel(); - let (disconnect_tx, disconnect_rx) = mpsc::channel(); + let (disconnect_tx, disconnect_rx) = mpsc::channel::(); let (ready_tx, ready_rx) = mpsc::channel(); - // Spawn dedicated thread to own the AudioObjectPropertyListener - let disconnect_tx_clone = disconnect_tx.clone(); + // Spawn a dedicated thread to own both listeners. CoreAudio requires that + // AudioObjectPropertyListeners are added and removed on the same thread. + let disconnect_tx_alive = disconnect_tx.clone(); + let disconnect_tx_rate = disconnect_tx; std::thread::spawn(move || { - let property_address = AudioObjectPropertyAddress { + let alive_address = AudioObjectPropertyAddress { mSelector: kAudioDevicePropertyDeviceIsAlive, mScope: kAudioObjectPropertyScopeGlobal, mElement: kAudioObjectPropertyElementMain, }; + let alive_listener = + AudioObjectPropertyListener::new(device_id, alive_address, move || { + let _ = disconnect_tx_alive.send(crate::StreamError::DeviceNotAvailable); + }); - // Create the listener on this dedicated thread - let disconnect_fn = move || { - let _ = disconnect_tx_clone.send(()); + let rate_address = AudioObjectPropertyAddress { + mSelector: kAudioDevicePropertyNominalSampleRate, + mScope: kAudioObjectPropertyScopeGlobal, + mElement: kAudioObjectPropertyElementMain, }; - match AudioObjectPropertyListener::new(device_id, property_address, disconnect_fn) { - Ok(_listener) => { + let rate_listener = + AudioObjectPropertyListener::new(device_id, rate_address, move || { + let _ = disconnect_tx_rate.send(crate::StreamError::StreamInvalidated); + }); + + match (alive_listener, rate_listener) { + (Ok(_alive), Ok(_rate)) => { let _ = ready_tx.send(Ok(())); - // Drop the listener on this thread after receiving a shutdown signal + // Block until the stream is dropped; listeners are removed on drop. let _ = shutdown_rx.recv(); } - Err(e) => { + (Err(e), _) | (_, Err(e)) => { let _ = ready_tx.send(Err(e)); } } @@ -144,21 +156,13 @@ impl DisconnectManager { let stream_weak_clone = stream_weak.clone(); let error_callback_clone = error_callback.clone(); std::thread::spawn(move || { - while disconnect_rx.recv().is_ok() { - // Check if stream still exists + while let Ok(err) = disconnect_rx.recv() { if let Some(stream_arc) = stream_weak_clone.upgrade() { - // First, try to pause the stream to stop playback if let Ok(mut stream_inner) = stream_arc.try_lock() { let _ = stream_inner.pause(); } - - // Always try to notify about device disconnection - invoke_error_callback( - &error_callback_clone, - crate::StreamError::DeviceNotAvailable, - ); + invoke_error_callback(&error_callback_clone, err); } else { - // Stream is gone, exit the handler thread break; } } diff --git a/src/host/coreaudio/macos/property_listener.rs b/src/host/coreaudio/macos/property_listener.rs index f65f513f1..ffb638c8f 100644 --- a/src/host/coreaudio/macos/property_listener.rs +++ b/src/host/coreaudio/macos/property_listener.rs @@ -11,7 +11,7 @@ use crate::BuildStreamError; /// A double-indirection to be able to pass a closure (a fat pointer) /// via a single c_void. -struct PropertyListenerCallbackWrapper(Box); +struct PropertyListenerCallbackWrapper(Box); /// Maintain an audio object property listener. /// The listener will be removed when this type is dropped. @@ -24,7 +24,7 @@ pub struct AudioObjectPropertyListener { impl AudioObjectPropertyListener { /// Attach the provided callback as a audio object property listener. - pub fn new( + pub fn new( audio_object_id: AudioObjectID, property_address: AudioObjectPropertyAddress, callback: F, @@ -49,6 +49,7 @@ impl AudioObjectPropertyListener { /// Explicitly remove the property listener. /// Use this method if you need to explicitly handle failure to remove /// the property listener. + #[allow(dead_code)] pub fn remove(mut self) -> Result<(), BuildStreamError> { self.remove_inner() }