diff --git a/modules/audio_device/audio_engine_device.h b/modules/audio_device/audio_engine_device.h index d4d43b3b2a4..33db86c5717 100644 --- a/modules/audio_device/audio_engine_device.h +++ b/modules/audio_device/audio_engine_device.h @@ -17,7 +17,6 @@ #ifndef SDK_OBJC_NATIVE_SRC_AUDIO_AUDIO_DEVICE_AUDIOENGINE_H_ #define SDK_OBJC_NATIVE_SRC_AUDIO_AUDIO_DEVICE_AUDIOENGINE_H_ -#include #include #include "api/scoped_refptr.h" @@ -137,6 +136,7 @@ class AudioEngineDevice : public AudioDeviceModule, public AudioSessionObserver bool output_available = true; bool input_available = true; + bool output_running_persistent_mode = false; bool input_enabled_persistent_mode = false; bool input_muted = true; @@ -161,6 +161,7 @@ class AudioEngineDevice : public AudioDeviceModule, public AudioSessionObserver return input_enabled == rhs.input_enabled && input_running == rhs.input_running && output_enabled == rhs.output_enabled && output_running == rhs.output_running && input_available == rhs.input_available && output_available == rhs.output_available && + output_running_persistent_mode == rhs.output_running_persistent_mode && input_enabled_persistent_mode == rhs.input_enabled_persistent_mode && input_muted == rhs.input_muted && is_interrupted == rhs.is_interrupted && render_mode == rhs.render_mode && mute_mode == rhs.mute_mode && @@ -178,9 +179,8 @@ class AudioEngineDevice : public AudioDeviceModule, public AudioSessionObserver // AUDIO STATE LOGIC // // Device Mode: - // - Output follows input only when voice_processing_enabled=true (for AEC) + // - Output follows input to keep AVAudioEngine IO active for capture. // - Input respects mute mode restrictions (RestartEngine + input_muted) - // - Independent operation when voice processing is disabled // // Manual Mode: // - Bidirectional coupling: if ANY component is enabled/running, BOTH are considered @@ -192,10 +192,11 @@ class AudioEngineDevice : public AudioDeviceModule, public AudioSessionObserver bool IsOutputEnabled() const { if (!output_available) return false; + if (output_running_persistent_mode) return true; switch (render_mode) { case RenderMode::Device: - return voice_processing_enabled ? (IsInputEnabled() || output_enabled) : output_enabled; + return IsInputEnabled() || output_enabled; case RenderMode::Manual: return output_enabled || input_enabled || input_enabled_persistent_mode; } @@ -203,10 +204,11 @@ class AudioEngineDevice : public AudioDeviceModule, public AudioSessionObserver bool IsOutputRunning() const { if (!output_available) return false; + if (output_running_persistent_mode) return true; switch (render_mode) { case RenderMode::Device: - return voice_processing_enabled ? (IsInputRunning() || output_running) : output_running; + return IsInputRunning() || output_running; case RenderMode::Manual: return output_running || input_running; } @@ -220,7 +222,8 @@ class AudioEngineDevice : public AudioDeviceModule, public AudioSessionObserver return !(mute_mode == MuteMode::RestartEngine && input_muted) && (input_enabled || input_enabled_persistent_mode); case RenderMode::Manual: - return input_enabled || input_enabled_persistent_mode || output_enabled; + return (input_enabled || input_enabled_persistent_mode) || + (output_enabled || output_running_persistent_mode); } } @@ -231,7 +234,7 @@ class AudioEngineDevice : public AudioDeviceModule, public AudioSessionObserver case RenderMode::Device: return !(mute_mode == MuteMode::RestartEngine && input_muted) && input_running; case RenderMode::Manual: - return input_running || output_running; + return input_running || (output_running || output_running_persistent_mode); } } @@ -367,6 +370,9 @@ class AudioEngineDevice : public AudioDeviceModule, public AudioSessionObserver int32_t SetDuckingLevel(AudioDuckingLevel level); int32_t DuckingLevel(AudioDuckingLevel* level); + int32_t SetOutputRunningPersistentMode(bool enable); + int32_t OutputRunningPersistentMode(bool* enabled); + int32_t SetInitRecordingPersistentMode(bool enable); int32_t InitRecordingPersistentMode(bool* enabled); @@ -429,11 +435,7 @@ class AudioEngineDevice : public AudioDeviceModule, public AudioSessionObserver bool DidUpdateMuteMode() const { return prev.mute_mode != next.mute_mode; } - bool IsEngineRestartRequired() const { - return DidUpdateAudioGraph() || - // Voice processing enable state updates - DidUpdateVoiceProcessingEnabled(); - } + bool IsEngineRestartRequired() const { return DidUpdateAudioGraph(); } bool IsEngineRecreateRequired() const { // Device id specified @@ -448,7 +450,11 @@ class AudioEngineDevice : public AudioDeviceModule, public AudioSessionObserver bool special_case = (prev.IsOutputEnabled() && next.IsOutputEnabled()) && (prev.IsInputEnabled() && !next.IsInputEnabled()); - return device || default_device || special_case; + // Toggling voice processing requires a full engine recreate to ensure + // a clean audio hardware state. + bool voice_processing = DidUpdateVoiceProcessingEnabled(); + + return device || default_device || special_case || voice_processing; } bool DidEnableManualRenderingMode() const { diff --git a/modules/audio_device/audio_engine_device.mm b/modules/audio_device/audio_engine_device.mm index 8da352ad47f..793cdd39274 100644 --- a/modules/audio_device/audio_engine_device.mm +++ b/modules/audio_device/audio_engine_device.mm @@ -1277,6 +1277,32 @@ AVAudioVoiceProcessingOtherAudioDuckingLevel ToAVDuckingLevel( return 0; } +int32_t AudioEngineDevice::SetOutputRunningPersistentMode(bool enable) { + RTC_DCHECK_RUN_ON(thread_); + LOGI() << "SetOutputRunningPersistentMode: " << enable; + + int32_t result = ModifyEngineState([enable](EngineState state) -> EngineState { + state.output_running_persistent_mode = enable; + return state; + }); + + return result; +} + +int32_t AudioEngineDevice::OutputRunningPersistentMode(bool* enabled) { + LOGI() << "OutputRunningPersistentMode"; + RTC_DCHECK_RUN_ON(thread_); + + if (enabled == nullptr) { + return -1; + } + + *enabled = engine_state_.output_running_persistent_mode; + LOGI() << "OutputRunningPersistentMode value: " << *enabled; + + return 0; +} + int32_t AudioEngineDevice::SetInitRecordingPersistentMode(bool enable) { RTC_DCHECK_RUN_ON(thread_); LOGI() << "SetInitRecordingPersistentMode: " << enable; @@ -1759,6 +1785,86 @@ AVAudioVoiceProcessingOtherAudioDuckingLevel ToAVDuckingLevel( RTC_DCHECK_RUN_ON(thread_); RTC_DCHECK(engine_manual_input_ == nullptr); + // --- Diagnostic: state transition summary --- + auto mute_mode_str = [](MuteMode m) -> const char* { + switch (m) { + case MuteMode::VoiceProcessing: return "VP"; + case MuteMode::RestartEngine: return "Restart"; + case MuteMode::InputMixer: return "Mixer"; + } + return "?"; + }; + + auto render_mode_str = [](RenderMode m) -> const char* { + switch (m) { + case RenderMode::Device: return "Device"; + case RenderMode::Manual: return "Manual"; + } + return "?"; + }; + + auto log_engine_state = [&](const char* label, const EngineState& s) { + LOGI() << label << ": " + << "in=" << s.input_enabled << "/" << s.input_running + << " out=" << s.output_enabled << "/" << s.output_running + << " persistent=" << s.input_enabled_persistent_mode + << " muted=" << s.input_muted + << " vp=" << s.voice_processing_enabled + << " vpBypass=" << s.voice_processing_bypassed + << " agc=" << s.voice_processing_agc_enabled + << " mute_mode=" << mute_mode_str(s.mute_mode) + << " render=" << render_mode_str(s.render_mode) + << " interrupted=" << s.is_interrupted + << " in_avail=" << s.input_available + << " out_avail=" << s.output_available + << " inDev=" << s.input_device_id + << " outDev=" << s.output_device_id + << " defInUpd=" << s.default_input_device_update_count + << " defOutUpd=" << s.default_output_device_update_count + << " | IsInEnabled=" << s.IsInputEnabled() + << " IsOutEnabled=" << s.IsOutputEnabled() + << " IsInRunning=" << s.IsInputRunning() + << " IsOutRunning=" << s.IsOutputRunning(); + }; + + log_engine_state(" [State] prev", state.prev); + log_engine_state(" [State] next", state.next); + + LOGI() << " [State] decisions: " + << "restart=" << state.IsEngineRestartRequired() + << " recreate=" << state.IsEngineRecreateRequired() + << " graphChanged=" << state.DidUpdateAudioGraph() + << " vpChanged=" << state.DidUpdateVoiceProcessingEnabled() + << " muteChanged=" << state.DidUpdateMuteMode() + << " interrupted=" << state.DidBeginInterruption() + << " uninterrupted=" << state.DidEndInterruption() + << " inDevChanged=" << state.DidUpdateInputDevice() + << " outDevChanged=" << state.DidUpdateOutputDevice() + << " defInDevChanged=" << state.DidUpdateDefaultInputDevice() + << " defOutDevChanged=" << state.DidUpdateDefaultOutputDevice(); + + // Log actual hardware state if engine exists + if (engine_device_ != nil) { + LOGI() << " [HW] engine: running=" << engine_device_.running + << " attachedNodes=" << engine_device_.attachedNodes.count; + if (state.prev.IsInputEnabled() || state.next.IsInputEnabled()) { + @try { + AVAudioInputNode* inode = engine_device_.inputNode; + LOGI() << " [HW] inputNode: vpEnabled=" << inode.voiceProcessingEnabled + << " vpMuted=" << inode.voiceProcessingInputMuted + << " vpBypassed=" << inode.voiceProcessingBypassed + << " vpAGC=" << inode.voiceProcessingAGCEnabled; + } @catch (NSException*) { + LOGI() << " [HW] inputNode: "; + } + } + if (input_mixer_node_ != nil) { + LOGI() << " [HW] mixerNode: volume=" << input_mixer_node_.outputVolume; + } + } else { + LOGI() << " [HW] engine: nil"; + } + std::vector> rollback_actions; auto rollback = [&](int32_t result) { @@ -1791,24 +1897,25 @@ AVAudioVoiceProcessingOtherAudioDuckingLevel ToAVDuckingLevel( (!state.next.IsAnyRunning() || state.IsEngineRestartRequired() || state.DidBeginInterruption() || state.IsEngineRecreateRequired())) { LOGI() << "Stopping AVAudioEngine..."; - RTC_DCHECK(engine_device_ != nil); if (configuration_observer_ != nullptr) { NSNotificationCenter* center = [NSNotificationCenter defaultCenter]; [center removeObserver:(__bridge_transfer id)configuration_observer_ name:AVAudioEngineConfigurationChangeNotification - object:engine_device_]; + object:nil]; configuration_observer_ = nil; } - [engine_device_ stop]; + if (engine_device_ != nil) { + [engine_device_ stop]; - if (observer_ != nullptr) { - int32_t result = observer_->OnEngineDidStop(engine_device_, state.next.IsOutputEnabled(), - state.next.IsInputEnabled()); - if (result != 0) { - LOGE() << "Call to OnEngineDidStop returned error: " << result; - return rollback(result); + if (observer_ != nullptr) { + int32_t result = observer_->OnEngineDidStop(engine_device_, state.next.IsOutputEnabled(), + state.next.IsInputEnabled()); + if (result != 0) { + LOGE() << "Call to OnEngineDidStop returned error: " << result; + return rollback(result); + } } } } @@ -1816,7 +1923,8 @@ AVAudioVoiceProcessingOtherAudioDuckingLevel ToAVDuckingLevel( // -------------------------------------------------------------------------------------------- // Step: Stop playout buffer // - if (!state.next.IsOutputEnabled() && audio_device_buffer_->IsPlaying()) { + if ((!state.next.IsOutputEnabled() || state.IsEngineRecreateRequired()) && + audio_device_buffer_->IsPlaying()) { LOGI() << "Stopping Playout buffer..."; if (engine_device_ != nullptr) { // Rendering must be stopped first. @@ -1828,7 +1936,8 @@ AVAudioVoiceProcessingOtherAudioDuckingLevel ToAVDuckingLevel( // -------------------------------------------------------------------------------------------- // Step: Stop recording buffer // - if (!state.next.IsInputEnabled() && audio_device_buffer_->IsRecording()) { + if ((!state.next.IsInputEnabled() || state.IsEngineRecreateRequired()) && + audio_device_buffer_->IsRecording()) { LOGI() << "Stopping Record buffer..."; if (engine_device_ != nullptr) { // Rendering must be stopped first. @@ -1842,13 +1951,26 @@ AVAudioVoiceProcessingOtherAudioDuckingLevel ToAVDuckingLevel( // if (state.IsEngineRecreateRequired()) { LOGI() << "Recreate required, releasing AVAudioEngine..."; - if (observer_ != nullptr) { + if (observer_ != nullptr && engine_device_ != nil) { int32_t result = observer_->OnEngineWillRelease(engine_device_); if (result != 0) { LOGE() << "Call to OnEngineWillRelease returned error: " << result; return rollback(result); } } + + if (state.DidUpdateVoiceProcessingEnabled() && engine_device_ != nil) { + AVAudioInputNode* input_node = engine_device_.inputNode; + AVAudioOutputNode* output_node = engine_device_.outputNode; + + if (input_node != nil && input_node.audioUnit != nullptr) { + AudioOutputUnitStop(input_node.audioUnit); + } + if (output_node != nil && output_node.audioUnit != nullptr) { + AudioOutputUnitStop(output_node.audioUnit); + } + } + engine_device_ = nil; } @@ -1923,14 +2045,8 @@ AVAudioVoiceProcessingOtherAudioDuckingLevel ToAVDuckingLevel( // -------------------------------------------------------------------------------------------- // Step: Configure Voice-Processing I/O // - // Use cached state to avoid accessing inputNode() when voice processing is - // disabled – AVAudioInputNode can crash (EXC_BAD_ACCESS) when the audio - // hardware is unavailable, e.g. Mac Catalyst in background. - // After engine recreate, a fresh AVAudioEngine defaults VP to disabled. - bool effective_prev_vp = - state.IsEngineRecreateRequired() ? false : state.prev.voice_processing_enabled; if (state.next.IsInputEnabled() && - effective_prev_vp != state.next.voice_processing_enabled) { + inputNode().voiceProcessingEnabled != state.next.voice_processing_enabled) { #if TARGET_OS_SIMULATOR LOGI() << "setVoiceProcessingEnabled (input): " << (state.next.voice_processing_enabled ? "YES" : "NO") << " (Ignored on Simulator)"; @@ -1947,7 +2063,7 @@ AVAudioVoiceProcessingOtherAudioDuckingLevel ToAVDuckingLevel( LOGI() << "setVoiceProcessingEnabled (input) result: " << set_vp_result ? "YES" : "NO"; #endif - if (state.next.voice_processing_enabled) { + if (inputNode().voiceProcessingEnabled) { // Always unmute vp if restart mute mode. if (state.next.mute_mode == MuteMode::RestartEngine && inputNode().voiceProcessingInputMuted) { @@ -2356,24 +2472,27 @@ AVAudioVoiceProcessingOtherAudioDuckingLevel ToAVDuckingLevel( } // -------------------------------------------------------------------------------------------- - // Step: Run-time mute toggling if vp mode. + // Step: Run-time mute toggling (voice processing). + // VP mute should be on ONLY when VoiceProcessing mode is active AND input is muted. // - if (state.next.mute_mode == MuteMode::VoiceProcessing && state.next.IsInputEnabled() && - state.next.voice_processing_enabled && - inputNode().voiceProcessingInputMuted != state.next.input_muted) { - LOGI() << "Update mute (voice processing) runtime update: " << state.next.input_muted; - inputNode().voiceProcessingInputMuted = state.next.input_muted; + if (state.next.IsInputEnabled() && state.next.voice_processing_enabled) { + bool should_vp_mute = + (state.next.mute_mode == MuteMode::VoiceProcessing) && state.next.input_muted; + if (inputNode().voiceProcessingInputMuted != should_vp_mute) { + LOGI() << "Update mute (voice processing): " << should_vp_mute; + inputNode().voiceProcessingInputMuted = should_vp_mute; + } } // -------------------------------------------------------------------------------------------- - // Step: Run-time mute toggling if mixer mute mode. + // Step: Run-time mute toggling (input mixer). + // Mixer volume should be 0 ONLY when InputMixer mode is active AND input is muted. // - if (state.next.mute_mode == MuteMode::InputMixer && state.next.IsInputEnabled() && - input_mixer_node_ != nil) { - // Only update if the volume has changed. - float mixer_volume = state.next.input_muted ? 0.0f : 1.0f; + if (state.next.IsInputEnabled() && input_mixer_node_ != nil) { + float mixer_volume = + (state.next.mute_mode == MuteMode::InputMixer && state.next.input_muted) ? 0.0f : 1.0f; if (input_mixer_node_.outputVolume != mixer_volume) { - LOGI() << "Update mute (input mixer) runtime update: " << state.next.input_muted; + LOGI() << "Update mute (input mixer): " << (mixer_volume == 0.0f); input_mixer_node_.outputVolume = mixer_volume; } } @@ -2424,19 +2543,26 @@ AVAudioVoiceProcessingOtherAudioDuckingLevel ToAVDuckingLevel( if (state.next.IsAnyEnabled() && (!state.prev.IsAnyEnabled() || state.IsEngineRecreateRequired())) { if (state.next.IsInputEnabled()) { - uint32_t input_device_id = state.next.input_device_id; - if (input_device_id == kAudioObjectUnknown) { + uint32_t requested_input_device_id = state.next.input_device_id; + + AudioUnit input_unit = inputNode().audioUnit; + + if (requested_input_device_id == kAudioObjectUnknown) { + // For default routing, avoid forcing kAudioOutputUnitProperty_CurrentDevice. On macOS this + // can fail during VoiceProcessingIO reconfiguration and the engine already follows the + // system default route. LOGI() << "Using default input device"; } else { - auto input_device_name = mac_audio_utils::GetDeviceName(input_device_id); + auto input_device_name = mac_audio_utils::GetDeviceName(requested_input_device_id); LOGI() << "Setting input device: " << input_device_name.value_or("Unknown") << " (" - << input_device_id << ")"; - AudioUnit inputUnit = inputNode().audioUnit; - OSStatus err = AudioUnitSetProperty(inputUnit, kAudioOutputUnitProperty_CurrentDevice, - kAudioUnitScope_Global, 1, &input_device_id, - sizeof(input_device_id)); - if (err != noErr) { - LOGE() << "Failed to set input device: " << input_device_id << ", error: " << err; + << requested_input_device_id << ")"; + + OSStatus set_input_err = AudioUnitSetProperty( + input_unit, kAudioOutputUnitProperty_CurrentDevice, kAudioUnitScope_Global, 1, + &requested_input_device_id, sizeof(requested_input_device_id)); + if (set_input_err != noErr) { + LOGE() << "Failed to set input device: requested=" << requested_input_device_id + << ", error: " << set_input_err; return rollback(kAudioEngineRecordingDeviceNotAvailableError); } } @@ -2627,6 +2753,31 @@ AVAudioVoiceProcessingOtherAudioDuckingLevel ToAVDuckingLevel( engine_device_ = nil; } + // --- Diagnostic: final state after apply --- + if (engine_device_ != nil) { + LOGI() << " [Post] engine: running=" << engine_device_.running; + if (state.next.IsInputEnabled()) { + @try { + AVAudioInputNode* inode = engine_device_.inputNode; + LOGI() << " [Post] inputNode: vpEnabled=" << inode.voiceProcessingEnabled + << " vpMuted=" << inode.voiceProcessingInputMuted + << " vpBypassed=" << inode.voiceProcessingBypassed + << " vpAGC=" << inode.voiceProcessingAGCEnabled; + } @catch (NSException*) { + LOGI() << " [Post] inputNode: "; + } + } + if (input_mixer_node_ != nil) { + LOGI() << " [Post] mixerNode: volume=" << input_mixer_node_.outputVolume; + } + if (audio_device_buffer_ != nullptr) { + LOGI() << " [Post] buffer: playing=" << audio_device_buffer_->IsPlaying() + << " recording=" << audio_device_buffer_->IsRecording(); + } + } else { + LOGI() << " [Post] engine: nil"; + } + return 0; } diff --git a/sdk/objc/api/peerconnection/RTCAudioDeviceModule.h b/sdk/objc/api/peerconnection/RTCAudioDeviceModule.h index 062bdc8ec65..013b0e7a827 100644 --- a/sdk/objc/api/peerconnection/RTCAudioDeviceModule.h +++ b/sdk/objc/api/peerconnection/RTCAudioDeviceModule.h @@ -173,6 +173,11 @@ RTC_OBJC_EXPORT BOOL recordingAlwaysPreparedMode; - (NSInteger)setRecordingAlwaysPreparedMode:(BOOL)enabled; +/// Forces effective playout to remain enabled/running (availability still wins). +/// Backed by `AudioEngineDevice::EngineState::output_running_persistent_mode`. +@property(nonatomic, readonly, getter=isOutputAlwaysRunningMode) BOOL outputAlwaysRunningMode; +- (NSInteger)setOutputAlwaysRunningMode:(BOOL)enabled; + @property(nonatomic, weak, nullable) id observer; // Manual rendering. diff --git a/sdk/objc/api/peerconnection/RTCAudioDeviceModule.mm b/sdk/objc/api/peerconnection/RTCAudioDeviceModule.mm index 9525a2af521..1bde61987ac 100644 --- a/sdk/objc/api/peerconnection/RTCAudioDeviceModule.mm +++ b/sdk/objc/api/peerconnection/RTCAudioDeviceModule.mm @@ -453,6 +453,24 @@ - (NSInteger)setRecordingAlwaysPreparedMode:(BOOL)enabled { [module, enabled] { return module->SetInitRecordingPersistentMode(enabled); }); } +- (BOOL)isOutputAlwaysRunningMode { + webrtc::AudioEngineDevice* module = dynamic_cast(_native.get()); + if (module == nullptr) return NO; + + return _workerThread->BlockingCall([module] { + bool value = false; + return module->OutputRunningPersistentMode(&value) == 0 ? value : NO; + }); +} + +- (NSInteger)setOutputAlwaysRunningMode:(BOOL)enabled { + webrtc::AudioEngineDevice* module = dynamic_cast(_native.get()); + if (module == nullptr) return -1; + + return _workerThread->BlockingCall( + [module, enabled] { return module->SetOutputRunningPersistentMode(enabled); }); +} + - (BOOL)isManualRenderingMode { webrtc::AudioEngineDevice *module = dynamic_cast(_native.get()); if (module == nullptr) return NO;