Add ExternalAudioSource for publishing app audio as an independent track

Sources/LiveKit/Audio/AudioDeviceModuleDelegateAdapter.swift

@@ -19,7 +19,7 @@
 internal import LiveKitWebRTC
 
 // Invoked on WebRTC's worker thread, do not block.
 class AudioDeviceModuleDelegateAdapter: NSObject, LKRTCAudioDeviceModuleDelegate, Loggable {
     weak var audioManager: AudioManager?
 
     func audioDeviceModule(_: LKRTCAudioDeviceModule, didReceiveSpeechActivityEvent speechActivityEvent: LKRTCSpeechActivityEvent) {
@@ -40,7 +40,10 @@
         return entryPoint?.engineDidCreate(engine) ?? 0
     }
 
-    func audioDeviceModule(_: LKRTCAudioDeviceModule, willEnableEngine engine: AVAudioEngine, isPlayoutEnabled: Bool, isRecordingEnabled: Bool) -> Int {
+    func audioDeviceModule(_: LKRTCAudioDeviceModule, willEnableEngine engine: AVAudioEngine, isPlayoutEnabled: Bool, isRecordingEnabled: Bool, isVoiceProcessingEnabled _: Bool) -> Int {
+        // isVoiceProcessingEnabled is new in the ADM delegate (webrtc-sdk
+        // PR 275). Ignored for now, exposing it through AudioEngineObserver
+        // is part of the framework version bump, not this feature.
         guard let audioManager else { return 0 }
         let entryPoint = audioManager.buildEngineObserverChain()
         return entryPoint?.engineWillEnable(engine, isPlayoutEnabled: isPlayoutEnabled, isRecordingEnabled: isRecordingEnabled) ?? 0

Sources/LiveKit/Broadcast/BroadcastScreenCapturer.swift

@@ -25,8 +25,14 @@ import UIKit
 internal import LiveKitWebRTC
 
 class BroadcastScreenCapturer: BufferCapturer, @unchecked Sendable {
+    /// Destination for captured app audio; falls back to the mic mixer.
+    weak var appAudioSink: AppAudioSink?
+
     private let appAudio: Bool
+    private let appAudioPublishMode: AppAudioPublishMode
     private var receiver: BroadcastReceiver?
+    // Only written from the incoming-samples loop.
+    private var didWarnMissingAppAudioSink = false
 
     override func startCapture() async throws -> Bool {
         let didStart = try await super.startCapture()
@@ -67,7 +73,7 @@ class BroadcastScreenCapturer: BufferCapturer, @unchecked Sendable {
                 for try await sample in receiver.incomingSamples {
                     switch sample {
                     case let .image(buffer, rotation): capture(buffer, rotation: rotation)
-                    case let .audio(buffer): AudioManager.shared.mixer.capture(appAudio: buffer)
+                    case let .audio(buffer): captureAppAudio(buffer)
                     }
                 }
                 log("Broadcast receiver closed", .debug)
@@ -88,8 +94,26 @@ class BroadcastScreenCapturer: BufferCapturer, @unchecked Sendable {
         return true
     }
 
+    private func captureAppAudio(_ buffer: AVAudioPCMBuffer) {
+        if let appAudioSink {
+            appAudioSink.capture(appAudio: buffer)
+        } else if appAudioPublishMode == .separateTrack {
+            // The separate app-audio track's sink is gone, e.g. the audio
+            // track failed to publish or was unpublished mid-share. Drop the
+            // buffer instead of silently mixing app audio back into the
+            // microphone track.
+            if !didWarnMissingAppAudioSink {
+                didWarnMissingAppAudioSink = true
+                log("App audio configured as a separate track but no sink is attached, dropping app audio", .warning)
+            }
+        } else {
+            AudioManager.shared.mixer.capture(appAudio: buffer)
+        }
+    }
+
     init(delegate: LKRTCVideoCapturerDelegate, options: ScreenShareCaptureOptions) {
         appAudio = options.appAudio
+        appAudioPublishMode = options.appAudioPublishMode
         super.init(delegate: delegate, options: BufferCaptureOptions(from: options))
     }
 }

Sources/LiveKit/Core/RTC.swift

@@ -101,6 +101,12 @@
         DispatchQueue.liveKitWebRTC.sync { peerConnectionFactory.audioSource(with: constraints) }
     }
 
+    static func createExternalAudioSource(sampleRate: Int, channels: Int, queueSizeMs: Int) -> LKRTCExternalAudioSource? {
+        DispatchQueue.liveKitWebRTC.sync { peerConnectionFactory.externalAudioSource(withSampleRate: Int32(sampleRate),
+                                                                                     channels: UInt(channels),
+                                                                                     queueSizeMs: Int32(queueSizeMs)) }
+    }
+
     static func createAudioTrack(source: LKRTCAudioSource) -> LKRTCAudioTrack {
         DispatchQueue.liveKitWebRTC.sync { peerConnectionFactory.audioTrack(with: source,
                                                                             trackId: UUID().uuidString) }

Sources/LiveKit/Participant/LocalParticipant.swift

@@ -336,7 +336,9 @@ extension LocalParticipant {
 
         for mediaTrack in mediaTracks {
             // Don't re-publish muted tracks
-            if mediaTrack.isMuted { continue }
+            if mediaTrack.isMuted {
+                continue
+            }
             try await _publish(track: mediaTrack, options: mediaTrack.publishOptions)
         }
     }
@@ -405,6 +407,11 @@ public extension LocalParticipant {
                         try await publication.mute()
                     } else {
                         try await self.unpublish(publication: publication)
+                        // App audio published as a separate track follows the
+                        // screen-share video lifecycle.
+                        if source == .screenShareVideo {
+                            try await self.unpublishAppAudioTrackIfNeeded()
+                        }
                     }
                     return publication
                 }
@@ -423,6 +430,7 @@ public extension LocalParticipant {
 
                     let localTrack: LocalVideoTrack
                     let defaultOptions = room._state.roomOptions.defaultScreenShareCaptureOptions
+                    var screenShareOptions = defaultOptions
 
                     if defaultOptions.useBroadcastExtension {
                         if captureOptions != nil {
@@ -437,17 +445,51 @@ public extension LocalParticipant {
                         localTrack = LocalVideoTrack.createBroadcastScreenCapturerTrack(options: defaultOptions,
                                                                                         reportStatistics: room._state.roomOptions.reportRemoteTrackStatistics)
                     } else {
-                        let options = (captureOptions as? ScreenShareCaptureOptions) ?? defaultOptions
-                        localTrack = LocalVideoTrack.createInAppScreenShareTrack(options: options)
+                        screenShareOptions = (captureOptions as? ScreenShareCaptureOptions) ?? defaultOptions
+                        localTrack = LocalVideoTrack.createInAppScreenShareTrack(options: screenShareOptions)
                     }
-                    return try await self._publish(track: localTrack, options: publishOptions)
+                    // Prepared before publishing so the capturer routes app
+                    // audio to the independent track from the first buffer.
+                    let appAudioTrack = self.prepareAppAudioTrack(options: screenShareOptions, videoTrack: localTrack, room: room)
+                    let publication = try await self._publish(track: localTrack, options: publishOptions)
+                    if let appAudioTrack {
+                        do {
+                            try await self._publish(track: appAudioTrack, options: AudioPublishOptions(encoding: .presetMusicHighQualityStereo,
+                                                                                                       dtx: false,
+                                                                                                       red: false))
+                        } catch {
+                            // Screen share continues without app audio. The
+                            // capturer's sink goes away with the track, so
+                            // audio is dropped, not mixed into the mic.
+                            self.log("Failed to publish app audio track: \(error)", .error)
+                        }
+                    }
+                    return publication
                     #elseif os(macOS)
                     if #available(macOS 12.3, *) {
                         let mainDisplay = try await MacOSScreenCapturer.mainDisplaySource()
+                        let screenShareOptions = (captureOptions as? ScreenShareCaptureOptions) ?? room._state.roomOptions.defaultScreenShareCaptureOptions
                         let track = LocalVideoTrack.createMacOSScreenShareTrack(source: mainDisplay,
-                                                                                options: (captureOptions as? ScreenShareCaptureOptions) ?? room._state.roomOptions.defaultScreenShareCaptureOptions,
+                                                                                options: screenShareOptions,
                                                                                 reportStatistics: room._state.roomOptions.reportRemoteTrackStatistics)
-                        return try await self._publish(track: track, options: publishOptions)
+                        // Prepared before publishing so ScreenCaptureKit audio
+                        // routes to the independent track from the first buffer.
+                        let appAudioTrack = self.prepareAppAudioTrack(options: screenShareOptions, videoTrack: track, room: room)
+                        let publication = try await self._publish(track: track, options: publishOptions)
+                        if let appAudioTrack {
+                            do {
+                                try await self._publish(track: appAudioTrack, options: AudioPublishOptions(encoding: .presetMusicHighQualityStereo,
+                                                                                                           dtx: false,
+                                                                                                           red: false))
+                            } catch {
+                                // Screen share continues without app audio.
+                                // The capturer's sink goes away with the
+                                // track, so audio is dropped, not mixed into
+                                // the mic.
+                                self.log("Failed to publish app audio track: \(error)", .error)
+                            }
+                        }
+                        return publication
                     }
                     #endif
                 }
@@ -456,6 +498,51 @@ public extension LocalParticipant {
             return nil
         }
     }
+
+    /// Unpublishes the independent app-audio track that follows the
+    /// screen-share video lifecycle, if one is published.
+    func unpublishAppAudioTrackIfNeeded() async throws {
+        // All matching publications, not just the first: a missed cleanup
+        // must not leave stale tracks behind on the next stop.
+        let appAudioPublications = audioTracks.compactMap { $0 as? LocalTrackPublication }
+            .filter { $0.source == .screenShareAudio }
+        for publication in appAudioPublications {
+            try await unpublish(publication: publication)
+        }
+    }
+
+    /// Creates an independent app-audio track and routes the screen-share
+    /// capturer's audio to it, when the capture options request it.
+    private func prepareAppAudioTrack(options: ScreenShareCaptureOptions,
+                                      videoTrack: LocalVideoTrack,
+                                      room: Room) -> LocalAudioTrack?
+    {
+        guard options.appAudio, options.appAudioPublishMode == .separateTrack else { return nil }
+        do {
+            let source = try ExternalAudioSource()
+            // Only capturers with an app-audio path can feed the source. Bail
+            // instead of publishing a permanently silent track (e.g. iOS
+            // in-app capture, which has no audio handling).
+            #if os(iOS)
+            guard let capturer = videoTrack.capturer as? BroadcastScreenCapturer else {
+                log("App audio as a separate track requires the broadcast extension capturer, skipping app audio track", .warning)
+                return nil
+            }
+            capturer.appAudioSink = source
+            #elseif os(macOS)
+            guard #available(macOS 12.3, *), let capturer = videoTrack.capturer as? MacOSScreenCapturer else {
+                log("App audio as a separate track requires the ScreenCaptureKit capturer, skipping app audio track", .warning)
+                return nil
+            }
+            capturer.appAudioSink = source
+            #endif
+            return LocalAudioTrack.createTrack(externalSource: source,
+                                               reportStatistics: room._state.roomOptions.reportRemoteTrackStatistics)
+        } catch {
+            log("Failed to create app audio track: \(error)", .error)
+            return nil
+        }
+    }
 }
 
 // MARK: - Simulcast codecs
@@ -735,8 +822,10 @@ extension LocalParticipant {
 
             // At this point at least 1 audio frame should be generated to continue
             if let track = track as? LocalAudioTrack {
-                // Only wait for frames if audio engine is allowed to start
-                if AudioManager.shared.engineAvailability.isInputAvailable {
+                // Externally fed tracks bypass the ADM capture path the frame
+                // watcher observes, and frames only flow once the app pushes
+                // audio, so there is nothing to wait for here.
+                if track.externalSource == nil, AudioManager.shared.engineAvailability.isInputAvailable {
                     log("[Publish] Waiting for audio frame...")
                     try await track.startWaitingForFrames()
                 }

Sources/LiveKit/Track/Capturers/MacOSScreenCapturer.swift

@@ -41,6 +41,12 @@ public class MacOSScreenCapturer: VideoCapturer, @unchecked Sendable {
     /// The ``ScreenShareCaptureOptions`` used for this capturer.
     public let options: ScreenShareCaptureOptions
 
+    /// Destination for captured app audio; falls back to the mic mixer.
+    weak var appAudioSink: AppAudioSink?
+
+    // Only written from the SCStream callback queue.
+    private var didWarnMissingAppAudioSink = false
+
     struct State {
         // SCStream
         var scStream: SCStream?
@@ -239,7 +245,20 @@ extension MacOSScreenCapturer: SCStreamOutput {
 
         if case .audio = outputType {
             guard let pcm = sampleBuffer.toAVAudioPCMBuffer() else { return }
-            AudioManager.shared.mixer.capture(appAudio: pcm)
+            if let appAudioSink {
+                appAudioSink.capture(appAudio: pcm)
+            } else if options.appAudioPublishMode == .separateTrack {
+                // The separate app-audio track's sink is gone, e.g. the audio
+                // track failed to publish or was unpublished mid-share. Drop
+                // the buffer instead of silently mixing app audio back into
+                // the microphone track.
+                if !didWarnMissingAppAudioSink {
+                    didWarnMissingAppAudioSink = true
+                    log("App audio configured as a separate track but no sink is attached, dropping app audio", .warning)
+                }
+            } else {
+                AudioManager.shared.mixer.capture(appAudio: pcm)
+            }
         } else if case .screen = outputType {
             // Retrieve the array of metadata attachments from the sample buffer.
             guard let attachmentsArray = CMSampleBufferGetSampleAttachmentsArray(sampleBuffer,
@@ -261,7 +280,9 @@ extension MacOSScreenCapturer: SCStreamOutput {
             let newTimer = Task.detached(priority: .utility) { [weak self] in
                 while true {
                     try? await Task.sleep(nanoseconds: UInt64(1 * 1_000_000_000))
-                    if Task.isCancelled { break }
+                    if Task.isCancelled {
+                        break
+                    }
                     guard let self else { break }
                     try await _capturePreviousFrame()
                 }