diff --git a/Sources/LiveKit/Audio/AudioDeviceModuleDelegateAdapter.swift b/Sources/LiveKit/Audio/AudioDeviceModuleDelegateAdapter.swift index b1ce7fec7..c27f5ff2c 100644 --- a/Sources/LiveKit/Audio/AudioDeviceModuleDelegateAdapter.swift +++ b/Sources/LiveKit/Audio/AudioDeviceModuleDelegateAdapter.swift @@ -40,7 +40,10 @@ class AudioDeviceModuleDelegateAdapter: NSObject, LKRTCAudioDeviceModuleDelegate return entryPoint?.engineDidCreate(engine) ?? 0 } - func audioDeviceModule(_: LKRTCAudioDeviceModule, willEnableEngine engine: AVAudioEngine, isPlayoutEnabled: Bool, isRecordingEnabled: Bool) -> Int { + func audioDeviceModule(_: LKRTCAudioDeviceModule, willEnableEngine engine: AVAudioEngine, isPlayoutEnabled: Bool, isRecordingEnabled: Bool, isVoiceProcessingEnabled _: Bool) -> Int { + // isVoiceProcessingEnabled is new in the ADM delegate (webrtc-sdk + // PR 275). Ignored for now, exposing it through AudioEngineObserver + // is part of the framework version bump, not this feature. guard let audioManager else { return 0 } let entryPoint = audioManager.buildEngineObserverChain() return entryPoint?.engineWillEnable(engine, isPlayoutEnabled: isPlayoutEnabled, isRecordingEnabled: isRecordingEnabled) ?? 0 diff --git a/Sources/LiveKit/Broadcast/BroadcastScreenCapturer.swift b/Sources/LiveKit/Broadcast/BroadcastScreenCapturer.swift index f3d2c3c09..cc35dff52 100644 --- a/Sources/LiveKit/Broadcast/BroadcastScreenCapturer.swift +++ b/Sources/LiveKit/Broadcast/BroadcastScreenCapturer.swift @@ -25,8 +25,14 @@ import UIKit internal import LiveKitWebRTC class BroadcastScreenCapturer: BufferCapturer, @unchecked Sendable { + /// Destination for captured app audio; falls back to the mic mixer. + weak var appAudioSink: AppAudioSink? + private let appAudio: Bool + private let appAudioPublishMode: AppAudioPublishMode private var receiver: BroadcastReceiver? + // Only written from the incoming-samples loop. + private var didWarnMissingAppAudioSink = false override func startCapture() async throws -> Bool { let didStart = try await super.startCapture() @@ -67,7 +73,7 @@ class BroadcastScreenCapturer: BufferCapturer, @unchecked Sendable { for try await sample in receiver.incomingSamples { switch sample { case let .image(buffer, rotation): capture(buffer, rotation: rotation) - case let .audio(buffer): AudioManager.shared.mixer.capture(appAudio: buffer) + case let .audio(buffer): captureAppAudio(buffer) } } log("Broadcast receiver closed", .debug) @@ -88,8 +94,26 @@ class BroadcastScreenCapturer: BufferCapturer, @unchecked Sendable { return true } + private func captureAppAudio(_ buffer: AVAudioPCMBuffer) { + if let appAudioSink { + appAudioSink.capture(appAudio: buffer) + } else if appAudioPublishMode == .separateTrack { + // The separate app-audio track's sink is gone, e.g. the audio + // track failed to publish or was unpublished mid-share. Drop the + // buffer instead of silently mixing app audio back into the + // microphone track. + if !didWarnMissingAppAudioSink { + didWarnMissingAppAudioSink = true + log("App audio configured as a separate track but no sink is attached, dropping app audio", .warning) + } + } else { + AudioManager.shared.mixer.capture(appAudio: buffer) + } + } + init(delegate: LKRTCVideoCapturerDelegate, options: ScreenShareCaptureOptions) { appAudio = options.appAudio + appAudioPublishMode = options.appAudioPublishMode super.init(delegate: delegate, options: BufferCaptureOptions(from: options)) } } diff --git a/Sources/LiveKit/Core/RTC.swift b/Sources/LiveKit/Core/RTC.swift index 6142716cb..579df1c04 100644 --- a/Sources/LiveKit/Core/RTC.swift +++ b/Sources/LiveKit/Core/RTC.swift @@ -101,6 +101,12 @@ actor RTC { DispatchQueue.liveKitWebRTC.sync { peerConnectionFactory.audioSource(with: constraints) } } + static func createExternalAudioSource(sampleRate: Int, channels: Int, queueSizeMs: Int) -> LKRTCExternalAudioSource? { + DispatchQueue.liveKitWebRTC.sync { peerConnectionFactory.externalAudioSource(withSampleRate: Int32(sampleRate), + channels: UInt(channels), + queueSizeMs: Int32(queueSizeMs)) } + } + static func createAudioTrack(source: LKRTCAudioSource) -> LKRTCAudioTrack { DispatchQueue.liveKitWebRTC.sync { peerConnectionFactory.audioTrack(with: source, trackId: UUID().uuidString) } diff --git a/Sources/LiveKit/Participant/LocalParticipant.swift b/Sources/LiveKit/Participant/LocalParticipant.swift index f1d4c41e1..0434fa753 100644 --- a/Sources/LiveKit/Participant/LocalParticipant.swift +++ b/Sources/LiveKit/Participant/LocalParticipant.swift @@ -336,7 +336,9 @@ extension LocalParticipant { for mediaTrack in mediaTracks { // Don't re-publish muted tracks - if mediaTrack.isMuted { continue } + if mediaTrack.isMuted { + continue + } try await _publish(track: mediaTrack, options: mediaTrack.publishOptions) } } @@ -405,6 +407,11 @@ public extension LocalParticipant { try await publication.mute() } else { try await self.unpublish(publication: publication) + // App audio published as a separate track follows the + // screen-share video lifecycle. + if source == .screenShareVideo { + try await self.unpublishAppAudioTrackIfNeeded() + } } return publication } @@ -423,6 +430,7 @@ public extension LocalParticipant { let localTrack: LocalVideoTrack let defaultOptions = room._state.roomOptions.defaultScreenShareCaptureOptions + var screenShareOptions = defaultOptions if defaultOptions.useBroadcastExtension { if captureOptions != nil { @@ -437,17 +445,51 @@ public extension LocalParticipant { localTrack = LocalVideoTrack.createBroadcastScreenCapturerTrack(options: defaultOptions, reportStatistics: room._state.roomOptions.reportRemoteTrackStatistics) } else { - let options = (captureOptions as? ScreenShareCaptureOptions) ?? defaultOptions - localTrack = LocalVideoTrack.createInAppScreenShareTrack(options: options) + screenShareOptions = (captureOptions as? ScreenShareCaptureOptions) ?? defaultOptions + localTrack = LocalVideoTrack.createInAppScreenShareTrack(options: screenShareOptions) } - return try await self._publish(track: localTrack, options: publishOptions) + // Prepared before publishing so the capturer routes app + // audio to the independent track from the first buffer. + let appAudioTrack = self.prepareAppAudioTrack(options: screenShareOptions, videoTrack: localTrack, room: room) + let publication = try await self._publish(track: localTrack, options: publishOptions) + if let appAudioTrack { + do { + try await self._publish(track: appAudioTrack, options: AudioPublishOptions(encoding: .presetMusicHighQualityStereo, + dtx: false, + red: false)) + } catch { + // Screen share continues without app audio. The + // capturer's sink goes away with the track, so + // audio is dropped, not mixed into the mic. + self.log("Failed to publish app audio track: \(error)", .error) + } + } + return publication #elseif os(macOS) if #available(macOS 12.3, *) { let mainDisplay = try await MacOSScreenCapturer.mainDisplaySource() + let screenShareOptions = (captureOptions as? ScreenShareCaptureOptions) ?? room._state.roomOptions.defaultScreenShareCaptureOptions let track = LocalVideoTrack.createMacOSScreenShareTrack(source: mainDisplay, - options: (captureOptions as? ScreenShareCaptureOptions) ?? room._state.roomOptions.defaultScreenShareCaptureOptions, + options: screenShareOptions, reportStatistics: room._state.roomOptions.reportRemoteTrackStatistics) - return try await self._publish(track: track, options: publishOptions) + // Prepared before publishing so ScreenCaptureKit audio + // routes to the independent track from the first buffer. + let appAudioTrack = self.prepareAppAudioTrack(options: screenShareOptions, videoTrack: track, room: room) + let publication = try await self._publish(track: track, options: publishOptions) + if let appAudioTrack { + do { + try await self._publish(track: appAudioTrack, options: AudioPublishOptions(encoding: .presetMusicHighQualityStereo, + dtx: false, + red: false)) + } catch { + // Screen share continues without app audio. + // The capturer's sink goes away with the + // track, so audio is dropped, not mixed into + // the mic. + self.log("Failed to publish app audio track: \(error)", .error) + } + } + return publication } #endif } @@ -456,6 +498,51 @@ public extension LocalParticipant { return nil } } + + /// Unpublishes the independent app-audio track that follows the + /// screen-share video lifecycle, if one is published. + func unpublishAppAudioTrackIfNeeded() async throws { + // All matching publications, not just the first: a missed cleanup + // must not leave stale tracks behind on the next stop. + let appAudioPublications = audioTracks.compactMap { $0 as? LocalTrackPublication } + .filter { $0.source == .screenShareAudio } + for publication in appAudioPublications { + try await unpublish(publication: publication) + } + } + + /// Creates an independent app-audio track and routes the screen-share + /// capturer's audio to it, when the capture options request it. + private func prepareAppAudioTrack(options: ScreenShareCaptureOptions, + videoTrack: LocalVideoTrack, + room: Room) -> LocalAudioTrack? + { + guard options.appAudio, options.appAudioPublishMode == .separateTrack else { return nil } + do { + let source = try ExternalAudioSource() + // Only capturers with an app-audio path can feed the source. Bail + // instead of publishing a permanently silent track (e.g. iOS + // in-app capture, which has no audio handling). + #if os(iOS) + guard let capturer = videoTrack.capturer as? BroadcastScreenCapturer else { + log("App audio as a separate track requires the broadcast extension capturer, skipping app audio track", .warning) + return nil + } + capturer.appAudioSink = source + #elseif os(macOS) + guard #available(macOS 12.3, *), let capturer = videoTrack.capturer as? MacOSScreenCapturer else { + log("App audio as a separate track requires the ScreenCaptureKit capturer, skipping app audio track", .warning) + return nil + } + capturer.appAudioSink = source + #endif + return LocalAudioTrack.createTrack(externalSource: source, + reportStatistics: room._state.roomOptions.reportRemoteTrackStatistics) + } catch { + log("Failed to create app audio track: \(error)", .error) + return nil + } + } } // MARK: - Simulcast codecs @@ -735,8 +822,10 @@ extension LocalParticipant { // At this point at least 1 audio frame should be generated to continue if let track = track as? LocalAudioTrack { - // Only wait for frames if audio engine is allowed to start - if AudioManager.shared.engineAvailability.isInputAvailable { + // Externally fed tracks bypass the ADM capture path the frame + // watcher observes, and frames only flow once the app pushes + // audio, so there is nothing to wait for here. + if track.externalSource == nil, AudioManager.shared.engineAvailability.isInputAvailable { log("[Publish] Waiting for audio frame...") try await track.startWaitingForFrames() } diff --git a/Sources/LiveKit/Track/Capturers/MacOSScreenCapturer.swift b/Sources/LiveKit/Track/Capturers/MacOSScreenCapturer.swift index 685f3b94e..4eab4fe0d 100644 --- a/Sources/LiveKit/Track/Capturers/MacOSScreenCapturer.swift +++ b/Sources/LiveKit/Track/Capturers/MacOSScreenCapturer.swift @@ -41,6 +41,12 @@ public class MacOSScreenCapturer: VideoCapturer, @unchecked Sendable { /// The ``ScreenShareCaptureOptions`` used for this capturer. public let options: ScreenShareCaptureOptions + /// Destination for captured app audio; falls back to the mic mixer. + weak var appAudioSink: AppAudioSink? + + // Only written from the SCStream callback queue. + private var didWarnMissingAppAudioSink = false + struct State { // SCStream var scStream: SCStream? @@ -239,7 +245,20 @@ extension MacOSScreenCapturer: SCStreamOutput { if case .audio = outputType { guard let pcm = sampleBuffer.toAVAudioPCMBuffer() else { return } - AudioManager.shared.mixer.capture(appAudio: pcm) + if let appAudioSink { + appAudioSink.capture(appAudio: pcm) + } else if options.appAudioPublishMode == .separateTrack { + // The separate app-audio track's sink is gone, e.g. the audio + // track failed to publish or was unpublished mid-share. Drop + // the buffer instead of silently mixing app audio back into + // the microphone track. + if !didWarnMissingAppAudioSink { + didWarnMissingAppAudioSink = true + log("App audio configured as a separate track but no sink is attached, dropping app audio", .warning) + } + } else { + AudioManager.shared.mixer.capture(appAudio: pcm) + } } else if case .screen = outputType { // Retrieve the array of metadata attachments from the sample buffer. guard let attachmentsArray = CMSampleBufferGetSampleAttachmentsArray(sampleBuffer, @@ -261,7 +280,9 @@ extension MacOSScreenCapturer: SCStreamOutput { let newTimer = Task.detached(priority: .utility) { [weak self] in while true { try? await Task.sleep(nanoseconds: UInt64(1 * 1_000_000_000)) - if Task.isCancelled { break } + if Task.isCancelled { + break + } guard let self else { break } try await _capturePreviousFrame() } diff --git a/Sources/LiveKit/Track/Local/ExternalAudioSource.swift b/Sources/LiveKit/Track/Local/ExternalAudioSource.swift new file mode 100644 index 000000000..daf98fac3 --- /dev/null +++ b/Sources/LiveKit/Track/Local/ExternalAudioSource.swift @@ -0,0 +1,163 @@ +/* + * Copyright 2026 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import AVFAudio +import CoreMedia +import Foundation + +internal import LiveKitWebRTC + +/// Options for creating an ``ExternalAudioSource``. +public struct ExternalAudioSourceOptions: Sendable { + /// Sample rate of the audio delivered to WebRTC. + /// Pushed buffers in other formats are converted automatically. + public var sampleRate: Int + + /// Channel count of the audio delivered to WebRTC. + public var channels: Int + + /// Size of the internal jitter buffer in milliseconds, a multiple of 10. + /// + /// The default suits bursty producers such as ReplayKit or + /// ScreenCaptureKit: pushed audio is queued and delivered in 10 ms frames + /// by an internal pacer, with silence on underrun. + /// + /// A value of `0` selects synchronous mode for callers with their own + /// clock: each push must be exactly 10 ms and is delivered inline. + public var queueSizeMs: Int + + public init(sampleRate: Int = 48000, + channels: Int = 2, + queueSizeMs: Int = 100) + { + self.sampleRate = sampleRate + self.channels = channels + self.queueSizeMs = queueSizeMs + } +} + +/// An audio source the application pushes buffers into, independent of the +/// microphone capture path. +/// +/// Use ``LocalAudioTrack/createTrack(name:source:externalSource:reportStatistics:)`` +/// to publish the pushed audio as its own track, e.g. app audio during screen +/// share instead of mixing it into the microphone track. +/// +/// Note: WebRTC capture-side processing (echo cancellation, noise +/// suppression, gain control) is bypassed for this source by design. +public final class ExternalAudioSource: Loggable, @unchecked Sendable { + public let options: ExternalAudioSourceOptions + + // MARK: - Internal + + let rtcSource: LKRTCExternalAudioSource + + // MARK: - Private + + private struct State { + var converter: AudioConverter? + } + + private let _state = StateSync(State()) + + private let targetFormat: AVAudioFormat + + public init(options: ExternalAudioSourceOptions = ExternalAudioSourceOptions()) throws { + guard let rtcSource = RTC.createExternalAudioSource(sampleRate: options.sampleRate, + channels: options.channels, + queueSizeMs: options.queueSizeMs) + else { + throw LiveKitError(.invalidState, message: "Failed to create external audio source, check options") + } + guard let format = AVAudioFormat(commonFormat: .pcmFormatInt16, + sampleRate: Double(options.sampleRate), + channels: AVAudioChannelCount(options.channels), + interleaved: true) + else { + throw LiveKitError(.invalidState, message: "Failed to create audio format for options") + } + self.options = options + self.rtcSource = rtcSource + targetFormat = format + } + + /// Pushes an audio buffer. Converted to the source's declared format + /// (sample rate, channel count, int16) automatically when needed. + @discardableResult + public func push(_ buffer: AVAudioPCMBuffer) -> Bool { + // Fast path: formats the WebRTC layer accepts directly. + if buffer.format.sampleRate == targetFormat.sampleRate, + buffer.format.channelCount == targetFormat.channelCount + { + return rtcSource.capture(buffer, completionHandler: nil) + } + + guard let converter = converter(for: buffer.format) else { + log("Failed to get converter for input buffer format: \(buffer.format)", .warning) + return false + } + + let converted = converter.convert(from: buffer) + return rtcSource.capture(converted, completionHandler: nil) + } + + /// Pushes audio from a `CMSampleBuffer` carrying linear PCM, e.g. + /// ReplayKit app audio. + @discardableResult + public func push(_ sampleBuffer: CMSampleBuffer) -> Bool { + guard let pcm = sampleBuffer.toAVAudioPCMBuffer() else { + log("Failed to convert CMSampleBuffer to AVAudioPCMBuffer", .warning) + return false + } + return push(pcm) + } + + /// Drops any audio buffered inside the WebRTC layer. + public func clearBuffer() { + rtcSource.clearBuffer() + } + + /// Audio currently buffered inside the WebRTC layer, in milliseconds. + public var bufferedDurationMs: Int64 { + rtcSource.bufferedDurationMs + } + + private func converter(for format: AVAudioFormat) -> AudioConverter? { + _state.mutate { + if let converter = $0.converter, converter.inputFormat == format { + return converter + } + let converter = AudioConverter(from: format, to: targetFormat) + $0.converter = converter + return converter + } + } +} + +// MARK: - App audio routing + +/// Destination for app/screen-share audio buffers produced by capturers. +protocol AppAudioSink: AnyObject, Sendable { + func capture(appAudio: AVAudioPCMBuffer) +} + +extension ExternalAudioSource: AppAudioSink { + func capture(appAudio buffer: AVAudioPCMBuffer) { + push(buffer) + } +} + +extension MixerEngineObserver: AppAudioSink {} diff --git a/Sources/LiveKit/Track/Local/LocalAudioTrack.swift b/Sources/LiveKit/Track/Local/LocalAudioTrack.swift index 87461336d..65d4f2d43 100644 --- a/Sources/LiveKit/Track/Local/LocalAudioTrack.swift +++ b/Sources/LiveKit/Track/Local/LocalAudioTrack.swift @@ -25,6 +25,10 @@ public class LocalAudioTrack: Track, LocalTrackProtocol, AudioTrackProtocol, @un /// ``AudioCaptureOptions`` used to create this track. public let captureOptions: AudioCaptureOptions + /// The ``ExternalAudioSource`` feeding this track, or nil for tracks + /// backed by the microphone capture path. + public let externalSource: ExternalAudioSource? + // MARK: - Internal struct FrameWatcherState { @@ -37,9 +41,11 @@ public class LocalAudioTrack: Track, LocalTrackProtocol, AudioTrackProtocol, @un source: Track.Source, track: LKRTCMediaStreamTrack, reportStatistics: Bool, - captureOptions: AudioCaptureOptions) + captureOptions: AudioCaptureOptions, + externalSource: ExternalAudioSource? = nil) { self.captureOptions = captureOptions + self.externalSource = externalSource super.init(name: name, kind: .audio, @@ -86,6 +92,25 @@ public class LocalAudioTrack: Track, LocalTrackProtocol, AudioTrackProtocol, @un captureOptions: options) } + /// Creates a track fed by an ``ExternalAudioSource`` instead of the + /// microphone, e.g. for publishing app audio during screen share as an + /// independent track. + public static func createTrack(name: String = Track.screenShareAudioName, + source: Track.Source = .screenShareAudio, + externalSource: ExternalAudioSource, + reportStatistics: Bool = false) -> LocalAudioTrack + { + let rtcTrack = RTC.createAudioTrack(source: externalSource.rtcSource) + rtcTrack.isEnabled = true + + return LocalAudioTrack(name: name, + source: source, + track: rtcTrack, + reportStatistics: reportStatistics, + captureOptions: AudioCaptureOptions(), + externalSource: externalSource) + } + public func mute() async throws { try await super._mute() } @@ -121,6 +146,9 @@ public class LocalAudioTrack: Track, LocalTrackProtocol, AudioTrackProtocol, @un // MARK: - Internal override func startCapture() async throws { + // Externally-fed tracks bypass the AudioDeviceModule entirely, so the + // microphone engine must not be started for them. + guard externalSource == nil else { return } // AudioDeviceModule's InitRecording() and StartRecording() automatically get called by WebRTC, but // explicitly init & start it early to detect audio engine failures (mic not accessible for some reason, etc.). try AudioManager.shared.startLocalRecording( diff --git a/Sources/LiveKit/TrackPublications/LocalTrackPublication.swift b/Sources/LiveKit/TrackPublications/LocalTrackPublication.swift index 2b498e62a..4badebc0c 100644 --- a/Sources/LiveKit/TrackPublications/LocalTrackPublication.swift +++ b/Sources/LiveKit/TrackPublications/LocalTrackPublication.swift @@ -113,6 +113,7 @@ extension LocalTrackPublication: VideoCapturerDelegate { } try await participant.unpublish(publication: self) + try await participant.unpublishAppAudioTrackIfNeeded() } } // A similar check for macOS may be triggered e.g. when the display is powered off. @@ -124,6 +125,7 @@ extension LocalTrackPublication: VideoCapturerDelegate { } try await participant.unpublish(publication: self) + try await participant.unpublishAppAudioTrackIfNeeded() } } #endif diff --git a/Sources/LiveKit/Types/Options/ScreenShareCaptureOptions.swift b/Sources/LiveKit/Types/Options/ScreenShareCaptureOptions.swift index a3aa8b4bf..a98efefce 100644 --- a/Sources/LiveKit/Types/Options/ScreenShareCaptureOptions.swift +++ b/Sources/LiveKit/Types/Options/ScreenShareCaptureOptions.swift @@ -16,6 +16,15 @@ import Foundation +/// How captured app audio is published when ``ScreenShareCaptureOptions/appAudio`` is enabled. +@objc +public enum AppAudioPublishMode: Int, Sendable { + /// Mix app audio into the microphone track (default). + case mix + /// Publish app audio as an independent ``Track/Source/screenShareAudio`` track. + case separateTrack +} + @objcMembers public final class ScreenShareCaptureOptions: NSObject, VideoCaptureOptions, Sendable { public let dimensions: Dimensions @@ -27,6 +36,9 @@ public final class ScreenShareCaptureOptions: NSObject, VideoCaptureOptions, Sen public let appAudio: Bool + /// How app audio is published when ``appAudio`` is enabled. + public let appAudioPublishMode: AppAudioPublishMode + /// Use broadcast extension for screen capture (iOS only). /// /// If a broadcast extension has been properly configured, this defaults to `true`. @@ -50,6 +62,7 @@ public final class ScreenShareCaptureOptions: NSObject, VideoCaptureOptions, Sen fps: Int = 30, showCursor: Bool = true, appAudio: Bool = false, + appAudioPublishMode: AppAudioPublishMode = .mix, useBroadcastExtension: Bool = defaultToBroadcastExtension, includeCurrentApplication: Bool = false, excludeWindowIDs: [UInt32] = []) @@ -58,6 +71,7 @@ public final class ScreenShareCaptureOptions: NSObject, VideoCaptureOptions, Sen self.fps = fps self.showCursor = showCursor self.appAudio = appAudio + self.appAudioPublishMode = appAudioPublishMode self.useBroadcastExtension = useBroadcastExtension self.includeCurrentApplication = includeCurrentApplication self.excludeWindowIDs = excludeWindowIDs @@ -71,6 +85,7 @@ public final class ScreenShareCaptureOptions: NSObject, VideoCaptureOptions, Sen fps == other.fps && showCursor == other.showCursor && appAudio == other.appAudio && + appAudioPublishMode == other.appAudioPublishMode && useBroadcastExtension == other.useBroadcastExtension && includeCurrentApplication == other.includeCurrentApplication && excludeWindowIDs == other.excludeWindowIDs @@ -82,6 +97,7 @@ public final class ScreenShareCaptureOptions: NSObject, VideoCaptureOptions, Sen hasher.combine(fps) hasher.combine(showCursor) hasher.combine(appAudio) + hasher.combine(appAudioPublishMode) hasher.combine(useBroadcastExtension) hasher.combine(includeCurrentApplication) hasher.combine(excludeWindowIDs) diff --git a/Tests/LiveKitAudioTests/ExternalAudioSourceTests.swift b/Tests/LiveKitAudioTests/ExternalAudioSourceTests.swift new file mode 100644 index 000000000..b1aa6098c --- /dev/null +++ b/Tests/LiveKitAudioTests/ExternalAudioSourceTests.swift @@ -0,0 +1,167 @@ +/* + * Copyright 2026 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import AVFAudio +@testable import LiveKit +import Testing +#if canImport(LiveKitTestSupport) +import LiveKitTestSupport +#endif + +@Suite(.serialized, .tags(.audio)) struct ExternalAudioSourceTests { + /// Fills an int16 interleaved buffer with a sine wave. + private func makeSineBuffer(format: AVAudioFormat, frames: AVAudioFrameCount, frequency: Double = 440) -> AVAudioPCMBuffer { + let buffer = AVAudioPCMBuffer(pcmFormat: format, frameCapacity: frames)! + buffer.frameLength = frames + let channels = Int(format.channelCount) + let data = buffer.int16ChannelData![0] + for frame in 0 ..< Int(frames) { + let value = Int16(sin(2.0 * .pi * frequency * Double(frame) / format.sampleRate) * 8000.0) + for ch in 0 ..< channels { + data[frame * channels + ch] = value + } + } + return buffer + } + + /// Fills a float32 deinterleaved buffer with a sine wave, mimicking + /// ScreenCaptureKit output that requires conversion. + private func makeFloatSineBuffer(sampleRate: Double, channels: AVAudioChannelCount, frames: AVAudioFrameCount) -> AVAudioPCMBuffer { + let format = AVAudioFormat(standardFormatWithSampleRate: sampleRate, channels: channels)! + let buffer = AVAudioPCMBuffer(pcmFormat: format, frameCapacity: frames)! + buffer.frameLength = frames + for ch in 0 ..< Int(channels) { + let data = buffer.floatChannelData![ch] + for frame in 0 ..< Int(frames) { + data[frame] = Float(sin(2.0 * .pi * 440 * Double(frame) / sampleRate) * 0.5) + } + } + return buffer + } + + @Test func pushAndBufferDrain() async throws { + let source = try ExternalAudioSource() + let format = try #require(AVAudioFormat(commonFormat: .pcmFormatInt16, + sampleRate: 48000, + channels: 2, + interleaved: true)) + + // 100 ms buffer, capacity is 2x the queue size. + let buffer = makeSineBuffer(format: format, frames: 4800) // 100 ms + #expect(source.push(buffer)) + #expect(source.bufferedDurationMs > 0) + + // The 10 ms pacer drains the buffer even with no sinks attached. + try await Task.sleep(nanoseconds: 300_000_000) + #expect(source.bufferedDurationMs == 0) + } + + @Test func pushConvertsForeignFormats() throws { + let source = try ExternalAudioSource() + + // Float32 deinterleaved at a different sample rate (SCK-like). + let buffer = makeFloatSineBuffer(sampleRate: 44100, channels: 2, frames: 1024) + #expect(source.push(buffer)) + + // Mono at declared rate is remixed to the declared channel count. + let mono = makeFloatSineBuffer(sampleRate: 48000, channels: 1, frames: 480) + #expect(source.push(mono)) + } + + @Test func pushRejectsOverflow() throws { + let source = try ExternalAudioSource(options: ExternalAudioSourceOptions(queueSizeMs: 20)) + let format = try #require(AVAudioFormat(commonFormat: .pcmFormatInt16, + sampleRate: 48000, + channels: 2, + interleaved: true)) + + // Capacity is 2x queue size (40 ms); a 50 ms push must be rejected. + let tooBig = makeSineBuffer(format: format, frames: 2400) + #expect(!source.push(tooBig)) + + // A 40 ms push fits. + let fits = makeSineBuffer(format: format, frames: 1920) + #expect(source.push(fits)) + + source.clearBuffer() + #expect(source.bufferedDurationMs == 0) + } + + /// End-to-end: pushed audio reaches a remote participant as an + /// independent screen-share-audio track, with the ADM never started. + @Test(.tags(.e2e)) func publishExternalAudioTrack() async throws { + try await TestEnvironment.withRooms([RoomTestingOptions(canPublish: true), RoomTestingOptions(canSubscribe: true)]) { rooms in + let room1 = rooms[0] + let room2 = rooms[1] + + let publisherIdentity = try #require(room1.localParticipant.identity, "Publisher's identity is nil") + let remoteParticipant = try #require(room2.remoteParticipants[publisherIdentity], "Failed to lookup Publisher (RemoteParticipant)") + + let source = try ExternalAudioSource() + let localTrack = LocalAudioTrack.createTrack(externalSource: source) + #expect(localTrack.source == .screenShareAudio) + try await room1.localParticipant.publish(audioTrack: localTrack) + + // Keep pushing sine audio while the test observes the remote side. + let format = AVAudioFormat(commonFormat: .pcmFormatInt16, + sampleRate: 48000, + channels: 2, + interleaved: true)! + let pushTask = Task { + while !Task.isCancelled { + _ = source.push(makeSineBuffer(format: format, frames: 4800)) + try? await Task.sleep(nanoseconds: 100_000_000) + } + } + defer { pushTask.cancel() } + + // Wait for the remote track. + let deadline = Date().addingTimeInterval(30) + var remoteAudioTrack: RemoteAudioTrack? + var remotePublication: RemoteTrackPublication? + while Date() < deadline { + if let publication = remoteParticipant.audioTracks.first(where: { $0.source == .screenShareAudio }) as? RemoteTrackPublication, + let track = publication.track as? RemoteAudioTrack + { + remotePublication = publication + remoteAudioTrack = track + break + } + try await Task.sleep(nanoseconds: 200_000_000) + } + + let track = try #require(remoteAudioTrack, "Remote screen-share-audio track not found within timeout") + #expect(remotePublication?.source == .screenShareAudio) + + // Wait for audio frames to arrive remotely. + await confirmation("Did receive audio frame") { confirm in + let audioFrameWatcher = AudioTrackWatcher(id: "external01") { _ in + confirm() + } + track.add(audioRenderer: audioFrameWatcher) + try? await Task.sleep(nanoseconds: 30_000_000_000) + track.remove(audioRenderer: audioFrameWatcher) + } + + // The headline guarantee: audio flowed remotely while the ADM + // recording path never started. Playout may be running since the + // subscribing room shares the process, so only capture state is + // asserted. + #expect(!AudioManager.shared.isRecordingInitialized) + #expect(!AudioManager.shared.isRecording) + } + } +}