diff --git a/Package.swift b/Package.swift index bf7183a9a..bdabfd3ba 100644 --- a/Package.swift +++ b/Package.swift @@ -18,7 +18,7 @@ let package = Package( ], dependencies: [ // LK-Prefixed Dynamic WebRTC XCFramework - .package(url: "https://github.com/livekit/webrtc-xcframework.git", exact: "125.6422.11"), + .package(url: "https://github.com/livekit/webrtc-xcframework.git", exact: "125.6422.14"), .package(url: "https://github.com/apple/swift-protobuf.git", from: "1.26.0"), .package(url: "https://github.com/apple/swift-log.git", from: "1.5.4"), // Only used for DocC generation diff --git a/Package@swift-5.9.swift b/Package@swift-5.9.swift index 37e1c664d..8ad03927e 100644 --- a/Package@swift-5.9.swift +++ b/Package@swift-5.9.swift @@ -20,7 +20,7 @@ let package = Package( ], dependencies: [ // LK-Prefixed Dynamic WebRTC XCFramework - .package(url: "https://github.com/livekit/webrtc-xcframework.git", exact: "125.6422.11"), + .package(url: "https://github.com/livekit/webrtc-xcframework.git", exact: "125.6422.14"), .package(url: "https://github.com/apple/swift-protobuf.git", from: "1.26.0"), .package(url: "https://github.com/apple/swift-log.git", from: "1.5.4"), // Only used for DocC generation diff --git a/Sources/LiveKit/Audio/AudioDeviceModuleDelegateAdapter.swift b/Sources/LiveKit/Audio/AudioDeviceModuleDelegateAdapter.swift new file mode 100644 index 000000000..1c3710e95 --- /dev/null +++ b/Sources/LiveKit/Audio/AudioDeviceModuleDelegateAdapter.swift @@ -0,0 +1,88 @@ +/* + * Copyright 2025 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import Foundation + +#if swift(>=5.9) +internal import LiveKitWebRTC +#else +@_implementationOnly import LiveKitWebRTC +#endif + +// Invoked on WebRTC's worker thread, do not block. +class AudioDeviceModuleDelegateAdapter: NSObject, LKRTCAudioDeviceModuleDelegate { + weak var audioManager: AudioManager? + + func audioDeviceModule(_: LKRTCAudioDeviceModule, didReceiveSpeechActivityEvent speechActivityEvent: RTCSpeechActivityEvent) { + guard let audioManager else { return } + audioManager._state.onMutedSpeechActivity?(audioManager, speechActivityEvent.toLKType()) + } + + func audioDeviceModuleDidUpdateDevices(_: LKRTCAudioDeviceModule) { + guard let audioManager else { return } + audioManager._state.onDevicesDidUpdate?(audioManager) + } + + // Engine events + + func audioDeviceModule(_: LKRTCAudioDeviceModule, didCreateEngine engine: AVAudioEngine) { + guard let audioManager else { return } + let entryPoint = audioManager._state.engineObservers.buildChain() + entryPoint?.engineDidCreate(engine) + } + + func audioDeviceModule(_: LKRTCAudioDeviceModule, willEnableEngine engine: AVAudioEngine, isPlayoutEnabled: Bool, isRecordingEnabled: Bool) { + guard let audioManager else { return } + let entryPoint = audioManager._state.engineObservers.buildChain() + entryPoint?.engineWillEnable(engine, isPlayoutEnabled: isPlayoutEnabled, isRecordingEnabled: isRecordingEnabled) + } + + func audioDeviceModule(_: LKRTCAudioDeviceModule, willStartEngine engine: AVAudioEngine, isPlayoutEnabled: Bool, isRecordingEnabled: Bool) { + guard let audioManager else { return } + let entryPoint = audioManager._state.engineObservers.buildChain() + entryPoint?.engineWillStart(engine, isPlayoutEnabled: isPlayoutEnabled, isRecordingEnabled: isRecordingEnabled) + } + + func audioDeviceModule(_: LKRTCAudioDeviceModule, didStopEngine engine: AVAudioEngine, isPlayoutEnabled: Bool, isRecordingEnabled: Bool) { + guard let audioManager else { return } + let entryPoint = audioManager._state.engineObservers.buildChain() + entryPoint?.engineDidStop(engine, isPlayoutEnabled: isPlayoutEnabled, isRecordingEnabled: isRecordingEnabled) + } + + func audioDeviceModule(_: LKRTCAudioDeviceModule, didDisableEngine engine: AVAudioEngine, isPlayoutEnabled: Bool, isRecordingEnabled: Bool) { + guard let audioManager else { return } + let entryPoint = audioManager._state.engineObservers.buildChain() + entryPoint?.engineDidDisable(engine, isPlayoutEnabled: isPlayoutEnabled, isRecordingEnabled: isRecordingEnabled) + } + + func audioDeviceModule(_: LKRTCAudioDeviceModule, willReleaseEngine engine: AVAudioEngine) { + guard let audioManager else { return } + let entryPoint = audioManager._state.engineObservers.buildChain() + entryPoint?.engineWillRelease(engine) + } + + func audioDeviceModule(_: LKRTCAudioDeviceModule, engine: AVAudioEngine, configureInputFromSource src: AVAudioNode?, toDestination dst: AVAudioNode, format: AVAudioFormat) -> Bool { + guard let audioManager else { return false } + let entryPoint = audioManager._state.engineObservers.buildChain() + return entryPoint?.engineWillConnectInput(engine, src: src, dst: dst, format: format) ?? false + } + + func audioDeviceModule(_: LKRTCAudioDeviceModule, engine: AVAudioEngine, configureOutputFromSource src: AVAudioNode, toDestination dst: AVAudioNode?, format: AVAudioFormat) -> Bool { + guard let audioManager else { return false } + let entryPoint = audioManager._state.engineObservers.buildChain() + return entryPoint?.engineWillConnectOutput(engine, src: src, dst: dst, format: format) ?? false + } +} diff --git a/Sources/LiveKit/Audio/AudioEngineObserver.swift b/Sources/LiveKit/Audio/AudioEngineObserver.swift new file mode 100644 index 000000000..b3f6cc9d5 --- /dev/null +++ b/Sources/LiveKit/Audio/AudioEngineObserver.swift @@ -0,0 +1,63 @@ +/* + * Copyright 2025 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import AVFAudio + +/// Do not retain the engine object. +public protocol AudioEngineObserver: NextInvokable, Sendable { + func setNext(_ handler: any AudioEngineObserver) + + func engineDidCreate(_ engine: AVAudioEngine) + func engineWillEnable(_ engine: AVAudioEngine, isPlayoutEnabled: Bool, isRecordingEnabled: Bool) + func engineWillStart(_ engine: AVAudioEngine, isPlayoutEnabled: Bool, isRecordingEnabled: Bool) + func engineDidStop(_ engine: AVAudioEngine, isPlayoutEnabled: Bool, isRecordingEnabled: Bool) + func engineDidDisable(_ engine: AVAudioEngine, isPlayoutEnabled: Bool, isRecordingEnabled: Bool) + func engineWillRelease(_ engine: AVAudioEngine) + + /// Provide custom implementation for internal AVAudioEngine's output configuration. + /// Buffers flow from `src` to `dst`. Preferred format to connect node is provided as `format`. + /// Return true if custom implementation is provided, otherwise default implementation will be used. + func engineWillConnectOutput(_ engine: AVAudioEngine, src: AVAudioNode, dst: AVAudioNode?, format: AVAudioFormat) -> Bool + /// Provide custom implementation for internal AVAudioEngine's input configuration. + /// Buffers flow from `src` to `dst`. Preferred format to connect node is provided as `format`. + /// Return true if custom implementation is provided, otherwise default implementation will be used. + func engineWillConnectInput(_ engine: AVAudioEngine, src: AVAudioNode?, dst: AVAudioNode, format: AVAudioFormat) -> Bool +} + +/// Default implementation to make it optional. +public extension AudioEngineObserver { + func engineDidCreate(_: AVAudioEngine) {} + func engineWillEnable(_: AVAudioEngine, isPlayoutEnabled _: Bool, isRecordingEnabled _: Bool) {} + func engineWillStart(_: AVAudioEngine, isPlayoutEnabled _: Bool, isRecordingEnabled _: Bool) {} + func engineDidStop(_: AVAudioEngine, isPlayoutEnabled _: Bool, isRecordingEnabled _: Bool) {} + func engineDidDisable(_: AVAudioEngine, isPlayoutEnabled _: Bool, isRecordingEnabled _: Bool) {} + func engineWillRelease(_: AVAudioEngine) {} + + func engineWillConnectOutput(_: AVAudioEngine, src _: AVAudioNode, dst _: AVAudioNode?, format _: AVAudioFormat) -> Bool { false } + func engineWillConnectInput(_: AVAudioEngine, src _: AVAudioNode?, dst _: AVAudioNode, format _: AVAudioFormat) -> Bool { false } +} + +extension [any AudioEngineObserver] { + func buildChain() -> Element? { + guard let first else { return nil } + + for i in 0 ..< count - 1 { + self[i].setNext(self[i + 1]) + } + + return first + } +} diff --git a/Sources/LiveKit/Audio/DefaultAudioSessionObserver.swift b/Sources/LiveKit/Audio/DefaultAudioSessionObserver.swift new file mode 100644 index 000000000..959ef429f --- /dev/null +++ b/Sources/LiveKit/Audio/DefaultAudioSessionObserver.swift @@ -0,0 +1,127 @@ +/* + * Copyright 2025 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#if os(iOS) || os(visionOS) || os(tvOS) + +import AVFoundation + +#if swift(>=5.9) +internal import LiveKitWebRTC +#else +@_implementationOnly import LiveKitWebRTC +#endif + +public final class DefaultAudioSessionObserver: AudioEngineObserver, Loggable { + struct State { + var isSessionActive = false + var next: (any AudioEngineObserver)? + + // Used for backward compatibility with `customConfigureAudioSessionFunc`. + var isPlayoutEnabled: Bool = false + var isRecordingEnabled: Bool = false + } + + let _state = StateSync(State()) + + init() { + // Backward compatibility with `customConfigureAudioSessionFunc`. + _state.onDidMutate = { new_, old_ in + if let config_func = AudioManager.shared._state.customConfigureFunc, + new_.isPlayoutEnabled != old_.isPlayoutEnabled || + new_.isRecordingEnabled != old_.isRecordingEnabled + { + // Simulate state and invoke custom config func. + let old_state = AudioManager.State(localTracksCount: old_.isRecordingEnabled ? 1 : 0, remoteTracksCount: old_.isPlayoutEnabled ? 1 : 0) + let new_state = AudioManager.State(localTracksCount: new_.isRecordingEnabled ? 1 : 0, remoteTracksCount: new_.isPlayoutEnabled ? 1 : 0) + config_func(new_state, old_state) + } + } + } + + public func setNext(_ nextHandler: any AudioEngineObserver) { + _state.mutate { $0.next = nextHandler } + } + + public func engineWillEnable(_ engine: AVAudioEngine, isPlayoutEnabled: Bool, isRecordingEnabled: Bool) { + if AudioManager.shared._state.customConfigureFunc == nil { + log("Configuring audio session...") + let session = LKRTCAudioSession.sharedInstance() + session.lockForConfiguration() + defer { session.unlockForConfiguration() } + + let config: AudioSessionConfiguration = isRecordingEnabled ? .playAndRecordSpeaker : .playback + do { + if _state.isSessionActive { + log("AudioSession deactivating due to category switch") + try session.setActive(false) // Deactivate first + _state.mutate { $0.isSessionActive = false } + } + + log("AudioSession activating category to: \(config.category)") + try session.setConfiguration(config.toRTCType(), active: true) + _state.mutate { $0.isSessionActive = true } + } catch { + log("AudioSession failed to configure with error: \(error)", .error) + } + + log("AudioSession activationCount: \(session.activationCount), webRTCSessionCount: \(session.webRTCSessionCount)") + } + + _state.mutate { + $0.isPlayoutEnabled = isPlayoutEnabled + $0.isRecordingEnabled = isRecordingEnabled + } + + // Call next last + _state.next?.engineWillEnable(engine, isPlayoutEnabled: isPlayoutEnabled, isRecordingEnabled: isRecordingEnabled) + } + + public func engineDidDisable(_ engine: AVAudioEngine, isPlayoutEnabled: Bool, isRecordingEnabled: Bool) { + // Call next first + _state.next?.engineDidDisable(engine, isPlayoutEnabled: isPlayoutEnabled, isRecordingEnabled: isRecordingEnabled) + + _state.mutate { + $0.isPlayoutEnabled = isPlayoutEnabled + $0.isRecordingEnabled = isRecordingEnabled + } + + if AudioManager.shared._state.customConfigureFunc == nil { + log("Configuring audio session...") + let session = LKRTCAudioSession.sharedInstance() + session.lockForConfiguration() + defer { session.unlockForConfiguration() } + + do { + if isPlayoutEnabled, !isRecordingEnabled { + let config: AudioSessionConfiguration = .playback + log("AudioSession switching category to: \(config.category)") + try session.setConfiguration(config.toRTCType()) + } + if !isPlayoutEnabled, !isRecordingEnabled { + log("AudioSession deactivating") + try session.setActive(false) + _state.mutate { $0.isSessionActive = false } + } + } catch { + log("AudioSession failed to configure with error: \(error)", .error) + } + + log("AudioSession activationCount: \(session.activationCount), webRTCSessionCount: \(session.webRTCSessionCount)") + } + } +} + +#endif diff --git a/Sources/LiveKit/Core/RTC.swift b/Sources/LiveKit/Core/RTC.swift index 1d658d73c..e1632cee7 100644 --- a/Sources/LiveKit/Core/RTC.swift +++ b/Sources/LiveKit/Core/RTC.swift @@ -50,19 +50,6 @@ private class VideoEncoderFactorySimulcast: LKRTCVideoEncoderFactorySimulcast { } class RTC { - private static var _bypassVoiceProcessing: Bool = false - private static var _peerConnectionFactoryInitialized = false - - static var bypassVoiceProcessing: Bool { - get { _bypassVoiceProcessing } - set { - if _peerConnectionFactoryInitialized { - logger.log("Warning: Setting bypassVoiceProcessing after PeerConnectionFactory initialization has no effect. Set it at application launch.", .warning, type: Room.self) - } - _bypassVoiceProcessing = newValue - } - } - static let h264BaselineLevel5CodecInfo: LKRTCVideoCodecInfo = { // this should never happen guard let profileLevelId = LKRTCH264ProfileLevelId(profile: .constrainedBaseline, level: .level5) else { @@ -100,8 +87,7 @@ class RTC { logger.log("Initializing PeerConnectionFactory...", type: Room.self) - _peerConnectionFactoryInitialized = true - return LKRTCPeerConnectionFactory(bypassVoiceProcessing: bypassVoiceProcessing, + return LKRTCPeerConnectionFactory(bypassVoiceProcessing: false, encoderFactory: encoderFactory, decoderFactory: decoderFactory, audioProcessingModule: audioProcessingModule) diff --git a/Sources/LiveKit/Core/Room.swift b/Sources/LiveKit/Core/Room.swift index 96ed89908..da800b788 100644 --- a/Sources/LiveKit/Core/Room.swift +++ b/Sources/LiveKit/Core/Room.swift @@ -518,12 +518,11 @@ extension Room: AppStateDelegate { public extension Room { /// Set this to true to bypass initialization of voice processing. - /// Must be set before RTCPeerConnectionFactory gets initialized. - /// The most reliable place to set this is in your application's initialization process. + @available(*, deprecated, renamed: "AudioManager.shared.isVoiceProcessingBypassed") @objc static var bypassVoiceProcessing: Bool { - get { RTC.bypassVoiceProcessing } - set { RTC.bypassVoiceProcessing = newValue } + get { AudioManager.shared.isVoiceProcessingBypassed } + set { AudioManager.shared.isVoiceProcessingBypassed = newValue } } } diff --git a/Sources/LiveKit/Extensions/CustomStringConvertible.swift b/Sources/LiveKit/Extensions/CustomStringConvertible.swift index 79b09f409..b34d1dabd 100644 --- a/Sources/LiveKit/Extensions/CustomStringConvertible.swift +++ b/Sources/LiveKit/Extensions/CustomStringConvertible.swift @@ -178,3 +178,14 @@ extension AVCaptureDevice.Format { return "Format(\(values.joined(separator: ", ")))" } } + +extension LKRTCAudioProcessingConfig { + func toDebugString() -> String { + "RTCAudioProcessingConfig(" + + "isEchoCancellationEnabled: \(isEchoCancellationEnabled), " + + "isNoiseSuppressionEnabled: \(isNoiseSuppressionEnabled), " + + "isAutoGainControl1Enabled: \(isAutoGainControl1Enabled), " + + "isHighpassFilterEnabled: \(isHighpassFilterEnabled)" + + ")" + } +} diff --git a/Sources/LiveKit/Protocols/NextInvokable.swift b/Sources/LiveKit/Protocols/NextInvokable.swift new file mode 100644 index 000000000..4dc516155 --- /dev/null +++ b/Sources/LiveKit/Protocols/NextInvokable.swift @@ -0,0 +1,22 @@ +/* + * Copyright 2025 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import Foundation + +public protocol NextInvokable { + associatedtype Next + func setNext(_ handler: Next) +} diff --git a/Sources/LiveKit/Support/StateSync.swift b/Sources/LiveKit/Support/StateSync.swift index 9c7c2ae70..3ba4955af 100644 --- a/Sources/LiveKit/Support/StateSync.swift +++ b/Sources/LiveKit/Support/StateSync.swift @@ -18,7 +18,7 @@ import Combine import Foundation @dynamicMemberLookup -public final class StateSync { +public final class StateSync: @unchecked Sendable { // MARK: - Types public typealias OnDidMutate = (_ newState: State, _ oldState: State) -> Void diff --git a/Sources/LiveKit/Track/AudioManager.swift b/Sources/LiveKit/Track/AudioManager.swift index a1cca0c09..ad45ebf86 100644 --- a/Sources/LiveKit/Track/AudioManager.swift +++ b/Sources/LiveKit/Track/AudioManager.swift @@ -24,39 +24,6 @@ internal import LiveKitWebRTC @_implementationOnly import LiveKitWebRTC #endif -// Wrapper for LKRTCAudioBuffer -@objc -public class LKAudioBuffer: NSObject { - private let _audioBuffer: LKRTCAudioBuffer - - @objc - public var channels: Int { _audioBuffer.channels } - - @objc - public var frames: Int { _audioBuffer.frames } - - @objc - public var framesPerBand: Int { _audioBuffer.framesPerBand } - - @objc - public var bands: Int { _audioBuffer.bands } - - @objc - @available(*, deprecated, renamed: "rawBuffer(forChannel:)") - public func rawBuffer(for channel: Int) -> UnsafeMutablePointer { - _audioBuffer.rawBuffer(forChannel: channel) - } - - @objc - public func rawBuffer(forChannel channel: Int) -> UnsafeMutablePointer { - _audioBuffer.rawBuffer(forChannel: channel) - } - - init(audioBuffer: LKRTCAudioBuffer) { - _audioBuffer = audioBuffer - } -} - // Audio Session Configuration related public class AudioManager: Loggable { // MARK: - Public @@ -67,10 +34,13 @@ public class AudioManager: Loggable { public static let shared = AudioManager() #endif - public typealias DeviceUpdateFunc = (_ audioManager: AudioManager) -> Void + public typealias OnDevicesDidUpdate = (_ audioManager: AudioManager) -> Void + + public typealias OnSpeechActivity = (_ audioManager: AudioManager, _ event: SpeechActivityEvent) -> Void #if os(iOS) || os(visionOS) || os(tvOS) + @available(*, deprecated) public typealias ConfigureAudioSessionFunc = @Sendable (_ newState: State, _ oldState: State) -> Void @@ -83,9 +53,10 @@ public class AudioManager: Loggable { /// - ``isSpeakerOutputPreferred`` /// /// If you want to revert to default behavior, set this to `nil`. + @available(*, deprecated, message: "Use `set(engineObservers:)` instead. See `DefaultAudioSessionObserver` for example.") public var customConfigureAudioSessionFunc: ConfigureAudioSessionFunc? { - get { state.customConfigureFunc } - set { state.mutate { $0.customConfigureFunc = newValue } } + get { _state.customConfigureFunc } + set { _state.mutate { $0.customConfigureFunc = newValue } } } /// Determines whether the device's built-in speaker or receiver is preferred for audio output. @@ -96,8 +67,8 @@ public class AudioManager: Loggable { /// /// This property is ignored if ``customConfigureAudioSessionFunc`` is set. public var isSpeakerOutputPreferred: Bool { - get { state.isSpeakerOutputPreferred } - set { state.mutate { $0.isSpeakerOutputPreferred = newValue } } + get { _state.isSpeakerOutputPreferred } + set { _state.mutate { $0.isSpeakerOutputPreferred = newValue } } } /// Specifies a fixed configuration for the audio session, overriding dynamic adjustments. @@ -107,41 +78,31 @@ public class AudioManager: Loggable { /// /// This property is ignored if ``customConfigureAudioSessionFunc`` is set. public var sessionConfiguration: AudioSessionConfiguration? { - get { state.sessionConfiguration } - set { state.mutate { $0.sessionConfiguration = newValue } } + get { _state.sessionConfiguration } + set { _state.mutate { $0.sessionConfiguration = newValue } } } - #endif + @available(*, deprecated) public enum TrackState { case none case localOnly case remoteOnly case localAndRemote } + #endif - public struct State: Equatable, Sendable { - // Only consider State mutated when public vars change - public static func == (lhs: AudioManager.State, rhs: AudioManager.State) -> Bool { - var isEqual = lhs.localTracksCount == rhs.localTracksCount && - lhs.remoteTracksCount == rhs.remoteTracksCount - - #if os(iOS) || os(visionOS) || os(tvOS) - isEqual = isEqual && - lhs.isSpeakerOutputPreferred == rhs.isSpeakerOutputPreferred && - lhs.sessionConfiguration == rhs.sessionConfiguration - #endif - - return isEqual - } + public struct State: @unchecked Sendable { + var engineObservers = [any AudioEngineObserver]() + var onDevicesDidUpdate: OnDevicesDidUpdate? + var onMutedSpeechActivity: OnSpeechActivity? + #if os(iOS) || os(visionOS) || os(tvOS) + // Keep this var within State so it's protected by UnfairLock public var localTracksCount: Int = 0 public var remoteTracksCount: Int = 0 public var isSpeakerOutputPreferred: Bool = true - #if os(iOS) || os(visionOS) || os(tvOS) - // Keep this var within State so it's protected by UnfairLock public var customConfigureFunc: ConfigureAudioSessionFunc? public var sessionConfiguration: AudioSessionConfiguration? - #endif public var trackState: TrackState { switch (localTracksCount > 0, remoteTracksCount > 0) { @@ -151,6 +112,7 @@ public class AudioManager: Loggable { default: return .none } } + #endif } // MARK: - AudioProcessingModule @@ -213,120 +175,144 @@ public class AudioManager: Loggable { set { RTC.audioDeviceModule.inputDevice = newValue._ioDevice } } - public var onDeviceUpdate: DeviceUpdateFunc? { - didSet { - RTC.audioDeviceModule.setDevicesUpdatedHandler { [weak self] in - guard let self else { return } - self.onDeviceUpdate?(self) - } - } + public var onDeviceUpdate: OnDevicesDidUpdate? { + get { _state.onDevicesDidUpdate } + set { _state.mutate { $0.onDevicesDidUpdate = newValue } } } - // MARK: - Internal + /// Detect voice activity even if the mic is muted. + /// Internal audio engine must be initialized by calling ``prepareRecording()`` or + /// connecting to a room and subscribing to a remote audio track or publishing a local audio track. + public var onMutedSpeechActivity: OnSpeechActivity? { + get { _state.onMutedSpeechActivity } + set { _state.mutate { $0.onMutedSpeechActivity = newValue } } + } - enum `Type` { - case local - case remote + /// Enables advanced ducking which ducks other audio based on the presence of voice activity from local and remote chat participants. + /// Default: true. + public var isAdvancedDuckingEnabled: Bool { + get { RTC.audioDeviceModule.isAdvancedDuckingEnabled } + set { RTC.audioDeviceModule.isAdvancedDuckingEnabled = newValue } } - let state = StateSync(State()) + /// The ducking(audio reducing) level of other audio. + @available(iOS 17, macOS 14.0, visionOS 1.0, *) + public var duckingLevel: AudioDuckingLevel { + get { AudioDuckingLevel(rawValue: RTC.audioDeviceModule.duckingLevel) ?? .default } + set { RTC.audioDeviceModule.duckingLevel = newValue.rawValue } + } - // MARK: - Private + /// Bypass Voice-Processing I/O of internal AVAudioEngine. + /// It is valid to toggle this at runtime. + public var isVoiceProcessingBypassed: Bool { + get { RTC.audioDeviceModule.isVoiceProcessingBypassed } + set { RTC.audioDeviceModule.isVoiceProcessingBypassed = newValue } + } - private let _configureRunner = SerialRunnerActor() + /// Bypass the Auto Gain Control of internal AVAudioEngine. + /// It is valid to toggle this at runtime. + public var isVoiceProcessingAGCEnabled: Bool { + get { RTC.audioDeviceModule.isVoiceProcessingAGCEnabled } + set { RTC.audioDeviceModule.isVoiceProcessingAGCEnabled = newValue } + } - #if os(iOS) || os(visionOS) || os(tvOS) - private func _asyncConfigure(newState: State, oldState: State) async throws { - try await _configureRunner.run { - self.log("\(oldState) -> \(newState)") - let configureFunc = newState.customConfigureFunc ?? self.defaultConfigureAudioSessionFunc - configureFunc(newState, oldState) + /// Enables manual-rendering (no-device) mode of AVAudioEngine. + /// Currently experimental. + public var isManualRenderingMode: Bool { + get { RTC.audioDeviceModule.isManualRenderingMode } + set { + let result = RTC.audioDeviceModule.setManualRenderingMode(newValue) + if !result { + log("Failed to set manual rendering mode", .error) + } } } - #endif - func trackDidStart(_ type: Type) async throws { - let (newState, oldState) = state.mutate { state in - let oldState = state - if type == .local { state.localTracksCount += 1 } - if type == .remote { state.remoteTracksCount += 1 } - return (state, oldState) - } - #if os(iOS) || os(visionOS) || os(tvOS) - try await _asyncConfigure(newState: newState, oldState: oldState) - #endif + // MARK: - Recording + + /// Keep recording initialized (mic input) and pre-warm voice processing etc. + /// Mic permission is required and dialog will appear if not already granted. + /// This will per persisted accross Rooms and connections. + public var isRecordingAlwaysPrepared: Bool { + get { RTC.audioDeviceModule.isInitRecordingPersistentMode } + set { RTC.audioDeviceModule.isInitRecordingPersistentMode = newValue } } - func trackDidStop(_ type: Type) async throws { - let (newState, oldState) = state.mutate { state in - let oldState = state - if type == .local { state.localTracksCount = max(state.localTracksCount - 1, 0) } - if type == .remote { state.remoteTracksCount = max(state.remoteTracksCount - 1, 0) } - return (state, oldState) - } - #if os(iOS) || os(visionOS) || os(tvOS) - try await _asyncConfigure(newState: newState, oldState: oldState) - #endif + /// Starts mic input to the SDK even without any ``Room`` or a connection. + /// Audio buffers will flow into ``LocalAudioTrack/add(audioRenderer:)`` and ``capturePostProcessingDelegate``. + public func startLocalRecording() { + RTC.audioDeviceModule.initAndStartRecording() } - #if os(iOS) || os(visionOS) || os(tvOS) - /// The default implementation when audio session configuration is requested by the SDK. - /// Configure the `RTCAudioSession` of `WebRTC` framework. + /// Set a chain of ``AudioEngineObserver``s. + /// Defaults to having a single ``DefaultAudioSessionObserver`` initially. /// - /// > Note: It is recommended to use `RTCAudioSessionConfiguration.webRTC()` to obtain an instance of `RTCAudioSessionConfiguration` instead of instantiating directly. + /// The first object will be invoked and is responsible for calling the next object. + /// See ``NextInvokable`` protocol for details. /// - /// - Parameters: - /// - configuration: A configured RTCAudioSessionConfiguration - /// - setActive: passing true/false will call `AVAudioSession.setActive` internally - public func defaultConfigureAudioSessionFunc(newState: State, oldState: State) { - // Lazily computed config - let computeConfiguration: (() -> AudioSessionConfiguration) = { - switch newState.trackState { - case .none: - // Use .soloAmbient configuration - return .soloAmbient - case .remoteOnly where newState.isSpeakerOutputPreferred: - // Use .playback configuration with spoken audio - return .playback - default: - // Use .playAndRecord configuration - return newState.isSpeakerOutputPreferred ? .playAndRecordSpeaker : .playAndRecordReceiver - } - } + /// Objects set here will be retained. + public func set(engineObservers: [any AudioEngineObserver]) { + _state.mutate { $0.engineObservers = engineObservers } + } - let configuration = newState.sessionConfiguration ?? computeConfiguration() + // MARK: - For testing - var setActive: Bool? - if newState.trackState != .none, oldState.trackState == .none { - // activate audio session when there is any local/remote audio track - setActive = true - } else if newState.trackState == .none, oldState.trackState != .none { - // deactivate audio session when there are no more local/remote audio tracks - setActive = false - } + var isPlayoutInitialized: Bool { + RTC.audioDeviceModule.isPlayoutInitialized + } - let session = LKRTCAudioSession.sharedInstance() - // Check if needs setConfiguration - guard configuration != session.toAudioSessionConfiguration() else { - log("Skipping configure audio session, no changes") - return - } + var isPlaying: Bool { + RTC.audioDeviceModule.isPlaying + } - session.lockForConfiguration() - defer { session.unlockForConfiguration() } + var isRecordingInitialized: Bool { + RTC.audioDeviceModule.isRecordingInitialized + } - do { - log("Configuring audio session: \(String(describing: configuration))") - if let setActive { - try session.setConfiguration(configuration.toRTCType(), active: setActive) - } else { - try session.setConfiguration(configuration.toRTCType()) - } - } catch { - log("Failed to configure audio session with error: \(error)", .error) - } + var isRecording: Bool { + RTC.audioDeviceModule.isRecording + } + + func initPlayout() { + RTC.audioDeviceModule.initPlayout() + } + + func startPlayout() { + RTC.audioDeviceModule.startPlayout() + } + + func stopPlayout() { + RTC.audioDeviceModule.stopPlayout() + } + + func initRecording() { + RTC.audioDeviceModule.initRecording() + } + + func startRecording() { + RTC.audioDeviceModule.startRecording() + } + + func stopRecording() { + RTC.audioDeviceModule.stopRecording() + } + + // MARK: - Internal + + let _state: StateSync + + let _admDelegateAdapter = AudioDeviceModuleDelegateAdapter() + + init() { + #if os(iOS) || os(visionOS) || os(tvOS) + let engineObservers: [any AudioEngineObserver] = [DefaultAudioSessionObserver()] + #else + let engineObservers: [any AudioEngineObserver] = [] + #endif + _state = StateSync(State(engineObservers: engineObservers)) + _admDelegateAdapter.audioManager = self + RTC.audioDeviceModule.observer = _admDelegateAdapter } - #endif } public extension AudioManager { diff --git a/Sources/LiveKit/Track/Local/LocalAudioTrack.swift b/Sources/LiveKit/Track/Local/LocalAudioTrack.swift index 35eaae63c..5f64a5481 100644 --- a/Sources/LiveKit/Track/Local/LocalAudioTrack.swift +++ b/Sources/LiveKit/Track/Local/LocalAudioTrack.swift @@ -78,16 +78,6 @@ public class LocalAudioTrack: Track, LocalTrack, AudioTrack { public func unmute() async throws { try await super._unmute() } - - // MARK: - Internal - - override func startCapture() async throws { - try await AudioManager.shared.trackDidStart(.local) - } - - override func stopCapture() async throws { - try await AudioManager.shared.trackDidStop(.local) - } } public extension LocalAudioTrack { diff --git a/Sources/LiveKit/Track/Remote/RemoteAudioTrack.swift b/Sources/LiveKit/Track/Remote/RemoteAudioTrack.swift index 4b0776482..a62066e7c 100644 --- a/Sources/LiveKit/Track/Remote/RemoteAudioTrack.swift +++ b/Sources/LiveKit/Track/Remote/RemoteAudioTrack.swift @@ -75,14 +75,4 @@ public class RemoteAudioTrack: Track, RemoteTrack, AudioTrack { audioTrack.remove(_adapter) } } - - // MARK: - Internal - - override func startCapture() async throws { - try await AudioManager.shared.trackDidStart(.remote) - } - - override func stopCapture() async throws { - try await AudioManager.shared.trackDidStop(.remote) - } } diff --git a/Sources/LiveKit/Types/AudioBuffer.swift b/Sources/LiveKit/Types/AudioBuffer.swift new file mode 100644 index 000000000..63b39fd2f --- /dev/null +++ b/Sources/LiveKit/Types/AudioBuffer.swift @@ -0,0 +1,56 @@ +/* + * Copyright 2025 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#if swift(>=5.9) +internal import LiveKitWebRTC +#else +@_implementationOnly import LiveKitWebRTC +#endif + +import Foundation + +// Wrapper for LKRTCAudioBuffer +@objc +public class LKAudioBuffer: NSObject { + private let _audioBuffer: LKRTCAudioBuffer + + @objc + public var channels: Int { _audioBuffer.channels } + + @objc + public var frames: Int { _audioBuffer.frames } + + @objc + public var framesPerBand: Int { _audioBuffer.framesPerBand } + + @objc + public var bands: Int { _audioBuffer.bands } + + @objc + @available(*, deprecated, renamed: "rawBuffer(forChannel:)") + public func rawBuffer(for channel: Int) -> UnsafeMutablePointer { + _audioBuffer.rawBuffer(forChannel: channel) + } + + @objc + public func rawBuffer(forChannel channel: Int) -> UnsafeMutablePointer { + _audioBuffer.rawBuffer(forChannel: channel) + } + + init(audioBuffer: LKRTCAudioBuffer) { + _audioBuffer = audioBuffer + } +} diff --git a/Sources/LiveKit/Types/AudioDuckingLevel.swift b/Sources/LiveKit/Types/AudioDuckingLevel.swift new file mode 100644 index 000000000..6caa49673 --- /dev/null +++ b/Sources/LiveKit/Types/AudioDuckingLevel.swift @@ -0,0 +1,22 @@ +/* + * Copyright 2025 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +public enum AudioDuckingLevel: Int { + case `default` = 0 + case min = 10 + case mid = 20 + case max = 30 +} diff --git a/Sources/LiveKit/Types/AudioSessionConfiguration.swift b/Sources/LiveKit/Types/AudioSessionConfiguration.swift index 171da686b..89f855ea7 100644 --- a/Sources/LiveKit/Types/AudioSessionConfiguration.swift +++ b/Sources/LiveKit/Types/AudioSessionConfiguration.swift @@ -37,11 +37,11 @@ public extension AudioSessionConfiguration { mode: .spokenAudio) static let playAndRecordSpeaker = AudioSessionConfiguration(category: .playAndRecord, - categoryOptions: [.allowBluetooth, .allowBluetoothA2DP, .allowAirPlay], + categoryOptions: [.mixWithOthers, .allowBluetooth, .allowBluetoothA2DP, .allowAirPlay], mode: .videoChat) static let playAndRecordReceiver = AudioSessionConfiguration(category: .playAndRecord, - categoryOptions: [.allowBluetooth, .allowBluetoothA2DP, .allowAirPlay], + categoryOptions: [.mixWithOthers, .allowBluetooth, .allowBluetoothA2DP, .allowAirPlay], mode: .voiceChat) } diff --git a/Sources/LiveKit/Types/Options/AudioCaptureOptions.swift b/Sources/LiveKit/Types/Options/AudioCaptureOptions.swift index 1e0680b0d..3588f16fe 100644 --- a/Sources/LiveKit/Types/Options/AudioCaptureOptions.swift +++ b/Sources/LiveKit/Types/Options/AudioCaptureOptions.swift @@ -24,26 +24,40 @@ internal import LiveKitWebRTC @objc public final class AudioCaptureOptions: NSObject, CaptureOptions, Sendable { - @objc - public let echoCancellation: Bool + public static let noProcessing = AudioCaptureOptions( + echoCancellation: false, + autoGainControl: false, + noiseSuppression: false, + highpassFilter: false, + typingNoiseDetection: false + ) + /// Whether to enable software (WebRTC's) echo cancellation. + /// By default, Apple's voice processing is already enabled. + /// See ``AudioManager/isVoiceProcessingBypassed`` for details. @objc - public let noiseSuppression: Bool + public let echoCancellation: Bool + /// Whether to enable software (WebRTC's) gain control. + /// By default, Apple's gain control is already enabled. + /// See ``AudioManager/isVoiceProcessingAGCEnabled`` for details. @objc public let autoGainControl: Bool @objc - public let typingNoiseDetection: Bool + public let noiseSuppression: Bool @objc public let highpassFilter: Bool - public init(echoCancellation: Bool = true, - noiseSuppression: Bool = true, - autoGainControl: Bool = true, - typingNoiseDetection: Bool = true, - highpassFilter: Bool = true) + @objc + public let typingNoiseDetection: Bool + + public init(echoCancellation: Bool = false, + autoGainControl: Bool = false, + noiseSuppression: Bool = false, + highpassFilter: Bool = false, + typingNoiseDetection: Bool = false) { self.echoCancellation = echoCancellation self.noiseSuppression = noiseSuppression diff --git a/Sources/LiveKit/Types/SpeechActivityEvent.swift b/Sources/LiveKit/Types/SpeechActivityEvent.swift new file mode 100644 index 000000000..98d59917f --- /dev/null +++ b/Sources/LiveKit/Types/SpeechActivityEvent.swift @@ -0,0 +1,36 @@ +/* + * Copyright 2025 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#if swift(>=5.9) +internal import LiveKitWebRTC +#else +@_implementationOnly import LiveKitWebRTC +#endif + +public enum SpeechActivityEvent { + case started + case ended +} + +extension RTCSpeechActivityEvent { + func toLKType() -> SpeechActivityEvent { + switch self { + case .started: return .started + case .ended: return .ended + @unknown default: return .ended + } + } +} diff --git a/Tests/LiveKitTests/AudioEngineTests.swift b/Tests/LiveKitTests/AudioEngineTests.swift new file mode 100644 index 000000000..c481d304f --- /dev/null +++ b/Tests/LiveKitTests/AudioEngineTests.swift @@ -0,0 +1,329 @@ +/* + * Copyright 2025 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +@preconcurrency import AVFoundation +@testable import LiveKit +import LiveKitWebRTC +import XCTest + +class AudioEngineTests: XCTestCase { + override class func setUp() { + LiveKitSDK.setLoggerStandardOutput() + RTCSetMinDebugLogLevel(.info) + } + + override func tearDown() async throws {} + + #if !targetEnvironment(simulator) + // Test if mic is authorized. Only works on device. + func testMicAuthorized() async { + let status = AVCaptureDevice.authorizationStatus(for: .audio) + if status != .authorized { + let result = await AVCaptureDevice.requestAccess(for: .audio) + XCTAssert(result) + } + + XCTAssert(status == .authorized) + } + #endif + + // Test if state transitions pass internal checks. + func testStateTransitions() async { + let adm = AudioManager.shared + // Start Playout + adm.initPlayout() + XCTAssert(adm.isPlayoutInitialized) + adm.startPlayout() + XCTAssert(adm.isPlaying) + + // Start Recording + adm.initRecording() + XCTAssert(adm.isRecordingInitialized) + adm.startRecording() + XCTAssert(adm.isRecording) + + // Stop engine + adm.stopRecording() + XCTAssert(!adm.isRecording) + XCTAssert(!adm.isRecordingInitialized) + + adm.stopPlayout() + XCTAssert(!adm.isPlaying) + XCTAssert(!adm.isPlayoutInitialized) + } + + func testRecordingAlwaysPreparedMode() async { + let adm = AudioManager.shared + + // Ensure initially not initialized. + XCTAssert(!adm.isRecordingInitialized) + + // Ensure recording is initialized after set to true. + adm.isRecordingAlwaysPrepared = true + XCTAssert(adm.isRecordingInitialized) + + adm.startRecording() + XCTAssert(adm.isRecordingInitialized) + + // Should be still initialized after stopRecording() is called. + adm.stopRecording() + XCTAssert(adm.isRecordingInitialized) + } + + func testConfigureDucking() async { + AudioManager.shared.isAdvancedDuckingEnabled = false + XCTAssert(!AudioManager.shared.isAdvancedDuckingEnabled) + + AudioManager.shared.isAdvancedDuckingEnabled = true + XCTAssert(AudioManager.shared.isAdvancedDuckingEnabled) + + if #available(iOS 17, macOS 14.0, visionOS 1.0, *) { + AudioManager.shared.duckingLevel = .default + XCTAssert(AudioManager.shared.duckingLevel == .default) + + AudioManager.shared.duckingLevel = .min + XCTAssert(AudioManager.shared.duckingLevel == .min) + + AudioManager.shared.duckingLevel = .max + XCTAssert(AudioManager.shared.duckingLevel == .max) + + AudioManager.shared.duckingLevel = .mid + XCTAssert(AudioManager.shared.duckingLevel == .mid) + } + } + + // Test start generating local audio buffer without joining to room. + func testPrejoinLocalAudioBuffer() async throws { + // Set up expectation... + let didReceiveAudioFrame = expectation(description: "Did receive audio frame") + didReceiveAudioFrame.assertForOverFulfill = false + + // Start watching for audio frame... + let audioFrameWatcher = AudioTrackWatcher(id: "notifier01") { _ in + didReceiveAudioFrame.fulfill() + } + + let localMicTrack = LocalAudioTrack.createTrack() + // Attach audio frame watcher... + localMicTrack.add(audioRenderer: audioFrameWatcher) + + Task.detached { + print("Starting audio track in 3 seconds...") + try? await Task.sleep(nanoseconds: 3 * 1_000_000_000) + AudioManager.shared.startLocalRecording() + } + + // Wait for audio frame... + print("Waiting for first audio frame...") + await fulfillment(of: [didReceiveAudioFrame], timeout: 10) + + // Remove audio frame watcher... + localMicTrack.remove(audioRenderer: audioFrameWatcher) + } + + // Test the manual rendering mode (no-device mode) of AVAudioEngine based AudioDeviceModule. + // In manual rendering, no device access will be initialized such as mic and speaker. + func testManualRenderingModeSineGenerator() async throws { + // Set manual rendering mode... + AudioManager.shared.isManualRenderingMode = true + // Attach sine wave generator when engine requests input node. + // inputMixerNode will automatically convert to RTC's internal format (int16). + AudioManager.shared.set(engineObservers: [SineWaveNodeHook()]) + + // Check if manual rendering mode is set... + let isManualRenderingMode = AudioManager.shared.isManualRenderingMode + print("manualRenderingMode: \(isManualRenderingMode)") + XCTAssert(isManualRenderingMode) + + let recorder = try AudioRecorder() + + // Note: AudioCaptureOptions will not be applied since track is not published. + let track = LocalAudioTrack.createTrack(options: .noProcessing) + track.add(audioRenderer: recorder) + + // Start engine... + AudioManager.shared.startLocalRecording() + + // Render for 5 seconds... + try? await Task.sleep(nanoseconds: 5 * 1_000_000_000) + + recorder.close() + print("Written to: \(recorder.filePath)") + + // Stop engine + AudioManager.shared.stopRecording() + + // Play the recorded file... + let player = try AVAudioPlayer(contentsOf: recorder.filePath) + player.play() + while player.isPlaying { + try? await Task.sleep(nanoseconds: 1 * 100_000_000) // 10ms + } + } + + func testManualRenderingModeAudioFile() async throws { + // Sample audio + let url = URL(string: "https://github.com/rafaelreis-hotmart/Audio-Sample-files/raw/refs/heads/master/sample.wav")! + + print("Downloading sample audio from \(url)...") + let (downloadedLocalUrl, _) = try await URLSession.shared.downloadBackport(from: url) + + // Move the file to a new temporary location with a more descriptive name, if desired + let tempLocalUrl = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString).appendingPathExtension("wav") + try FileManager.default.moveItem(at: downloadedLocalUrl, to: tempLocalUrl) + print("Original file: \(tempLocalUrl)") + + let audioFile = try AVAudioFile(forReading: tempLocalUrl) + let audioFileFormat = audioFile.processingFormat // AVAudioFormat object + + print("Sample Rate: \(audioFileFormat.sampleRate)") + print("Channel Count: \(audioFileFormat.channelCount)") + print("Common Format: \(audioFileFormat.commonFormat)") + print("Interleaved: \(audioFileFormat.isInterleaved)") + + // Set manual rendering mode... + AudioManager.shared.isManualRenderingMode = true + + let playerNodeHook = PlayerNodeHook(playerNodeFormat: audioFileFormat) + AudioManager.shared.set(engineObservers: [playerNodeHook]) + + // Check if manual rendering mode is set... + let isManualRenderingMode = AudioManager.shared.isManualRenderingMode + print("manualRenderingMode: \(isManualRenderingMode)") + XCTAssert(isManualRenderingMode) + + let recorder = try AudioRecorder() + + // Note: AudioCaptureOptions will not be applied since track is not published. + let track = LocalAudioTrack.createTrack(options: .noProcessing) + track.add(audioRenderer: recorder) + + // Start engine... + AudioManager.shared.startLocalRecording() + + let scheduleAndPlayTask = Task { + print("Will scheduleFile") + await playerNodeHook.playerNode.scheduleFile(audioFile, at: nil) + print("Did scheduleFile") + } + + // Wait for audio file to be consumed... + playerNodeHook.playerNode.play() + await scheduleAndPlayTask.value + + recorder.close() + print("Processed file: \(recorder.filePath)") + + // Stop engine + AudioManager.shared.stopRecording() + + // Play the recorded file... + let player = try AVAudioPlayer(contentsOf: recorder.filePath) + player.play() + while player.isPlaying { + try? await Task.sleep(nanoseconds: 1 * 100_000_000) // 10ms + } + } + + #if os(iOS) || os(visionOS) || os(tvOS) + func testBackwardCompatibility() async throws { + struct TestState { + var trackState: AudioManager.TrackState = .none + } + let _testState = StateSync(TestState()) + + AudioManager.shared.customConfigureAudioSessionFunc = { newState, oldState in + print("New trackState: \(newState.trackState), Old trackState: \(oldState.trackState)") + _testState.mutate { $0.trackState = newState.trackState } + } + + // Configure session since we are setting a empty config func. + let config = AudioSessionConfiguration.playAndRecordSpeaker + let session = LKRTCAudioSession.sharedInstance() + session.lockForConfiguration() + try session.setConfiguration(config.toRTCType(), active: true) + session.unlockForConfiguration() + + XCTAssert(_testState.trackState == .none) + + AudioManager.shared.initPlayout() + XCTAssert(_testState.trackState == .remoteOnly) + + AudioManager.shared.initRecording() + XCTAssert(_testState.trackState == .localAndRemote) + + AudioManager.shared.stopRecording() + XCTAssert(_testState.trackState == .remoteOnly) + + AudioManager.shared.stopPlayout() + XCTAssert(_testState.trackState == .none) + } + + func testDefaultAudioSessionConfiguration() async throws { + AudioManager.shared.initPlayout() + AudioManager.shared.initRecording() + AudioManager.shared.stopRecording() + AudioManager.shared.stopPlayout() + } + #endif +} + +final class SineWaveNodeHook: AudioEngineObserver { + let sineWaveNode = SineWaveSourceNode() + + func setNext(_: any LiveKit.AudioEngineObserver) {} + func engineDidCreate(_ engine: AVAudioEngine) { + engine.attach(sineWaveNode) + } + + func engineWillRelease(_ engine: AVAudioEngine) { + engine.detach(sineWaveNode) + } + + func engineWillConnectInput(_ engine: AVAudioEngine, src _: AVAudioNode?, dst: AVAudioNode, format: AVAudioFormat) -> Bool { + print("engineWillConnectInput") + engine.connect(sineWaveNode, to: dst, format: format) + return true + } +} + +final class PlayerNodeHook: AudioEngineObserver { + public let playerNode = AVAudioPlayerNode() + public let playerMixerNode = AVAudioMixerNode() + public let playerNodeFormat: AVAudioFormat + + init(playerNodeFormat: AVAudioFormat) { + self.playerNodeFormat = playerNodeFormat + } + + func setNext(_: any LiveKit.AudioEngineObserver) {} + public func engineDidCreate(_ engine: AVAudioEngine) { + engine.attach(playerNode) + engine.attach(playerMixerNode) + } + + public func engineWillRelease(_ engine: AVAudioEngine) { + engine.detach(playerNode) + engine.detach(playerMixerNode) + } + + func engineWillConnectInput(_ engine: AVAudioEngine, src _: AVAudioNode?, dst: AVAudioNode, format: AVAudioFormat) -> Bool { + print("engineWillConnectInput") + engine.connect(playerNode, to: playerMixerNode, format: playerNodeFormat) + engine.connect(playerMixerNode, to: dst, format: format) + return true + } +} diff --git a/Tests/LiveKitTests/AudioProcessingTests.swift b/Tests/LiveKitTests/AudioProcessingTests.swift index 0078552aa..62c03911d 100644 --- a/Tests/LiveKitTests/AudioProcessingTests.swift +++ b/Tests/LiveKitTests/AudioProcessingTests.swift @@ -17,6 +17,7 @@ import Accelerate import AVFoundation import Foundation +import LiveKitWebRTC @testable import LiveKit import XCTest @@ -61,4 +62,65 @@ class AudioProcessingTests: XCTestCase, AudioCustomProcessingDelegate { try await Task.sleep(nanoseconds: ns) } } + + func testOptionsAppliedToAudioProcessingModule() async throws { + // Disable Apple VPIO. + AudioManager.shared.isVoiceProcessingBypassed = true + + try await withRooms([RoomTestingOptions(canPublish: true)]) { rooms in + // Alias to Room1 + let room1 = rooms[0] + + let allOnOptions = AudioCaptureOptions( + echoCancellation: true, + autoGainControl: true, + noiseSuppression: true, + highpassFilter: true + ) + + let allOffOptions = AudioCaptureOptions( + echoCancellation: false, + autoGainControl: false, + noiseSuppression: false, + highpassFilter: false + ) + + let pub1 = try await room1.localParticipant.setMicrophone(enabled: true, captureOptions: allOnOptions) + guard let pub1 else { + XCTFail("Publication is nil") + return + } + + let ns = UInt64(3 * 1_000_000_000) + try await Task.sleep(nanoseconds: ns) + + // Directly read config from the apm + let allOnConfigResult = RTC.audioProcessingModule.config + print("Config result for all on: \(allOnConfigResult.toDebugString()))") + XCTAssert(allOnConfigResult.isEchoCancellationEnabled) + XCTAssert(allOnConfigResult.isNoiseSuppressionEnabled) + XCTAssert(allOnConfigResult.isAutoGainControl1Enabled) + XCTAssert(allOnConfigResult.isHighpassFilterEnabled) + + try await room1.localParticipant.unpublish(publication: pub1) + + let pub2 = try await room1.localParticipant.setMicrophone(enabled: true, captureOptions: allOffOptions) + guard let pub2 else { + XCTFail("Publication is nil") + return + } + + try await Task.sleep(nanoseconds: ns) + + // Directly read config from the apm + let allOffConfigResult = RTC.audioProcessingModule.config + print("Config result for all off: \(allOffConfigResult.toDebugString())") + XCTAssert(!allOffConfigResult.isEchoCancellationEnabled) + XCTAssert(!allOffConfigResult.isNoiseSuppressionEnabled) + XCTAssert(!allOffConfigResult.isAutoGainControl1Enabled) + XCTAssert(!allOffConfigResult.isHighpassFilterEnabled) + + try await room1.localParticipant.unpublish(publication: pub2) + } + } } diff --git a/Tests/LiveKitTests/Support/AudioRecorder.swift b/Tests/LiveKitTests/Support/AudioRecorder.swift new file mode 100644 index 000000000..dc0cffe99 --- /dev/null +++ b/Tests/LiveKitTests/Support/AudioRecorder.swift @@ -0,0 +1,63 @@ +/* + * Copyright 2025 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import AVFAudio +@testable import LiveKit + +// Used to save audio data for inspecting the correct format, etc. +class AudioRecorder { + public let sampleRate: Double + public let filePath: URL + private var audioFile: AVAudioFile? + + init(sampleRate: Double = 16000, channels: Int = 1) throws { + self.sampleRate = sampleRate + + let settings: [String: Any] = [ + AVFormatIDKey: kAudioFormatLinearPCM, + AVSampleRateKey: sampleRate, + AVNumberOfChannelsKey: channels, + AVLinearPCMBitDepthKey: 16, + AVLinearPCMIsFloatKey: false, + AVLinearPCMIsNonInterleaved: false, + AVLinearPCMIsBigEndianKey: false, + ] + + let fileName = UUID().uuidString + ".wav" + let filePath = URL(fileURLWithPath: NSTemporaryDirectory()).appendingPathComponent(fileName) + self.filePath = filePath + + audioFile = try AVAudioFile(forWriting: filePath, + settings: settings, + commonFormat: .pcmFormatInt16, + interleaved: true) + } + + func write(pcmBuffer: AVAudioPCMBuffer) throws { + guard let audioFile else { return } + try audioFile.write(from: pcmBuffer) + } + + func close() { + audioFile = nil + } +} + +extension AudioRecorder: AudioRenderer { + func render(pcmBuffer: AVAudioPCMBuffer) { + try? write(pcmBuffer: pcmBuffer) + } +} diff --git a/Tests/LiveKitTests/Support/SinWaveSourceNode.swift b/Tests/LiveKitTests/Support/SinWaveSourceNode.swift new file mode 100644 index 000000000..258a720b3 --- /dev/null +++ b/Tests/LiveKitTests/Support/SinWaveSourceNode.swift @@ -0,0 +1,67 @@ +/* + * Copyright 2025 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import AVFAudio + +class SineWaveSourceNode: AVAudioSourceNode, @unchecked Sendable { + private let sampleRate: Double + private let frequency: Double + + init(frequency: Double = 400.0, sampleRate: Double = 48000.0) { + self.frequency = frequency + self.sampleRate = sampleRate + + let format = AVAudioFormat(standardFormatWithSampleRate: sampleRate, channels: 1)! + + let twoPi = 2 * Float.pi + let amplitude: Float = 0.5 + var currentPhase: Float = 0.0 + let phaseIncrement: Float = (twoPi / Float(sampleRate)) * Float(frequency) + + let renderBlock: AVAudioSourceNodeRenderBlock = { _, _, frameCount, audioBufferList in + print("AVAudioSourceNodeRenderBlock frameCount: \(frameCount)") + + // Accessing the AudioBufferList manually + let audioBuffers = audioBufferList.pointee + + // Assuming a single channel setup + guard audioBuffers.mNumberBuffers > 0 else { + return noErr + } + + let audioBuffer = audioBuffers.mBuffers // Access first buffer + guard let dataPointer = audioBuffer.mData?.assumingMemoryBound(to: Float.self) else { + return noErr + } + + let bufferPointer = UnsafeMutableBufferPointer(start: dataPointer, count: Int(frameCount)) + + // Generate sine wave samples + for frame in 0 ..< bufferPointer.count { + let value = sin(currentPhase) * amplitude + currentPhase += phaseIncrement + if currentPhase >= twoPi { currentPhase -= twoPi } + if currentPhase < 0.0 { currentPhase += twoPi } + + bufferPointer[frame] = value + } + + return noErr + } + + super.init(format: format, renderBlock: renderBlock) + } +} diff --git a/Tests/LiveKitTests/TrackTests.swift b/Tests/LiveKitTests/TrackTests.swift index 55bae6534..04559e6f7 100644 --- a/Tests/LiveKitTests/TrackTests.swift +++ b/Tests/LiveKitTests/TrackTests.swift @@ -29,12 +29,10 @@ class TestTrack: LocalAudioTrack { override func startCapture() async throws { try? await Task.sleep(nanoseconds: UInt64(Double.random(in: 0.0 ... 1.0) * 1_000_000)) - try await AudioManager.shared.trackDidStart(.local) } override func stopCapture() async throws { try? await Task.sleep(nanoseconds: UInt64(Double.random(in: 0.0 ... 1.0) * 1_000_000)) - try await AudioManager.shared.trackDidStop(.local) } } @@ -71,7 +69,7 @@ class TrackTests: XCTestCase { AudioManager.shared.customConfigureAudioSessionFunc = nil - XCTAssertEqual(AudioManager.shared.state.localTracksCount, 0, "localTracksCount should be 0") + XCTAssertEqual(AudioManager.shared._state.localTracksCount, 0, "localTracksCount should be 0") } #endif }