From 8c3da061306942bb701b909f1e1f49b3c17bdc95 Mon Sep 17 00:00:00 2001 From: cloudwebrtc Date: Sun, 22 Sep 2024 21:13:47 +0800 Subject: [PATCH 1/5] feat: Try to add AEC3 for rust-core. --- .gitignore | 3 ++- webrtc-sys/include/livekit/audio_device.h | 2 ++ webrtc-sys/src/audio_device.cpp | 13 ++++++++++++- webrtc-sys/src/audio_track.cpp | 4 ++-- webrtc-sys/src/peer_connection_factory.cpp | 10 +++++++++- 5 files changed, 27 insertions(+), 5 deletions(-) diff --git a/.gitignore b/.gitignore index 3d98c20c..b54de811 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ target -/.idea \ No newline at end of file +/.idea +.DS_Store diff --git a/webrtc-sys/include/livekit/audio_device.h b/webrtc-sys/include/livekit/audio_device.h index 903e22d6..d8d24f92 100644 --- a/webrtc-sys/include/livekit/audio_device.h +++ b/webrtc-sys/include/livekit/audio_device.h @@ -20,6 +20,7 @@ #include "api/task_queue/task_queue_factory.h" #include "modules/audio_device/include/audio_device.h" +#include "modules/audio_device/audio_device_buffer.h" #include "rtc_base/synchronization/mutex.h" #include "rtc_base/task_queue.h" #include "rtc_base/task_utils/repeating_task.h" @@ -123,6 +124,7 @@ class AudioDevice : public webrtc::AudioDeviceModule { webrtc::RepeatingTaskHandle audio_task_; webrtc::AudioTransport* audio_transport_; webrtc::TaskQueueFactory* task_queue_factory_; + webrtc::AudioDeviceBuffer audio_device_buffer_; bool playing_{false}; bool initialized_{false}; }; diff --git a/webrtc-sys/src/audio_device.cpp b/webrtc-sys/src/audio_device.cpp index 75911917..9ae77c18 100644 --- a/webrtc-sys/src/audio_device.cpp +++ b/webrtc-sys/src/audio_device.cpp @@ -25,7 +25,8 @@ namespace livekit { AudioDevice::AudioDevice(webrtc::TaskQueueFactory* task_queue_factory) : task_queue_factory_(task_queue_factory), - data_(kSamplesPer10Ms * kChannels) {} + data_(kSamplesPer10Ms * kChannels), + audio_device_buffer_(task_queue_factory) {} AudioDevice::~AudioDevice() { Terminate(); @@ -39,6 +40,7 @@ int32_t AudioDevice::ActiveAudioLayer(AudioLayer* audioLayer) const { int32_t AudioDevice::RegisterAudioCallback(webrtc::AudioTransport* transport) { webrtc::MutexLock lock(&mutex_); audio_transport_ = transport; + audio_device_buffer_.RegisterAudioCallback(transport); return 0; } @@ -47,6 +49,11 @@ int32_t AudioDevice::Init() { if (initialized_) return 0; + audio_device_buffer_.SetRecordingSampleRate(kSampleRate); + audio_device_buffer_.SetPlayoutSampleRate(kSampleRate); + audio_device_buffer_.SetRecordingChannels(kChannels); + audio_device_buffer_.SetPlayoutChannels(kChannels); + audio_queue_ = std::make_unique(task_queue_factory_->CreateTaskQueue( "AudioDevice", webrtc::TaskQueueFactory::Priority::NORMAL)); @@ -63,9 +70,13 @@ int32_t AudioDevice::Init() { // Request the AudioData, otherwise WebRTC will ignore the packets. // 10ms of audio data. + audio_device_buffer_.RequestPlayoutData(kSamplesPer10Ms); + audio_device_buffer_.GetPlayoutData(data); + /* audio_transport_->NeedMorePlayData( kSamplesPer10Ms, kBytesPerSample, kChannels, kSampleRate, data, n_samples_out, &elapsed_time_ms, &ntp_time_ms); + */ } return webrtc::TimeDelta::Millis(10); diff --git a/webrtc-sys/src/audio_track.cpp b/webrtc-sys/src/audio_track.cpp index 1d42f14e..2931cfc1 100644 --- a/webrtc-sys/src/audio_track.cpp +++ b/webrtc-sys/src/audio_track.cpp @@ -163,7 +163,7 @@ AudioTrackSource::InternalSource::InternalSource( if (buffer_.size() >= samples10ms) { for (auto sink : sinks_) - sink->OnData(buffer_.data(), sizeof(int16_t), sample_rate_, + sink->OnData(buffer_.data(), sizeof(int16_t) * 8, sample_rate_, num_channels_, samples10ms / num_channels_); buffer_.erase(buffer_.begin(), buffer_.begin() + samples10ms); @@ -171,7 +171,7 @@ AudioTrackSource::InternalSource::InternalSource( missed_frames_++; if (missed_frames_ >= silence_frames_threshold) { for (auto sink : sinks_) - sink->OnData(silence_buffer_, sizeof(int16_t), sample_rate_, + sink->OnData(silence_buffer_, sizeof(int16_t) * 8, sample_rate_, num_channels_, samples10ms / num_channels_); } } diff --git a/webrtc-sys/src/peer_connection_factory.cpp b/webrtc-sys/src/peer_connection_factory.cpp index bbc73bfd..3a5d9a96 100644 --- a/webrtc-sys/src/peer_connection_factory.cpp +++ b/webrtc-sys/src/peer_connection_factory.cpp @@ -21,6 +21,8 @@ #include "api/audio_codecs/builtin_audio_decoder_factory.h" #include "api/audio_codecs/builtin_audio_encoder_factory.h" +#include "api/audio/echo_canceller3_factory.h" +#include "api/audio/echo_canceller3_config.h" #include "api/peer_connection_interface.h" #include "api/rtc_error.h" #include "api/rtc_event_log/rtc_event_log_factory.h" @@ -76,7 +78,13 @@ PeerConnectionFactory::PeerConnectionFactory( std::move(std::make_unique()); media_deps.audio_encoder_factory = webrtc::CreateBuiltinAudioEncoderFactory(); media_deps.audio_decoder_factory = webrtc::CreateBuiltinAudioDecoderFactory(); - media_deps.audio_processing = webrtc::AudioProcessingBuilder().Create(); + + auto apm = webrtc::AudioProcessingBuilder(); + auto cfg = webrtc::EchoCanceller3Config(); + auto echo_control = std::make_unique(cfg); + + apm.SetEchoControlFactory(std::move(echo_control)); + media_deps.audio_processing = apm.Create(); media_deps.trials = dependencies.trials.get(); dependencies.media_engine = cricket::CreateMediaEngine(std::move(media_deps)); From 60503a23bef027fa2c14bafe154f443efc572a06 Mon Sep 17 00:00:00 2001 From: cloudwebrtc Date: Sun, 22 Sep 2024 21:43:32 +0800 Subject: [PATCH 2/5] add audio mixer for playout. --- webrtc-sys/src/peer_connection_factory.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/webrtc-sys/src/peer_connection_factory.cpp b/webrtc-sys/src/peer_connection_factory.cpp index 3a5d9a96..4ff83a90 100644 --- a/webrtc-sys/src/peer_connection_factory.cpp +++ b/webrtc-sys/src/peer_connection_factory.cpp @@ -41,6 +41,7 @@ #include "rtc_base/thread.h" #include "webrtc-sys/src/peer_connection.rs.h" #include "webrtc-sys/src/peer_connection_factory.rs.h" +#include "modules/audio_mixer/audio_mixer_impl.h" namespace livekit { @@ -85,6 +86,11 @@ PeerConnectionFactory::PeerConnectionFactory( apm.SetEchoControlFactory(std::move(echo_control)); media_deps.audio_processing = apm.Create(); + + auto audio_mixer = webrtc::AudioMixerImpl::Create(); + + media_deps.audio_mixer = audio_mixer; + media_deps.trials = dependencies.trials.get(); dependencies.media_engine = cricket::CreateMediaEngine(std::move(media_deps)); From 365de80e5a91dc1db5b798e7a3ef81119f6156c5 Mon Sep 17 00:00:00 2001 From: cloudwebrtc Date: Mon, 23 Sep 2024 10:42:10 +0800 Subject: [PATCH 3/5] move AudioDevice/AudioMixer to audio_context. --- webrtc-sys/build.rs | 1 + webrtc-sys/include/livekit/audio_context.h | 54 ++++++++++++++ webrtc-sys/include/livekit/audio_device.h | 8 +++ .../include/livekit/peer_connection_factory.h | 3 +- webrtc-sys/src/audio_context.cpp | 71 +++++++++++++++++++ webrtc-sys/src/peer_connection_factory.cpp | 17 ++--- 6 files changed, 140 insertions(+), 14 deletions(-) create mode 100644 webrtc-sys/include/livekit/audio_context.h create mode 100644 webrtc-sys/src/audio_context.cpp diff --git a/webrtc-sys/build.rs b/webrtc-sys/build.rs index 5e87b0c9..18efe6af 100644 --- a/webrtc-sys/build.rs +++ b/webrtc-sys/build.rs @@ -54,6 +54,7 @@ fn main() { "src/peer_connection_factory.cpp", "src/media_stream.cpp", "src/media_stream_track.cpp", + "src/audio_context.cpp", "src/audio_track.cpp", "src/video_track.cpp", "src/data_channel.cpp", diff --git a/webrtc-sys/include/livekit/audio_context.h b/webrtc-sys/include/livekit/audio_context.h new file mode 100644 index 00000000..64c433e5 --- /dev/null +++ b/webrtc-sys/include/livekit/audio_context.h @@ -0,0 +1,54 @@ +/* + * Copyright 2023 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the “License”); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an “AS IS” BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include + +#include "api/scoped_refptr.h" +#include "livekit/audio_device.h" +#include "livekit/webrtc.h" +#include "modules/audio_mixer/audio_mixer_impl.h" + +namespace webrtc { +class TaskQueueFactory; +class AudioDeviceBuffer; +class AudioTransport; +} // namespace webrtc + +namespace livekit { + +class AudioContext { + public: + AudioContext(std::shared_ptr rtc_runtime); + virtual ~AudioContext(); + + rtc::scoped_refptr audio_device( + webrtc::TaskQueueFactory* task_queue_factory); + + rtc::scoped_refptr audio_mixer(); + + webrtc::AudioDeviceBuffer* audio_device_buffer(); + + webrtc::AudioTransport* audio_transport(); + + private: + std::shared_ptr rtc_runtime_; + rtc::scoped_refptr audio_device_; + rtc::scoped_refptr audio_mixer_; +}; + +} // namespace livekit \ No newline at end of file diff --git a/webrtc-sys/include/livekit/audio_device.h b/webrtc-sys/include/livekit/audio_device.h index d8d24f92..ab0d5845 100644 --- a/webrtc-sys/include/livekit/audio_device.h +++ b/webrtc-sys/include/livekit/audio_device.h @@ -117,6 +117,14 @@ class AudioDevice : public webrtc::AudioDeviceModule { int32_t SetAudioDeviceSink(webrtc::AudioDeviceSink* sink) const override; + webrtc::AudioDeviceBuffer *audio_device_buffer() { + return &audio_device_buffer_; + } + + webrtc::AudioTransport* audio_transport() { + return audio_transport_; + } + private: mutable webrtc::Mutex mutex_; std::vector data_; diff --git a/webrtc-sys/include/livekit/peer_connection_factory.h b/webrtc-sys/include/livekit/peer_connection_factory.h index ae49842b..89ee9d24 100644 --- a/webrtc-sys/include/livekit/peer_connection_factory.h +++ b/webrtc-sys/include/livekit/peer_connection_factory.h @@ -20,6 +20,7 @@ #include "api/scoped_refptr.h" #include "api/task_queue/task_queue_factory.h" #include "livekit/audio_device.h" +#include "livekit/audio_context.h" #include "media_stream.h" #include "rtp_parameters.h" #include "rust/cxx.h" @@ -64,7 +65,7 @@ class PeerConnectionFactory { private: std::shared_ptr rtc_runtime_; - rtc::scoped_refptr audio_device_; + AudioContext audio_context_; rtc::scoped_refptr peer_factory_; webrtc::TaskQueueFactory* task_queue_factory_; }; diff --git a/webrtc-sys/src/audio_context.cpp b/webrtc-sys/src/audio_context.cpp new file mode 100644 index 00000000..32445ec3 --- /dev/null +++ b/webrtc-sys/src/audio_context.cpp @@ -0,0 +1,71 @@ +/* + * Copyright 2023 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the “License”); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an “AS IS” BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "livekit/audio_context.h" + +#include "modules/audio_mixer/audio_mixer_impl.h" +#include "rtc_base/thread.h" + +namespace livekit { + +AudioContext::AudioContext(std::shared_ptr rtc_runtime) + : rtc_runtime_(rtc_runtime) {} + +AudioContext::~AudioContext() { + if (audio_device_) { + rtc_runtime_->worker_thread()->BlockingCall( + [&] { audio_device_ = nullptr; }); + } + if (audio_mixer_) { + rtc_runtime_->worker_thread()->BlockingCall( + [&] { audio_mixer_ = nullptr; }); + } +} + +rtc::scoped_refptr AudioContext::audio_device( + webrtc::TaskQueueFactory* task_queue_factory) { + if (!audio_device_) { + audio_device_ = rtc_runtime_->worker_thread()->BlockingCall([&] { + return rtc::make_ref_counted(task_queue_factory); + }); + } + + return audio_device_; +} + +rtc::scoped_refptr AudioContext::audio_mixer() { + if (!audio_mixer_) { + audio_mixer_ = rtc_runtime_->worker_thread()->BlockingCall( + [&] { return webrtc::AudioMixerImpl::Create(); }); + } + return audio_mixer_; +} + +webrtc::AudioDeviceBuffer* AudioContext::audio_device_buffer() { + if (audio_device_) { + return audio_device_->audio_device_buffer(); + } + return nullptr; +} + +webrtc::AudioTransport* AudioContext::audio_transport() { + if (audio_device_) { + return audio_device_->audio_transport(); + } + return nullptr; +} + +} // namespace livekit \ No newline at end of file diff --git a/webrtc-sys/src/peer_connection_factory.cpp b/webrtc-sys/src/peer_connection_factory.cpp index 4ff83a90..c91261ff 100644 --- a/webrtc-sys/src/peer_connection_factory.cpp +++ b/webrtc-sys/src/peer_connection_factory.cpp @@ -41,7 +41,6 @@ #include "rtc_base/thread.h" #include "webrtc-sys/src/peer_connection.rs.h" #include "webrtc-sys/src/peer_connection_factory.rs.h" -#include "modules/audio_mixer/audio_mixer_impl.h" namespace livekit { @@ -49,7 +48,8 @@ class PeerConnectionObserver; PeerConnectionFactory::PeerConnectionFactory( std::shared_ptr rtc_runtime) - : rtc_runtime_(rtc_runtime) { + : rtc_runtime_(rtc_runtime), + audio_context_(rtc_runtime) { RTC_LOG(LS_VERBOSE) << "PeerConnectionFactory::PeerConnectionFactory()"; webrtc::PeerConnectionFactoryDependencies dependencies; @@ -66,12 +66,7 @@ PeerConnectionFactory::PeerConnectionFactory( cricket::MediaEngineDependencies media_deps; media_deps.task_queue_factory = dependencies.task_queue_factory.get(); - audio_device_ = rtc_runtime_->worker_thread()->BlockingCall([&] { - return rtc::make_ref_counted( - media_deps.task_queue_factory); - }); - - media_deps.adm = audio_device_; + media_deps.adm = audio_context_.audio_device(media_deps.task_queue_factory); media_deps.video_encoder_factory = std::move(std::make_unique()); @@ -87,9 +82,7 @@ PeerConnectionFactory::PeerConnectionFactory( apm.SetEchoControlFactory(std::move(echo_control)); media_deps.audio_processing = apm.Create(); - auto audio_mixer = webrtc::AudioMixerImpl::Create(); - - media_deps.audio_mixer = audio_mixer; + media_deps.audio_mixer = audio_context_.audio_mixer(); media_deps.trials = dependencies.trials.get(); @@ -111,8 +104,6 @@ PeerConnectionFactory::~PeerConnectionFactory() { RTC_LOG(LS_VERBOSE) << "PeerConnectionFactory::~PeerConnectionFactory()"; peer_factory_ = nullptr; - rtc_runtime_->worker_thread()->BlockingCall( - [this] { audio_device_ = nullptr; }); } std::shared_ptr PeerConnectionFactory::create_peer_connection( From 3cea202f98780b4a631c739b10c13f1756015b0c Mon Sep 17 00:00:00 2001 From: cloudwebrtc Date: Mon, 23 Sep 2024 11:04:05 +0800 Subject: [PATCH 4/5] received audio frame. --- examples/wgpu_room/src/app.rs | 13 +++++++++++-- webrtc-sys/src/audio_device.cpp | 4 ++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/examples/wgpu_room/src/app.rs b/examples/wgpu_room/src/app.rs index 99001f27..aaeb59db 100644 --- a/examples/wgpu_room/src/app.rs +++ b/examples/wgpu_room/src/app.rs @@ -5,7 +5,9 @@ use crate::{ }; use egui::{Rounding, Stroke}; use livekit::{e2ee::EncryptionType, prelude::*, SimulateScenario}; +use livekit::webrtc::{audio_stream::native::NativeAudioStream}; use std::collections::HashMap; +use futures::StreamExt; /// The state of the application are saved on app exit and restored on app start. #[derive(serde::Deserialize, serde::Serialize)] @@ -88,8 +90,15 @@ impl LkApp { ); self.video_renderers .insert((participant.identity(), track.sid()), video_renderer); - } else if let RemoteTrack::Audio(_) = track { - // TODO(theomonnom): Once we support media devices, we can play audio tracks here + } else if let RemoteTrack::Audio(ref audio_track) = track { + let rtc_track = audio_track.rtc_track(); + let mut audio_stream = NativeAudioStream::new(rtc_track, 48000, 2); + // Receive the audio frames in a new task + tokio::spawn(async move { + while let Some(frame) = audio_stream.next().await { + println!("Received audio frame {:?}", frame); + } + }); } } RoomEvent::TrackUnsubscribed { diff --git a/webrtc-sys/src/audio_device.cpp b/webrtc-sys/src/audio_device.cpp index 9ae77c18..f3877498 100644 --- a/webrtc-sys/src/audio_device.cpp +++ b/webrtc-sys/src/audio_device.cpp @@ -167,12 +167,14 @@ bool AudioDevice::RecordingIsInitialized() const { int32_t AudioDevice::StartPlayout() { webrtc::MutexLock lock(&mutex_); playing_ = true; + audio_device_buffer_.StartPlayout(); return 0; } int32_t AudioDevice::StopPlayout() { webrtc::MutexLock lock(&mutex_); playing_ = false; + audio_device_buffer_.StopPlayout(); return 0; } @@ -182,10 +184,12 @@ bool AudioDevice::Playing() const { } int32_t AudioDevice::StartRecording() { + audio_device_buffer_.StartRecording(); return 0; } int32_t AudioDevice::StopRecording() { + audio_device_buffer_.StopRecording(); return 0; } From 26813039e01e8b09a3531364dd775ce9ce5117b6 Mon Sep 17 00:00:00 2001 From: "duanweiwei1982@gmail.com" Date: Tue, 24 Sep 2024 10:21:13 +0800 Subject: [PATCH 5/5] fix crash. --- examples/wgpu_room/src/app.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/wgpu_room/src/app.rs b/examples/wgpu_room/src/app.rs index aaeb59db..4bf2efcd 100644 --- a/examples/wgpu_room/src/app.rs +++ b/examples/wgpu_room/src/app.rs @@ -94,7 +94,7 @@ impl LkApp { let rtc_track = audio_track.rtc_track(); let mut audio_stream = NativeAudioStream::new(rtc_track, 48000, 2); // Receive the audio frames in a new task - tokio::spawn(async move { + self.async_runtime.spawn(async move { while let Some(frame) = audio_stream.next().await { println!("Received audio frame {:?}", frame); }